Merge pull request #156 from scub-france/feature/ingestion-pipeline

feat: ingestion pipeline, My Documents, and ingest button (#72-#76)
This commit is contained in:
Pier-Jean Malandrino 2026-04-10 22:39:52 +02:00 committed by GitHub
commit b32781a055
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 1154 additions and 17 deletions

View file

@ -16,6 +16,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
- OpenSearch adapter (`OpenSearchStore`): kNN vector search, full-text search, bulk indexing, document CRUD
- Embedding microservice (`embedding-service/`): sentence-transformers REST API with batch processing and Dockerfile
- `EmbeddingService` port and `EmbeddingClient` HTTP adapter for remote embedding generation
- Orchestrated ingestion pipeline: Docling → chunking → embedding → OpenSearch indexing (idempotent)
- Ingestion REST API: `POST /api/ingestion/{jobId}`, `DELETE /api/ingestion/{docId}`, `GET /api/ingestion/status`
- Production docker-compose with OpenSearch and embedding service
- E2E Karate test for full ingestion workflow (PDF → chunks in OpenSearch)
- My Documents screen: search, filter (all/indexed/not indexed), sort (name/date), ingestion status badges
- Ingest button in Studio: one-click ingestion from completed analysis with progress feedback
### Fixed

View file

@ -1,4 +1,38 @@
services:
# --- OpenSearch (single-node, security disabled) ---
opensearch:
image: opensearchproject/opensearch:2
environment:
discovery.type: single-node
DISABLE_SECURITY_PLUGIN: "true"
OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
volumes:
- opensearch_data:/usr/share/opensearch/data
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
interval: 10s
timeout: 5s
retries: 10
# --- Embedding service (sentence-transformers) ---
embedding:
build:
context: ./embedding-service
environment:
EMBEDDING_MODEL: ${EMBEDDING_MODEL:-all-MiniLM-L6-v2}
EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:8001/health || exit 1"]
interval: 15s
timeout: 10s
retries: 20
start_period: 120s
deploy:
resources:
limits:
memory: 2g
# --- Backend (FastAPI) ---
document-parser:
build:
context: ./document-parser
@ -15,11 +49,19 @@ services:
RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100}
MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50}
BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-0}
OPENSEARCH_URL: http://opensearch:9200
EMBEDDING_URL: http://embedding:8001
depends_on:
opensearch:
condition: service_healthy
embedding:
condition: service_healthy
deploy:
resources:
limits:
memory: 4g
# --- Frontend (nginx) ---
frontend:
build:
context: ./frontend
@ -29,5 +71,6 @@ services:
- document-parser
volumes:
opensearch_data:
uploads_data:
db_data:

View file

@ -0,0 +1,82 @@
"""Ingestion API router — trigger and manage vector ingestion pipeline."""
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request
from api.schemas import IngestionResponse, IngestionStatusResponse
from services.analysis_service import AnalysisService
from services.ingestion_service import IngestionService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/ingestion", tags=["ingestion"])
def _get_ingestion_service(request: Request) -> IngestionService:
svc = request.app.state.ingestion_service
if svc is None:
raise HTTPException(
status_code=503,
detail="Ingestion not available (EMBEDDING_URL and OPENSEARCH_URL required)",
)
return svc
def _get_analysis_service(request: Request) -> AnalysisService:
return request.app.state.analysis_service
IngestionDep = Annotated[IngestionService, Depends(_get_ingestion_service)]
AnalysisDep = Annotated[AnalysisService, Depends(_get_analysis_service)]
@router.post("/{job_id}", response_model=IngestionResponse)
async def ingest_analysis(
job_id: str,
ingestion: IngestionDep,
analysis: AnalysisDep,
) -> IngestionResponse:
"""Ingest a completed analysis into the vector index.
Takes the chunks from an existing analysis job, embeds them,
and indexes them into OpenSearch.
"""
job = await analysis.find_by_id(job_id)
if not job:
raise HTTPException(status_code=404, detail="Analysis not found")
if job.status.value != "COMPLETED":
raise HTTPException(status_code=400, detail="Analysis is not completed")
if not job.chunks_json:
raise HTTPException(status_code=400, detail="Analysis has no chunks — run chunking first")
try:
result = await ingestion.ingest(
doc_id=job.document_id,
filename=job.document_filename or "unknown",
chunks_json=job.chunks_json,
)
except Exception as e:
logger.exception("Ingestion failed for job %s", job_id)
raise HTTPException(status_code=500, detail=f"Ingestion failed: {e}") from e
return IngestionResponse(
doc_id=result.doc_id,
chunks_indexed=result.chunks_indexed,
embedding_dimension=result.embedding_dimension,
)
@router.delete("/{doc_id}", status_code=204)
async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None:
"""Delete all indexed chunks for a document."""
await ingestion.delete_document(doc_id)
@router.get("/status", response_model=IngestionStatusResponse)
async def ingestion_status(request: Request) -> IngestionStatusResponse:
"""Check if the ingestion pipeline is available."""
available = request.app.state.ingestion_service is not None
return IngestionStatusResponse(available=available)

View file

@ -180,3 +180,13 @@ class RechunkRequest(BaseModel):
chunkingOptions: ChunkingOptionsRequest = Field(
validation_alias=AliasChoices("chunkingOptions", "chunking_options")
)
class IngestionResponse(_CamelModel):
doc_id: str
chunks_indexed: int
embedding_dimension: int
class IngestionStatusResponse(_CamelModel):
available: bool

View file

@ -20,6 +20,7 @@ from fastapi.middleware.cors import CORSMiddleware
from api.analyses import router as analyses_router
from api.documents import router as documents_router
from api.ingestion import router as ingestion_router
from api.schemas import HealthResponse
from infra.rate_limiter import RateLimiterMiddleware
from infra.settings import settings
@ -28,6 +29,7 @@ from persistence.database import get_connection, init_db
from persistence.document_repo import SqliteDocumentRepository
from services.analysis_service import AnalysisConfig, AnalysisService
from services.document_service import DocumentConfig, DocumentService
from services.ingestion_service import IngestionConfig, IngestionService
logging.basicConfig(
level=logging.INFO,
@ -87,6 +89,28 @@ def _build_analysis_service(
)
def _build_ingestion_service() -> IngestionService | None:
"""Build the ingestion service — only if embedding + opensearch are configured."""
if not settings.embedding_url or not settings.opensearch_url:
logger.info("Ingestion disabled (EMBEDDING_URL or OPENSEARCH_URL not set)")
return None
from infra.embedding_client import EmbeddingClient
from infra.opensearch_store import OpenSearchStore
embedding = EmbeddingClient(settings.embedding_url)
vector_store = OpenSearchStore(settings.opensearch_url)
config = IngestionConfig(
embedding_dimension=settings.embedding_dimension,
)
logger.info(
"Ingestion enabled (embedding=%s, opensearch=%s)",
settings.embedding_url,
settings.opensearch_url,
)
return IngestionService(embedding, vector_store, config)
def _build_document_service(
document_repo: SqliteDocumentRepository,
analysis_repo: SqliteAnalysisRepository,
@ -114,6 +138,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
document_repo, analysis_repo = _build_repos()
app.state.analysis_service = _build_analysis_service(document_repo, analysis_repo)
app.state.document_service = _build_document_service(document_repo, analysis_repo)
app.state.ingestion_service = _build_ingestion_service()
logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine)
yield
@ -128,7 +153,7 @@ app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origins,
allow_credentials=True,
allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"],
allow_headers=["Content-Type", "Authorization"],
)
if settings.rate_limit_rpm > 0:
@ -140,6 +165,7 @@ if settings.rate_limit_rpm > 0:
app.include_router(documents_router)
app.include_router(analyses_router)
app.include_router(ingestion_router)
@app.get("/api/health", response_model=HealthResponse)

View file

@ -0,0 +1,166 @@
"""Ingestion service — orchestrates Docling → embedding → OpenSearch.
Chains the full ingestion pipeline:
1. Convert document via Docling (reuse existing analysis)
2. Chunk with selected strategy
3. Embed all chunk texts via EmbeddingService
4. Index into OpenSearch via VectorStore
Idempotent: re-ingesting a document deletes old chunks before re-indexing.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass
from typing import TYPE_CHECKING
from domain.vector_schema import (
ChunkBboxEntry,
ChunkOrigin,
IndexedChunk,
build_index_mapping,
)
if TYPE_CHECKING:
from domain.ports import EmbeddingService, VectorStore
logger = logging.getLogger(__name__)
@dataclass
class IngestionConfig:
"""Configuration for the ingestion pipeline."""
index_name: str = "docling-studio-chunks"
embedding_dimension: int = 384
@dataclass
class IngestionResult:
"""Result of an ingestion pipeline run."""
doc_id: str
chunks_indexed: int
embedding_dimension: int
class IngestionService:
"""Orchestrates the embedding + indexing pipeline."""
def __init__(
self,
embedding_service: EmbeddingService,
vector_store: VectorStore,
config: IngestionConfig | None = None,
) -> None:
self._embedding = embedding_service
self._vector_store = vector_store
self._config = config or IngestionConfig()
async def ensure_index(self) -> None:
"""Ensure the vector index exists with the correct mapping."""
mapping = build_index_mapping(self._config.embedding_dimension)
await self._vector_store.ensure_index(self._config.index_name, mapping)
async def ingest(
self,
doc_id: str,
filename: str,
chunks_json: str,
*,
binary_hash: str | None = None,
) -> IngestionResult:
"""Run the embedding + indexing pipeline on pre-chunked data.
This method is idempotent: it deletes any existing chunks for the
document before re-indexing.
Args:
doc_id: Unique document identifier.
filename: Original filename.
chunks_json: JSON-serialized list of chunk dicts (from analysis).
binary_hash: Optional hash of the source file for provenance.
Returns:
IngestionResult with the number of chunks indexed.
"""
await self.ensure_index()
chunks_data: list[dict] = json.loads(chunks_json)
active_chunks = [c for c in chunks_data if not c.get("deleted")]
if not active_chunks:
logger.info("No active chunks for doc %s — skipping ingestion", doc_id)
return IngestionResult(doc_id=doc_id, chunks_indexed=0, embedding_dimension=0)
# 1. Embed all chunk texts
texts = [c["text"] for c in active_chunks]
logger.info("Embedding %d chunks for doc %s", len(texts), doc_id)
embeddings = await self._embedding.embed(texts)
# 2. Build IndexedChunk domain objects
origin = (
ChunkOrigin(binary_hash=binary_hash or "", filename=filename) if binary_hash else None
)
indexed_chunks: list[IndexedChunk] = []
for i, (chunk_data, embedding) in enumerate(zip(active_chunks, embeddings, strict=True)):
bboxes = [
ChunkBboxEntry(
page=b["page"],
x=b["bbox"][0] if b.get("bbox") else 0,
y=b["bbox"][1] if b.get("bbox") else 0,
w=(b["bbox"][2] - b["bbox"][0]) if b.get("bbox") and len(b["bbox"]) >= 4 else 0,
h=(b["bbox"][3] - b["bbox"][1]) if b.get("bbox") and len(b["bbox"]) >= 4 else 0,
)
for b in chunk_data.get("bboxes", [])
]
indexed_chunks.append(
IndexedChunk(
doc_id=doc_id,
filename=filename,
content=chunk_data["text"],
embedding=embedding,
chunk_index=i,
chunk_type=chunk_data.get("chunkType", "text"),
page_number=chunk_data.get("sourcePage", 0) or 0,
bboxes=bboxes,
headings=chunk_data.get("headings", []),
origin=origin,
)
)
# 3. Delete old chunks (idempotent re-indexing)
deleted = await self._vector_store.delete_document(self._config.index_name, doc_id)
if deleted:
logger.info("Deleted %d old chunks for doc %s", deleted, doc_id)
# 4. Index new chunks
indexed = await self._vector_store.index_chunks(self._config.index_name, indexed_chunks)
logger.info("Indexed %d/%d chunks for doc %s", indexed, len(indexed_chunks), doc_id)
return IngestionResult(
doc_id=doc_id,
chunks_indexed=indexed,
embedding_dimension=len(embeddings[0]) if embeddings else 0,
)
async def delete_document(self, doc_id: str) -> int:
"""Remove all indexed chunks for a document."""
return await self._vector_store.delete_document(self._config.index_name, doc_id)
async def search(
self,
query: str,
*,
k: int = 10,
doc_id: str | None = None,
) -> list:
"""Semantic search: embed the query then find nearest chunks."""
embeddings = await self._embedding.embed([query])
return await self._vector_store.search_similar(
self._config.index_name,
embeddings[0],
k=k,
doc_id=doc_id,
)

View file

@ -0,0 +1,114 @@
"""Tests for the ingestion API endpoints (api.ingestion)."""
from __future__ import annotations
from unittest.mock import AsyncMock
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from api.ingestion import router
from domain.models import AnalysisJob
from services.ingestion_service import IngestionResult
@pytest.fixture
def mock_ingestion_service() -> AsyncMock:
svc = AsyncMock()
svc.ingest.return_value = IngestionResult(
doc_id="doc-1", chunks_indexed=5, embedding_dimension=384
)
svc.delete_document.return_value = 3
return svc
@pytest.fixture
def mock_analysis_service() -> AsyncMock:
svc = AsyncMock()
job = AnalysisJob(document_id="doc-1")
job.document_filename = "test.pdf"
job.mark_running()
job.mark_completed(
markdown="# Test",
html="<h1>Test</h1>",
pages_json="[]",
document_json='{"doc": true}',
chunks_json='[{"text": "hello"}]',
)
svc.find_by_id.return_value = job
return svc
@pytest.fixture
def client(mock_ingestion_service: AsyncMock, mock_analysis_service: AsyncMock) -> TestClient:
app = FastAPI()
app.include_router(router)
app.state.ingestion_service = mock_ingestion_service
app.state.analysis_service = mock_analysis_service
return TestClient(app)
class TestIngestAnalysis:
def test_ingest_success(self, client: TestClient) -> None:
resp = client.post("/api/ingestion/job-1")
assert resp.status_code == 200
data = resp.json()
assert data["docId"] == "doc-1"
assert data["chunksIndexed"] == 5
assert data["embeddingDimension"] == 384
def test_ingest_not_found(self, client: TestClient, mock_analysis_service: AsyncMock) -> None:
mock_analysis_service.find_by_id.return_value = None
resp = client.post("/api/ingestion/missing")
assert resp.status_code == 404
def test_ingest_not_completed(
self, client: TestClient, mock_analysis_service: AsyncMock
) -> None:
job = AnalysisJob(document_id="doc-1")
mock_analysis_service.find_by_id.return_value = job
resp = client.post("/api/ingestion/job-1")
assert resp.status_code == 400
def test_ingest_no_chunks(self, client: TestClient, mock_analysis_service: AsyncMock) -> None:
job = AnalysisJob(document_id="doc-1")
job.mark_running()
job.mark_completed(markdown="x", html="x", pages_json="[]")
mock_analysis_service.find_by_id.return_value = job
resp = client.post("/api/ingestion/job-1")
assert resp.status_code == 400
class TestDeleteIngested:
def test_delete_success(self, client: TestClient) -> None:
resp = client.delete("/api/ingestion/doc-1")
assert resp.status_code == 204
class TestIngestionStatus:
def test_available(self, client: TestClient) -> None:
resp = client.get("/api/ingestion/status")
assert resp.status_code == 200
assert resp.json()["available"] is True
def test_not_available(self) -> None:
app = FastAPI()
app.include_router(router)
app.state.ingestion_service = None
app.state.analysis_service = AsyncMock()
tc = TestClient(app)
resp = tc.get("/api/ingestion/status")
assert resp.status_code == 200
assert resp.json()["available"] is False
class TestIngestionDisabled:
def test_returns_503_when_disabled(self) -> None:
app = FastAPI()
app.include_router(router)
app.state.ingestion_service = None
app.state.analysis_service = AsyncMock()
tc = TestClient(app)
resp = tc.post("/api/ingestion/job-1")
assert resp.status_code == 503

View file

@ -0,0 +1,150 @@
"""Tests for the ingestion service (services.ingestion_service)."""
from __future__ import annotations
import json
from unittest.mock import AsyncMock
import pytest
from services.ingestion_service import IngestionConfig, IngestionService
def _make_chunks_json(count: int = 3, *, with_deleted: bool = False) -> str:
chunks = []
for i in range(count):
chunk = {
"text": f"chunk text {i}",
"headings": [f"Heading {i}"],
"sourcePage": i + 1,
"tokenCount": 10,
"bboxes": [{"page": i + 1, "bbox": [0.0, 0.0, 100.0, 50.0]}],
}
if with_deleted and i == count - 1:
chunk["deleted"] = True
chunks.append(chunk)
return json.dumps(chunks)
@pytest.fixture
def mock_embedding() -> AsyncMock:
svc = AsyncMock()
svc.embed.return_value = [[0.1, 0.2, 0.3]] * 3
return svc
@pytest.fixture
def mock_vector_store() -> AsyncMock:
store = AsyncMock()
store.ensure_index.return_value = None
store.delete_document.return_value = 0
store.index_chunks.return_value = 3
return store
@pytest.fixture
def service(mock_embedding: AsyncMock, mock_vector_store: AsyncMock) -> IngestionService:
return IngestionService(
embedding_service=mock_embedding,
vector_store=mock_vector_store,
config=IngestionConfig(index_name="test-idx", embedding_dimension=3),
)
class TestIngest:
async def test_full_pipeline(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
result = await service.ingest("doc-1", "test.pdf", _make_chunks_json(3))
assert result.doc_id == "doc-1"
assert result.chunks_indexed == 3
mock_embedding.embed.assert_awaited_once()
texts = mock_embedding.embed.call_args[0][0]
assert len(texts) == 3
mock_vector_store.ensure_index.assert_awaited_once()
mock_vector_store.delete_document.assert_awaited_once_with("test-idx", "doc-1")
mock_vector_store.index_chunks.assert_awaited_once()
indexed = mock_vector_store.index_chunks.call_args[0][1]
assert len(indexed) == 3
assert indexed[0].doc_id == "doc-1"
assert indexed[0].filename == "test.pdf"
assert indexed[0].embedding == [0.1, 0.2, 0.3]
async def test_skips_deleted_chunks(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
mock_embedding.embed.return_value = [[0.1, 0.2, 0.3]] * 2
mock_vector_store.index_chunks.return_value = 2
result = await service.ingest("doc-1", "test.pdf", _make_chunks_json(3, with_deleted=True))
assert result.chunks_indexed == 2
texts = mock_embedding.embed.call_args[0][0]
assert len(texts) == 2
async def test_empty_chunks(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
result = await service.ingest("doc-1", "test.pdf", json.dumps([]))
assert result.chunks_indexed == 0
mock_embedding.embed.assert_not_awaited()
async def test_idempotent_deletes_old(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
mock_vector_store.delete_document.return_value = 5
await service.ingest("doc-1", "test.pdf", _make_chunks_json(3))
mock_vector_store.delete_document.assert_awaited_once_with("test-idx", "doc-1")
async def test_bbox_conversion(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
mock_embedding.embed.return_value = [[0.1, 0.2, 0.3]]
mock_vector_store.index_chunks.return_value = 1
await service.ingest("doc-1", "test.pdf", _make_chunks_json(1))
indexed = mock_vector_store.index_chunks.call_args[0][1]
bbox = indexed[0].bboxes[0]
assert bbox.x == 0.0
assert bbox.y == 0.0
assert bbox.w == 100.0
assert bbox.h == 50.0
async def test_with_binary_hash(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
mock_embedding = service._embedding
mock_embedding.embed.return_value = [[0.1]] * 1
await service.ingest("doc-1", "test.pdf", _make_chunks_json(1), binary_hash="abc123")
indexed = mock_vector_store.index_chunks.call_args[0][1]
assert indexed[0].origin is not None
assert indexed[0].origin.binary_hash == "abc123"
class TestDeleteDocument:
async def test_delegates_to_vector_store(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
mock_vector_store.delete_document.return_value = 3
result = await service.delete_document("doc-1")
assert result == 3
class TestSearch:
async def test_embeds_and_searches(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
mock_embedding.embed.return_value = [[0.5, 0.6, 0.7]]
mock_vector_store.search_similar.return_value = []
await service.search("test query", k=5)
mock_embedding.embed.assert_awaited_once_with(["test query"])
mock_vector_store.search_similar.assert_awaited_once()
class TestEnsureIndex:
async def test_calls_vector_store(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
await service.ensure_index()
mock_vector_store.ensure_index.assert_awaited_once()
call_args = mock_vector_store.ensure_index.call_args
assert call_args[0][0] == "test-idx"

View file

@ -0,0 +1,59 @@
@e2e @ingestion
Feature: Ingestion pipeline — PDF → chunks → embeddings → OpenSearch
Background:
* url baseUrl
Scenario: Upload PDF, analyze with chunking, ingest into OpenSearch, verify
# Step 1: Check ingestion is available
Given path '/api/ingestion/status'
When method GET
Then status 200
And match response.available == true
# Step 2: Upload a PDF
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/medium.pdf', filename: 'medium.pdf', contentType: 'application/pdf' }
When method POST
Then status 200
* def docId = response.id
# Step 3: Create analysis with chunking
Given path '/api/analyses'
And request { documentId: '#(docId)', pipelineOptions: { doOcr: true, tableMode: 'fast' }, chunkingOptions: { chunkerType: 'hybrid', maxTokens: 256 } }
When method POST
Then status 200
* def jobId = response.id
# Step 4: Poll until completed
Given path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
And match response.status == 'COMPLETED'
And match response.chunksJson == '#string'
# Step 5: Trigger ingestion
Given path '/api/ingestion', jobId
When method POST
Then status 200
And match response.docId == docId
And match response.chunksIndexed == '#number'
And assert response.chunksIndexed > 0
And match response.embeddingDimension == '#number'
And assert response.embeddingDimension > 0
# Step 6: Cleanup — delete ingested data
Given path '/api/ingestion', docId
When method DELETE
Then status 204
# Step 7: Cleanup — delete analysis and document
Given path '/api/analyses', jobId
When method DELETE
Then status 204
Given path '/api/documents', docId
When method DELETE
Then status 204

View file

@ -2,6 +2,8 @@ FROM python:3.12-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
# Install dependencies first (cache layer)
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

View file

@ -0,0 +1,48 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { ingestAnalysis, deleteIngested, fetchIngestionStatus } from './api'
const mockFetch = vi.fn()
vi.stubGlobal('fetch', mockFetch)
beforeEach(() => {
mockFetch.mockReset()
})
describe('ingestAnalysis', () => {
it('posts to /api/ingestion/:jobId', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ docId: 'doc-1', chunksIndexed: 5, embeddingDimension: 384 }),
})
const result = await ingestAnalysis('job-1')
expect(mockFetch).toHaveBeenCalledWith(
'/api/ingestion/job-1',
expect.objectContaining({ method: 'POST' }),
)
expect(result.chunksIndexed).toBe(5)
})
})
describe('deleteIngested', () => {
it('deletes /api/ingestion/:docId', async () => {
mockFetch.mockResolvedValue({ ok: true, status: 204, json: () => Promise.resolve(null) })
await deleteIngested('doc-1')
expect(mockFetch).toHaveBeenCalledWith(
'/api/ingestion/doc-1',
expect.objectContaining({ method: 'DELETE' }),
)
})
})
describe('fetchIngestionStatus', () => {
it('gets /api/ingestion/status', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ available: true }),
})
const result = await fetchIngestionStatus()
expect(result.available).toBe(true)
})
})

View file

@ -0,0 +1,25 @@
import { apiFetch } from '../../shared/api/http'
export interface IngestionResult {
docId: string
chunksIndexed: number
embeddingDimension: number
}
export interface IngestionStatus {
available: boolean
}
export function ingestAnalysis(jobId: string): Promise<IngestionResult> {
return apiFetch<IngestionResult>(`/api/ingestion/${jobId}`, {
method: 'POST',
})
}
export function deleteIngested(docId: string): Promise<unknown> {
return apiFetch(`/api/ingestion/${docId}`, { method: 'DELETE' })
}
export function fetchIngestionStatus(): Promise<IngestionStatus> {
return apiFetch<IngestionStatus>('/api/ingestion/status')
}

View file

@ -0,0 +1 @@
export { useIngestionStore } from './store'

View file

@ -0,0 +1,66 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { setActivePinia, createPinia } from 'pinia'
import { useIngestionStore } from './store'
import * as api from './api'
vi.mock('./api', () => ({
fetchIngestionStatus: vi.fn(),
ingestAnalysis: vi.fn(),
deleteIngested: vi.fn(),
}))
beforeEach(() => {
setActivePinia(createPinia())
vi.clearAllMocks()
})
describe('useIngestionStore', () => {
describe('checkAvailability', () => {
it('sets available to true when API responds', async () => {
vi.mocked(api.fetchIngestionStatus).mockResolvedValue({ available: true })
const store = useIngestionStore()
await store.checkAvailability()
expect(store.available).toBe(true)
})
it('sets available to false on error', async () => {
vi.mocked(api.fetchIngestionStatus).mockRejectedValue(new Error('fail'))
const store = useIngestionStore()
await store.checkAvailability()
expect(store.available).toBe(false)
})
})
describe('ingest', () => {
it('calls API and tracks ingested doc', async () => {
vi.mocked(api.ingestAnalysis).mockResolvedValue({
docId: 'doc-1',
chunksIndexed: 5,
embeddingDimension: 384,
})
const store = useIngestionStore()
const result = await store.ingest('job-1')
expect(result?.chunksIndexed).toBe(5)
expect(store.ingestedDocs['doc-1']).toBe(5)
expect(store.ingesting).toBe(false)
})
it('sets error on failure', async () => {
vi.mocked(api.ingestAnalysis).mockRejectedValue(new Error('fail'))
const store = useIngestionStore()
const result = await store.ingest('job-1')
expect(result).toBeNull()
expect(store.error).toBe('fail')
})
})
describe('deleteIngested', () => {
it('removes doc from tracked map', async () => {
vi.mocked(api.deleteIngested).mockResolvedValue(null)
const store = useIngestionStore()
store.ingestedDocs['doc-1'] = 5
await store.deleteIngested('doc-1')
expect(store.ingestedDocs['doc-1']).toBeUndefined()
})
})
})

View file

@ -0,0 +1,56 @@
import { defineStore } from 'pinia'
import { ref } from 'vue'
import * as api from './api'
export const useIngestionStore = defineStore('ingestion', () => {
const available = ref(false)
const ingesting = ref(false)
const error = ref<string | null>(null)
/** Map of docId → chunks indexed count (tracks which docs are ingested) */
const ingestedDocs = ref<Record<string, number>>({})
async function checkAvailability(): Promise<void> {
try {
const status = await api.fetchIngestionStatus()
available.value = status.available
} catch {
available.value = false
}
}
async function ingest(jobId: string): Promise<api.IngestionResult | null> {
ingesting.value = true
error.value = null
try {
const result = await api.ingestAnalysis(jobId)
ingestedDocs.value[result.docId] = result.chunksIndexed
return result
} catch (e) {
error.value = (e as Error).message || 'Ingestion failed'
console.error('Ingestion failed', e)
return null
} finally {
ingesting.value = false
}
}
async function deleteIngested(docId: string): Promise<void> {
try {
await api.deleteIngested(docId)
delete ingestedDocs.value[docId]
} catch (e) {
error.value = (e as Error).message || 'Failed to delete ingested data'
console.error('Failed to delete ingested data', e)
}
}
return {
available,
ingesting,
error,
ingestedDocs,
checkAvailability,
ingest,
deleteIngested,
}
})

View file

@ -2,13 +2,40 @@
<div class="documents-page">
<div class="page-header">
<h1 class="page-title">{{ t('nav.documents') }}</h1>
<div class="header-actions">
<input
v-model="searchQuery"
type="text"
class="search-input"
:placeholder="t('ingestion.search')"
/>
<div class="filter-group">
<button
v-for="f in filters"
:key="f.value"
class="filter-btn"
:class="{ active: activeFilter === f.value }"
@click="activeFilter = f.value"
>
{{ f.label }}
</button>
</div>
<div class="sort-group">
<button class="sort-btn" :class="{ active: sortBy === 'name' }" @click="sortBy = 'name'">
{{ t('ingestion.sortName') }}
</button>
<button class="sort-btn" :class="{ active: sortBy === 'date' }" @click="sortBy = 'date'">
{{ t('ingestion.sortDate') }}
</button>
</div>
</div>
</div>
<div class="page-content">
<div v-if="docStore.documents.length === 0" class="tab-empty">
<div v-if="filteredDocs.length === 0" class="tab-empty">
{{ t('history.emptyDocs') }}
</div>
<div v-else class="doc-items">
<div v-for="doc in docStore.documents" :key="doc.id" class="doc-row">
<div v-for="doc in filteredDocs" :key="doc.id" class="doc-row">
<div class="doc-row-info">
<svg class="doc-row-icon" viewBox="0 0 20 20" fill="currentColor">
<path
@ -26,15 +53,39 @@
</span>
</div>
</div>
<button class="doc-row-delete" @click="docStore.remove(doc.id)">
<svg viewBox="0 0 20 20" fill="currentColor">
<path
fill-rule="evenodd"
d="M9 2a1 1 0 00-.894.553L7.382 4H4a1 1 0 000 2v10a2 2 0 002 2h8a2 2 0 002-2V6a1 1 0 100-2h-3.382l-.724-1.447A1 1 0 0011 2H9zM7 8a1 1 0 012 0v6a1 1 0 11-2 0V8zm5-1a1 1 0 00-1 1v6a1 1 0 102 0V8a1 1 0 00-1-1z"
clip-rule="evenodd"
/>
</svg>
</button>
<div class="doc-row-actions">
<span
v-if="ingestionStore.ingestedDocs[doc.id]"
class="status-badge indexed"
:title="t('ingestion.chunksIndexed', { n: ingestionStore.ingestedDocs[doc.id] })"
>
{{ t('ingestion.indexed') }}
<span class="badge-count">{{ ingestionStore.ingestedDocs[doc.id] }}</span>
</span>
<span v-else class="status-badge not-indexed">
{{ t('ingestion.notIndexed') }}
</span>
<button
class="action-btn"
:title="t('ingestion.openInStudio')"
@click="openInStudio(doc)"
>
<svg viewBox="0 0 20 20" fill="currentColor">
<path
d="M10.394 2.08a1 1 0 00-.788 0l-7 3a1 1 0 000 1.84L5.25 8.051a.999.999 0 01.356-.257l4-1.714a1 1 0 11.788 1.838l-2.727 1.17 1.94.831a1 1 0 00.787 0l7-3a1 1 0 000-1.838l-7-3z"
/>
</svg>
</button>
<button class="action-btn delete" @click="handleDelete(doc.id)">
<svg viewBox="0 0 20 20" fill="currentColor">
<path
fill-rule="evenodd"
d="M9 2a1 1 0 00-.894.553L7.382 4H4a1 1 0 000 2v10a2 2 0 002 2h8a2 2 0 002-2V6a1 1 0 100-2h-3.382l-.724-1.447A1 1 0 0011 2H9zM7 8a1 1 0 012 0v6a1 1 0 11-2 0V8zm5-1a1 1 0 00-1 1v6a1 1 0 102 0V8a1 1 0 00-1-1z"
clip-rule="evenodd"
/>
</svg>
</button>
</div>
</div>
</div>
</div>
@ -42,21 +93,75 @@
</template>
<script setup lang="ts">
import { onMounted } from 'vue'
import { computed, onMounted, ref } from 'vue'
import { useRouter } from 'vue-router'
import { useDocumentStore } from '../features/document/store'
import { useIngestionStore } from '../features/ingestion/store'
import { useI18n } from '../shared/i18n'
import { formatSize } from '../shared/format'
import type { Document } from '../shared/types'
const docStore = useDocumentStore()
const ingestionStore = useIngestionStore()
const router = useRouter()
const { t } = useI18n()
const searchQuery = ref('')
const activeFilter = ref<'all' | 'indexed' | 'not-indexed'>('all')
const sortBy = ref<'name' | 'date'>('date')
const filters = computed(() => [
{ value: 'all' as const, label: t('ingestion.filterAll') },
{ value: 'indexed' as const, label: t('ingestion.filterIndexed') },
{ value: 'not-indexed' as const, label: t('ingestion.filterNotIndexed') },
])
const filteredDocs = computed(() => {
let docs = [...docStore.documents]
// Search filter
if (searchQuery.value.trim()) {
const q = searchQuery.value.toLowerCase()
docs = docs.filter((d) => d.filename.toLowerCase().includes(q))
}
// Status filter
if (activeFilter.value === 'indexed') {
docs = docs.filter((d) => ingestionStore.ingestedDocs[d.id])
} else if (activeFilter.value === 'not-indexed') {
docs = docs.filter((d) => !ingestionStore.ingestedDocs[d.id])
}
// Sort
if (sortBy.value === 'name') {
docs.sort((a, b) => a.filename.localeCompare(b.filename))
} else {
docs.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
}
return docs
})
function formatDate(iso: string) {
if (!iso) return ''
return new Date(iso).toLocaleString()
}
function openInStudio(doc: Document) {
docStore.select(doc.id)
router.push('/studio')
}
async function handleDelete(docId: string) {
if (ingestionStore.ingestedDocs[docId]) {
await ingestionStore.deleteIngested(docId)
}
await docStore.remove(docId)
}
onMounted(() => {
docStore.load()
ingestionStore.checkAvailability()
})
</script>
@ -72,6 +177,11 @@ onMounted(() => {
padding: 16px 24px;
border-bottom: 1px solid var(--border);
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
flex-wrap: wrap;
}
.page-title {
@ -80,6 +190,57 @@ onMounted(() => {
color: var(--text);
}
.header-actions {
display: flex;
align-items: center;
gap: 12px;
}
.search-input {
padding: 6px 12px;
border: 1px solid var(--border);
border-radius: var(--radius-sm);
background: var(--bg);
color: var(--text);
font-size: 13px;
width: 180px;
outline: none;
transition: border-color var(--transition);
}
.search-input:focus {
border-color: var(--accent);
}
.filter-group,
.sort-group {
display: flex;
gap: 2px;
background: var(--bg-surface);
border-radius: var(--radius-sm);
padding: 2px;
border: 1px solid var(--border);
}
.filter-btn,
.sort-btn {
padding: 4px 10px;
border: none;
background: none;
color: var(--text-secondary);
font-size: 12px;
font-weight: 500;
border-radius: 4px;
cursor: pointer;
transition: all var(--transition);
}
.filter-btn.active,
.sort-btn.active {
background: var(--accent);
color: white;
}
.page-content {
flex: 1;
overflow-y: auto;
@ -152,7 +313,41 @@ onMounted(() => {
font-family: 'IBM Plex Mono', monospace;
}
.doc-row-delete {
.doc-row-actions {
display: flex;
align-items: center;
gap: 8px;
flex-shrink: 0;
}
.status-badge {
display: inline-flex;
align-items: center;
gap: 4px;
padding: 3px 8px;
border-radius: 10px;
font-size: 11px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.03em;
}
.status-badge.indexed {
background: rgba(34, 197, 94, 0.15);
color: var(--success);
}
.status-badge.not-indexed {
background: rgba(156, 163, 175, 0.15);
color: var(--text-muted);
}
.badge-count {
font-family: 'IBM Plex Mono', monospace;
font-size: 10px;
}
.action-btn {
background: none;
border: none;
padding: 6px;
@ -164,14 +359,21 @@ onMounted(() => {
transition: all var(--transition);
}
.doc-row:hover .doc-row-delete {
.doc-row:hover .action-btn {
opacity: 1;
}
.doc-row-delete:hover {
.action-btn:hover {
color: var(--accent);
background: rgba(249, 115, 22, 0.1);
}
.action-btn.delete:hover {
color: var(--error);
background: rgba(239, 68, 68, 0.1);
}
.doc-row-delete svg {
.action-btn svg {
width: 16px;
height: 16px;
}

View file

@ -94,6 +94,23 @@
</svg>
{{ analysisStore.running ? t('studio.analyzing') : t('studio.run') }}
</button>
<button
v-if="canIngest"
class="topbar-btn ingest"
data-e2e="ingest-btn"
:disabled="ingestionStore.ingesting"
@click="runIngestion"
>
<div v-if="ingestionStore.ingesting" class="spinner-sm" />
<svg v-else viewBox="0 0 20 20" fill="currentColor" class="btn-icon">
<path
fill-rule="evenodd"
d="M3 17a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1zM6.293 6.707a1 1 0 010-1.414l3-3a1 1 0 011.414 0l3 3a1 1 0 01-1.414 1.414L11 5.414V13a1 1 0 11-2 0V5.414L7.707 6.707a1 1 0 01-1.414 0z"
clip-rule="evenodd"
/>
</svg>
{{ ingestionStore.ingesting ? t('ingestion.ingesting') : t('ingestion.ingest') }}
</button>
</div>
</div>
@ -459,6 +476,7 @@ import { ref, computed, watch, nextTick, onMounted, onBeforeUnmount, reactive }
import { useRoute, useRouter } from 'vue-router'
import { useDocumentStore } from '../features/document/store'
import { useAnalysisStore } from '../features/analysis/store'
import { useIngestionStore } from '../features/ingestion/store'
import { DocumentUpload, DocumentList } from '../features/document/index'
import { ResultTabs } from '../features/analysis/index'
import BboxOverlay from '../features/analysis/ui/BboxOverlay.vue'
@ -472,6 +490,7 @@ const route = useRoute()
const router = useRouter()
const documentStore = useDocumentStore()
const analysisStore = useAnalysisStore()
const ingestionStore = useIngestionStore()
const { t } = useI18n()
const chunkingEnabled = useFeatureFlag('chunking')
@ -528,6 +547,14 @@ const pipelineOptions = reactive<PipelineOptions>({
images_scale: 1.0,
})
const canIngest = computed(() => {
return (
ingestionStore.available &&
analysisStore.currentAnalysis?.status === 'COMPLETED' &&
analysisStore.currentAnalysis?.chunksJson != null
)
})
const hasAnalysisResults = computed(() => {
return (
analysisStore.currentAnalysis?.status === 'COMPLETED' && analysisStore.currentPages?.length > 0
@ -564,6 +591,11 @@ async function runAnalysis() {
await analysisStore.run(documentStore.selectedId, { ...pipelineOptions })
}
async function runIngestion() {
if (!analysisStore.currentAnalysis?.id) return
await ingestionStore.ingest(analysisStore.currentAnalysis.id)
}
function addMore() {
documentStore.selectedId = null
}
@ -598,6 +630,7 @@ watch(
onMounted(async () => {
await documentStore.load()
analysisStore.load()
ingestionStore.checkAvailability()
// Restore analysis from history via query param
const analysisId = route.query.analysisId
@ -811,6 +844,21 @@ onBeforeUnmount(() => {
cursor: not-allowed;
}
.topbar-btn.ingest {
background: var(--success);
border-color: var(--success);
color: white;
}
.topbar-btn.ingest:hover:not(:disabled) {
filter: brightness(1.1);
}
.topbar-btn.ingest:disabled {
opacity: 0.6;
cursor: not-allowed;
}
.topbar-btn .btn-icon {
width: 16px;
height: 16px;

View file

@ -131,6 +131,23 @@ const messages: Messages = {
'chunking.batchNotice':
'Le chunking n\u2019est pas disponible pour cette analyse. Les documents volumineux trait\u00e9s par batch ne g\u00e9n\u00e8rent pas la structure interne n\u00e9cessaire au d\u00e9coupage.',
// Ingestion / My Documents
'ingestion.ingest': 'Ingérer',
'ingestion.ingesting': 'Ingestion...',
'ingestion.reindex': 'Ré-indexer',
'ingestion.indexed': 'Indexé',
'ingestion.notIndexed': 'Non indexé',
'ingestion.chunksIndexed': '{n} chunks indexés',
'ingestion.openInStudio': 'Ouvrir dans le Studio',
'ingestion.deleteIndex': "Supprimer de l'index",
'ingestion.unavailable': 'Ingestion non disponible',
'ingestion.filterAll': 'Tous',
'ingestion.filterIndexed': 'Indexés',
'ingestion.filterNotIndexed': 'Non indexés',
'ingestion.sortName': 'Nom',
'ingestion.sortDate': 'Date',
'ingestion.search': 'Rechercher...',
// Pagination
'pagination.pageOf': 'Page {current} sur {total}',
'pagination.perPage': '/ page',
@ -266,6 +283,22 @@ const messages: Messages = {
'chunking.batchNotice':
'Chunking is not available for this analysis. Large documents processed in batch mode do not generate the internal structure required for chunking.',
'ingestion.ingest': 'Ingest',
'ingestion.ingesting': 'Ingesting...',
'ingestion.reindex': 'Re-index',
'ingestion.indexed': 'Indexed',
'ingestion.notIndexed': 'Not indexed',
'ingestion.chunksIndexed': '{n} chunks indexed',
'ingestion.openInStudio': 'Open in Studio',
'ingestion.deleteIndex': 'Remove from index',
'ingestion.unavailable': 'Ingestion unavailable',
'ingestion.filterAll': 'All',
'ingestion.filterIndexed': 'Indexed',
'ingestion.filterNotIndexed': 'Not indexed',
'ingestion.sortName': 'Name',
'ingestion.sortDate': 'Date',
'ingestion.search': 'Search...',
'pagination.pageOf': 'Page {current} of {total}',
'pagination.perPage': '/ page',