diff --git a/.env.example b/.env.example index 832eb86..b423dce 100644 --- a/.env.example +++ b/.env.example @@ -36,3 +36,15 @@ # Database path (inside container) # DB_PATH=./data/docling_studio.db + +# OpenSearch URL (used by docker-compose.dev.yml, auto-set to service name) +# OPENSEARCH_URL=http://opensearch:9200 + +# Embedding service URL (used by docker-compose.dev.yml, auto-set to service name) +# EMBEDDING_URL=http://embedding:8001 + +# Embedding model (default: all-MiniLM-L6-v2, used by the embedding service) +# EMBEDDING_MODEL=all-MiniLM-L6-v2 + +# Embedding vector dimension (default: 384 for Granite Embedding 30M / all-MiniLM-L6-v2) +# EMBEDDING_DIMENSION=384 diff --git a/.gitignore b/.gitignore index eb53104..a1e287a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,8 @@ hs_err_pid* # Docker docker-compose.override.yml +# Audit profiles (internal tooling) +profiles/ + # E2E tests — Maven build outputs & Chrome user data e2e/**/target/ diff --git a/CHANGELOG.md b/CHANGELOG.md index dcdb5a7..961bcb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to Docling Studio will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). +## [0.4.0] - 2026-04-13 + +### Added + +- Inline chunk text editing: double-click or edit button to modify chunk text, with save/cancel and "modified" badge +- Docker Compose dev stack (`docker-compose.dev.yml`) with OpenSearch, Dashboards, hot-reload backend and Vite frontend +- Soft-delete chunks: delete button with confirmation dialog, chunks hidden from UI but preserved in data +- Vector index metadata schema: `IndexedChunk` domain model, OpenSearch mapping builder, configurable embedding dimension +- `VectorStore` port (Protocol): `ensure_index`, `index_chunks`, `search_similar`, `get_chunks`, `delete_document` +- OpenSearch adapter (`OpenSearchStore`): kNN vector search, full-text search, bulk indexing, document CRUD +- Embedding microservice (`embedding-service/`): sentence-transformers REST API with batch processing and Dockerfile +- `EmbeddingService` port and `EmbeddingClient` HTTP adapter for remote embedding generation +- Orchestrated ingestion pipeline: Docling → chunking → embedding → OpenSearch indexing (idempotent) +- Ingestion REST API: `POST /api/ingestion/{jobId}`, `DELETE /api/ingestion/{docId}`, `GET /api/ingestion/status` +- Production docker-compose with OpenSearch and embedding service +- E2E Karate test for full ingestion workflow (PDF → chunks in OpenSearch) +- My Documents screen: search, filter (all/indexed/not indexed), sort (name/date), ingestion status badges +- Ingest button in Studio: one-click ingestion from completed analysis with progress feedback + +### Fixed + +### Changed + ## [0.3.1] - 2026-04-09 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index baef553..e10aeaa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,35 @@ Thank you for your interest in contributing to Docling Studio! This guide will h ## Development Setup +### Docker Dev Stack (recommended) + +The fastest way to get the full stack running (backend + frontend + OpenSearch): + +```bash +docker compose -f docker-compose.dev.yml up +``` + +This starts: + +| Service | URL | Notes | +|---------|-----|-------| +| Frontend (Vite) | http://localhost:3000 | HMR enabled | +| Backend (FastAPI) | http://localhost:8000 | Auto-reload on file changes | +| OpenSearch | http://localhost:9200 | Single-node, security disabled | +| OpenSearch Dashboards | http://localhost:5601 | Index inspection UI | + +Source code is bind-mounted — edits on your host are reflected immediately. + +To use remote conversion mode instead of local: + +```bash +CONVERSION_MODE=remote docker compose -f docker-compose.dev.yml up +``` + +### Manual Setup + +If you prefer running services directly on your machine: + ### Backend (Python 3.12+) ```bash @@ -67,15 +96,43 @@ npx prettier --write src/ # auto-format ## Running Tests ```bash -# Backend (199 tests) +# Backend (377 tests) cd document-parser pytest tests/ -v -# Frontend (129 tests) +# Frontend (156 tests) cd frontend npm run test:run ``` +### E2E API (Karate) + +```bash +# Generate test PDFs + start stack +python e2e/generate-test-data.py +docker compose up -d --wait + +# Run all API tests +mvn test -f e2e/api/pom.xml + +# Or by tag: @smoke, @regression, @e2e +mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @smoke" +``` + +### E2E UI (Karate UI) + +```bash +# Generate test PDFs + start stack (if not already running) +python e2e/generate-test-data.py +docker compose up -d --wait + +# Run critical UI tests (CI scope) +mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @critical" + +# Run all UI tests (local scope) +mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @ui" +``` + All tests must pass before submitting a PR. ## Submitting Changes diff --git a/README.md b/README.md index 801094e..eff9564 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,13 @@ Upload a PDF, configure the extraction pipeline, and visualize the results — t - **Configurable Docling pipeline** — OCR, table extraction, code/formula enrichment, picture classification & description, image generation - **Bounding box visualization** — color-coded element overlay directly on the PDF - **Per-page results** — right panel syncs with the current PDF page +- **Chunking** — split extracted content into semantic chunks (hierarchical, hybrid, or page-based) with configurable token limits and inline editing +- **Ingestion pipeline** — Docling → chunking → embedding → OpenSearch vector indexing (one-click from Studio) - **Markdown & HTML export** of extracted content -- **Document management** — upload, list, delete +- **Document management** — upload, list, delete, search, filter by indexing status - **Analysis history** — re-visit and open past analyses +- **Upload limits** — configurable max file size and max page count per document +- **Rate limiting** — configurable requests per minute per IP - **Dark / Light theme** and **FR / EN** localization @@ -74,7 +78,7 @@ document-parser/ ├── services/ # Use case orchestration │ ├── document_service.py # Upload, delete, preview │ └── analysis_service.py # Async Docling processing -└── tests/ # 199 tests (pytest) +└── tests/ # 377 tests (pytest) ``` ### Frontend structure (feature-based) @@ -98,14 +102,24 @@ frontend/src/ ## Quick Start -Docling Studio ships two Docker image variants: +One command, nothing else to install: + +```bash +docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local +``` + +Open [http://localhost:3000](http://localhost:3000), upload a PDF, and get results. That's it. + +> **Note:** The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast. + +### Image variants | Variant | Image tag | Size | Description | |---------|-----------|------|-------------| +| **local** | `latest-local` | ~1.9 GB | Full — runs Docling in-process, CPU-only | | **remote** | `latest-remote` | ~270 MB | Lightweight — delegates to an external [Docling Serve](https://github.com/DS4SD/docling-serve) instance | -| **local** | `latest-local` | ~1.9 GB | Full — runs Docling in-process, CPU-only (downloads ML models on first run) | -### Docker — remote mode (fastest) +For remote mode: ```bash docker run -p 3000:3000 \ @@ -113,27 +127,17 @@ docker run -p 3000:3000 \ ghcr.io/scub-france/docling-studio:latest-remote ``` -### Docker — local mode (self-contained) - -```bash -docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local -``` - -> **Note:** The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast. - -Open [http://localhost:3000](http://localhost:3000) - -### Docker Compose (for development) +### Docker Compose ```bash git clone https://github.com/scub-france/Docling-Studio.git cd Docling-Studio -# Local mode (default) +# Simple mode (backend + frontend only) docker compose up --build -# Remote mode -CONVERSION_MODE=remote DOCLING_SERVE_URL=http://your-docling-serve:5001 docker compose up --build +# With ingestion pipeline (OpenSearch + embeddings) +docker compose --profile ingestion -f docker-compose.yml -f docker-compose.ingestion.yml up --build ``` ### Local Development @@ -162,12 +166,12 @@ npm run dev ### Running Tests ```bash -# Backend (199 tests) +# Backend (377 tests) cd document-parser pip install pytest pytest-asyncio httpx pytest tests/ -v -# Frontend (129 tests) +# Frontend (156 tests) cd frontend npm run test:run ``` @@ -202,6 +206,43 @@ All configuration is done via environment variables. See [`.env.example`](.env.e | `UPLOAD_DIR` | `./uploads` | File storage directory | | `DB_PATH` | `./data/docling_studio.db` | SQLite database path | | `CONVERSION_TIMEOUT` | `600` | Max seconds for a single Docling conversion | +| `BATCH_PAGE_SIZE` | `10` | Pages per batch (`0` = process all at once) | +| `MAX_FILE_SIZE_MB` | `50` | Maximum upload file size in MB (`0` = unlimited) | +| `MAX_PAGE_COUNT` | `0` | Maximum number of pages per document (`0` = unlimited) | +| `RATE_LIMIT_RPM` | `100` | Max requests per minute per IP (`0` = disabled) | + +## Upload Limits + +Docling Studio enforces configurable limits on uploaded documents to protect the server against oversized files and long-running analyses: + +- **`MAX_FILE_SIZE_MB`** (default `50`) — rejects uploads exceeding this size. Validated at two levels: early `Content-Length` check and streaming byte count. +- **`MAX_PAGE_COUNT`** (default `0` = unlimited) — rejects documents with more pages than allowed. Useful on shared instances or Hugging Face Spaces to cap processing time. + +Both limits are exposed in the `/api/health` endpoint so the frontend can display them to the user before upload. Set either to `0` to disable the corresponding check. + +## Ingestion Pipeline (opt-in) + +Docling Studio can optionally index extracted chunks into [OpenSearch](https://opensearch.org/) for vector and full-text search. This requires two additional services (OpenSearch + embedding) and is **disabled by default**. + +To enable ingestion with Docker Compose: + +```bash +docker compose --profile ingestion \ + -f docker-compose.yml -f docker-compose.ingestion.yml \ + up --build +``` + +When ingestion is enabled, the UI shows: +- An **Ingest** button in Studio to push chunks to OpenSearch +- An **OpenSearch** connection status badge in the sidebar +- **Indexed / Not indexed** filters on the Documents page +- A **Search** page for full-text and vector search across indexed documents + +| Variable | Default | Description | +|----------|---------|-------------| +| `OPENSEARCH_URL` | — | OpenSearch endpoint (empty = ingestion disabled) | +| `EMBEDDING_URL` | — | Embedding service endpoint (empty = ingestion disabled) | +| `EMBEDDING_DIMENSION` | `384` | Vector dimension (must match embedding model) | ## CI / Release diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..9255c3b --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,110 @@ +# ============================================================================= +# Docling Studio — Development stack +# +# Usage: +# docker compose -f docker-compose.dev.yml up +# +# Includes OpenSearch single-node + Dashboards for search/sync features. +# Frontend runs Vite dev server with HMR, backend runs with --reload. +# ============================================================================= + +services: + # --- OpenSearch (single-node, security disabled for local dev) --- + opensearch: + image: opensearchproject/opensearch:2 + environment: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" + ports: + - "9200:9200" + volumes: + - opensearch_data:/usr/share/opensearch/data + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + + # --- OpenSearch Dashboards (index inspection UI) --- + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:2 + environment: + OPENSEARCH_HOSTS: '["http://opensearch:9200"]' + DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true" + ports: + - "5601:5601" + depends_on: + opensearch: + condition: service_healthy + + # --- Embedding service (sentence-transformers) --- + embedding: + build: + context: ./embedding-service + ports: + - "8001:8001" + environment: + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-all-MiniLM-L6-v2} + EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64} + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8001/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + deploy: + resources: + limits: + memory: 2g + + # --- Backend (FastAPI with hot-reload) --- + document-parser: + build: + context: ./document-parser + target: ${CONVERSION_MODE:-local} + ports: + - "8000:8000" + volumes: + - ./document-parser:/app + - uploads_data:/app/uploads + - db_data:/app/data + environment: + CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173} + DOCLING_SERVE_URL: ${DOCLING_SERVE_URL:-} + DOCLING_SERVE_API_KEY: ${DOCLING_SERVE_API_KEY:-} + RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100} + MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50} + BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-10} + OPENSEARCH_URL: http://opensearch:9200 + EMBEDDING_URL: http://embedding:8001 + command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] + depends_on: + opensearch: + condition: service_healthy + embedding: + condition: service_healthy + deploy: + resources: + limits: + memory: 4g + + # --- Frontend (Vite dev server with HMR) --- + frontend: + image: node:20-alpine + working_dir: /app + ports: + - "3000:3000" + volumes: + - ./frontend:/app + - frontend_node_modules:/app/node_modules + environment: + VITE_APP_VERSION: dev + command: ["sh", "-c", "npm install && npm run dev -- --host"] + depends_on: + - document-parser + +volumes: + opensearch_data: + uploads_data: + db_data: + frontend_node_modules: diff --git a/docker-compose.ingestion.yml b/docker-compose.ingestion.yml new file mode 100644 index 0000000..ed15dc1 --- /dev/null +++ b/docker-compose.ingestion.yml @@ -0,0 +1,19 @@ +# Override to enable the ingestion pipeline (OpenSearch + embeddings). +# +# Usage: +# docker compose --profile ingestion -f docker-compose.yml -f docker-compose.ingestion.yml up --build +# +# This wires the backend to the OpenSearch and embedding services started +# by the "ingestion" profile and ensures they are healthy before the +# backend starts. + +services: + document-parser: + environment: + OPENSEARCH_URL: http://opensearch:9200 + EMBEDDING_URL: http://embedding:8001 + depends_on: + opensearch: + condition: service_healthy + embedding: + condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 3eb3d1e..918bfb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,40 @@ services: + # --- OpenSearch (single-node, security disabled) --- + opensearch: + profiles: ["ingestion"] + image: opensearchproject/opensearch:2 + environment: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" + volumes: + - opensearch_data:/usr/share/opensearch/data + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + + # --- Embedding service (sentence-transformers) --- + embedding: + profiles: ["ingestion"] + build: + context: ./embedding-service + environment: + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-all-MiniLM-L6-v2} + EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64} + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8001/health || exit 1"] + interval: 15s + timeout: 10s + retries: 20 + start_period: 120s + deploy: + resources: + limits: + memory: 2g + + # --- Backend (FastAPI) --- document-parser: build: context: ./document-parser @@ -14,12 +50,15 @@ services: DOCLING_SERVE_API_KEY: ${DOCLING_SERVE_API_KEY:-} RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100} MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50} - BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-0} + BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-10} + OPENSEARCH_URL: ${OPENSEARCH_URL:-} + EMBEDDING_URL: ${EMBEDDING_URL:-} deploy: resources: limits: memory: 4g + # --- Frontend (nginx) --- frontend: build: context: ./frontend @@ -29,5 +68,6 @@ services: - document-parser volumes: + opensearch_data: uploads_data: db_data: diff --git a/docs/getting-started.md b/docs/getting-started.md index b2ee2b8..c187e72 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,15 +1,28 @@ # Getting Started -Docling Studio ships two Docker image variants: +## Quick Start -| Variant | Image tag | Size | Description | -|---------|-----------|------|-------------| -| **remote** | `latest-remote` | ~270 MB | Lightweight — delegates to an external [Docling Serve](https://github.com/DS4SD/docling-serve) instance | -| **local** | `latest-local` | ~1.9 GB | Full — runs Docling in-process, CPU-only (downloads ML models on first run) | +One command, nothing else to install: + +```bash +docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local +``` + +Open [http://localhost:3000](http://localhost:3000), upload a PDF, and get results. That's it. + +!!! note + The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast. ![Docker architecture](images/docker.png){ width="600" } -## Docker — remote mode (fastest) +## Image Variants + +| Variant | Image tag | Size | Description | +|---------|-----------|------|-------------| +| **local** | `latest-local` | ~1.9 GB | Full — runs Docling in-process, CPU-only | +| **remote** | `latest-remote` | ~270 MB | Lightweight — delegates to an external [Docling Serve](https://github.com/DS4SD/docling-serve) instance | + +For remote mode: ```bash docker run -p 3000:3000 \ @@ -17,27 +30,19 @@ docker run -p 3000:3000 \ ghcr.io/scub-france/docling-studio:latest-remote ``` -## Docker — local mode (self-contained) - -```bash -docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local -``` - -> **Note:** The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast. - -Open [http://localhost:3000](http://localhost:3000). - -## Docker Compose (recommended for development) +## Docker Compose ```bash git clone https://github.com/scub-france/Docling-Studio.git cd Docling-Studio -# Local mode (default) +# Simple mode (backend + frontend only) docker compose up --build -# Remote mode -CONVERSION_MODE=remote DOCLING_SERVE_URL=http://your-docling-serve:5001 docker compose up --build +# With ingestion pipeline (OpenSearch + embeddings) +docker compose --profile ingestion \ + -f docker-compose.yml -f docker-compose.ingestion.yml \ + up --build ``` ## Local Development @@ -136,10 +141,48 @@ All configuration is done via environment variables: | `UPLOAD_DIR` | `./uploads` | File storage directory | | `DB_PATH` | `./data/docling_studio.db` | SQLite database path | | `CONVERSION_TIMEOUT` | `600` | Max seconds per Docling conversion | +| `BATCH_PAGE_SIZE` | `10` | Pages per batch (`0` = process all at once) | | `MAX_CONCURRENT_ANALYSES` | `3` | Maximum parallel analysis jobs | | `DEPLOYMENT_MODE` | `self-hosted` | `self-hosted` or `huggingface` (shows disclaimer banner) | +| `MAX_FILE_SIZE_MB` | `50` | Maximum upload file size in MB (`0` = unlimited) | +| `MAX_PAGE_COUNT` | `0` | Maximum number of pages per document (`0` = unlimited) | +| `RATE_LIMIT_RPM` | `100` | Max requests per minute per IP (`0` = disabled) | | `APP_VERSION` | `dev` | Application version (set automatically by CI/Docker) | +## Upload Limits + +Docling Studio enforces configurable limits on uploaded documents to protect the server against oversized files and long-running analyses: + +- **`MAX_FILE_SIZE_MB`** (default `50`) — rejects uploads exceeding this size. Validated at two levels: early `Content-Length` check and streaming byte count. +- **`MAX_PAGE_COUNT`** (default `0` = unlimited) — rejects documents with more pages than allowed. Useful on shared instances or Hugging Face Spaces to cap processing time. + +Both limits are exposed in the `/api/health` endpoint so the frontend can display them to the user before upload. Set either to `0` to disable the corresponding check. + +## Ingestion Pipeline (opt-in) + +Docling Studio can optionally index extracted chunks into [OpenSearch](https://opensearch.org/) for vector and full-text search. This requires two additional services (OpenSearch + embedding) and is **disabled by default**. + +To enable ingestion with Docker Compose: + +```bash +docker compose --profile ingestion \ + -f docker-compose.yml -f docker-compose.ingestion.yml \ + up --build +``` + +When ingestion is enabled, the UI shows: + +- An **Ingest** button in Studio to push chunks to OpenSearch +- An **OpenSearch** connection status badge in the sidebar +- **Indexed / Not indexed** filters on the Documents page +- A **Search** page for full-text and vector search across indexed documents + +| Variable | Default | Description | +|----------|---------|-------------| +| `OPENSEARCH_URL` | — | OpenSearch endpoint (empty = ingestion disabled) | +| `EMBEDDING_URL` | — | Embedding service endpoint (empty = ingestion disabled) | +| `EMBEDDING_DIMENSION` | `384` | Vector dimension (must match embedding model) | + ## System Requirements | | Remote image | Local image | diff --git a/docs/index.md b/docs/index.md index b476021..2323b46 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,7 +18,8 @@ Upload a PDF, configure the extraction pipeline, and visualize the results — t - **Document management** — upload, list, delete - **Analysis history** — re-visit and open past analyses - **Feature flags** — capabilities adapt to the conversion engine (local vs remote) -- **Rate limiting** — 60 requests per minute per IP to protect the backend +- **Upload limits** — configurable max file size (`MAX_FILE_SIZE_MB`) and max page count (`MAX_PAGE_COUNT`) per document +- **Rate limiting** — configurable requests per minute per IP (`RATE_LIMIT_RPM`) - **Deployment modes** — self-hosted (default) or HuggingFace Spaces (with disclaimer banner) - **Health endpoint** — `/api/health` reports engine type, deployment mode, and database status - **Dark / Light theme** and **FR / EN** localization diff --git a/document-parser/api/analyses.py b/document-parser/api/analyses.py index f54ae9d..6c8a02a 100644 --- a/document-parser/api/analyses.py +++ b/document-parser/api/analyses.py @@ -13,6 +13,7 @@ from api.schemas import ( ChunkResponse, CreateAnalysisRequest, RechunkRequest, + UpdateChunkTextRequest, ) from services.analysis_service import AnalysisService @@ -110,6 +111,50 @@ async def rechunk_analysis( ] +@router.patch("/{job_id}/chunks/{chunk_index}", response_model=list[ChunkResponse]) +async def update_chunk_text( + job_id: str, chunk_index: int, body: UpdateChunkTextRequest, service: ServiceDep +) -> list[ChunkResponse]: + """Update the text of a single chunk by index.""" + try: + chunks = await service.update_chunk_text(job_id, chunk_index, body.text) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + return [ + ChunkResponse( + text=c["text"], + headings=c.get("headings", []), + source_page=c.get("sourcePage"), + token_count=c.get("tokenCount", 0), + bboxes=[ChunkBboxResponse(page=b["page"], bbox=b["bbox"]) for b in c.get("bboxes", [])], + modified=c.get("modified", False), + deleted=c.get("deleted", False), + ) + for c in chunks + ] + + +@router.delete("/{job_id}/chunks/{chunk_index}", response_model=list[ChunkResponse]) +async def delete_chunk(job_id: str, chunk_index: int, service: ServiceDep) -> list[ChunkResponse]: + """Soft-delete a chunk by index (marks it as deleted).""" + try: + chunks = await service.delete_chunk(job_id, chunk_index) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + return [ + ChunkResponse( + text=c["text"], + headings=c.get("headings", []), + source_page=c.get("sourcePage"), + token_count=c.get("tokenCount", 0), + bboxes=[ChunkBboxResponse(page=b["page"], bbox=b["bbox"]) for b in c.get("bboxes", [])], + modified=c.get("modified", False), + deleted=c.get("deleted", False), + ) + for c in chunks + ] + + @router.delete("/{job_id}", status_code=204) async def delete_analysis(job_id: str, service: ServiceDep) -> None: """Delete an analysis job.""" diff --git a/document-parser/api/ingestion.py b/document-parser/api/ingestion.py new file mode 100644 index 0000000..8915ce6 --- /dev/null +++ b/document-parser/api/ingestion.py @@ -0,0 +1,119 @@ +"""Ingestion API router — trigger and manage vector ingestion pipeline.""" + +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query, Request + +from api.schemas import ( + IngestionResponse, + IngestionStatusResponse, + SearchResponse, + SearchResultItem, +) +from services.analysis_service import AnalysisService +from services.ingestion_service import IngestionService + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/ingestion", tags=["ingestion"]) + + +def _get_ingestion_service(request: Request) -> IngestionService: + svc = request.app.state.ingestion_service + if svc is None: + raise HTTPException( + status_code=503, + detail="Ingestion not available (EMBEDDING_URL and OPENSEARCH_URL required)", + ) + return svc + + +def _get_analysis_service(request: Request) -> AnalysisService: + return request.app.state.analysis_service + + +IngestionDep = Annotated[IngestionService, Depends(_get_ingestion_service)] +AnalysisDep = Annotated[AnalysisService, Depends(_get_analysis_service)] + + +@router.post("/{job_id}", response_model=IngestionResponse) +async def ingest_analysis( + job_id: str, + ingestion: IngestionDep, + analysis: AnalysisDep, +) -> IngestionResponse: + """Ingest a completed analysis into the vector index. + + Takes the chunks from an existing analysis job, embeds them, + and indexes them into OpenSearch. + """ + job = await analysis.find_by_id(job_id) + if not job: + raise HTTPException(status_code=404, detail="Analysis not found") + if job.status.value != "COMPLETED": + raise HTTPException(status_code=400, detail="Analysis is not completed") + if not job.chunks_json: + raise HTTPException(status_code=400, detail="Analysis has no chunks — run chunking first") + + try: + result = await ingestion.ingest( + doc_id=job.document_id, + filename=job.document_filename or "unknown", + chunks_json=job.chunks_json, + ) + except Exception as e: + logger.exception("Ingestion failed for job %s", job_id) + raise HTTPException(status_code=500, detail=f"Ingestion failed: {e}") from e + + return IngestionResponse( + doc_id=result.doc_id, + chunks_indexed=result.chunks_indexed, + embedding_dimension=result.embedding_dimension, + ) + + +@router.delete("/{doc_id}", status_code=204) +async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None: + """Delete all indexed chunks for a document.""" + await ingestion.delete_document(doc_id) + + +@router.get("/status", response_model=IngestionStatusResponse) +async def ingestion_status(request: Request) -> IngestionStatusResponse: + """Check if the ingestion pipeline is available and OpenSearch is connected.""" + svc = request.app.state.ingestion_service + if svc is None: + return IngestionStatusResponse(available=False, opensearch_connected=False) + + connected = await svc.ping() + return IngestionStatusResponse(available=True, opensearch_connected=connected) + + +@router.get("/search", response_model=SearchResponse) +async def search_chunks( + ingestion: IngestionDep, + q: str = Query(..., min_length=1, description="Search query"), + doc_id: str | None = Query(None, description="Filter by document ID"), + k: int = Query(20, ge=1, le=100, description="Max results"), +) -> SearchResponse: + """Full-text search across indexed chunks. + + Returns matching chunks with content and metadata. + Optionally filter by document ID. + """ + results = await ingestion.search_fulltext(q, k=k, doc_id=doc_id) + items = [ + SearchResultItem( + doc_id=r.chunk.doc_id, + filename=r.chunk.filename, + content=r.chunk.content, + chunk_index=r.chunk.chunk_index, + page_number=r.chunk.page_number, + score=r.score, + headings=r.chunk.headings, + ) + for r in results + ] + return SearchResponse(results=items, total=len(items), query=q) diff --git a/document-parser/api/schemas.py b/document-parser/api/schemas.py index 1df46d6..2b4a1b1 100644 --- a/document-parser/api/schemas.py +++ b/document-parser/api/schemas.py @@ -34,6 +34,7 @@ class HealthResponse(_CamelModel): database: str max_page_count: int | None = None max_file_size_mb: int | None = None + ingestion_available: bool = False class DocumentResponse(_CamelModel): @@ -158,6 +159,12 @@ class ChunkResponse(_CamelModel): source_page: int | None = None token_count: int = 0 bboxes: list[ChunkBboxResponse] = [] + modified: bool = False + deleted: bool = False + + +class UpdateChunkTextRequest(BaseModel): + text: str class CreateAnalysisRequest(BaseModel): @@ -174,3 +181,33 @@ class RechunkRequest(BaseModel): chunkingOptions: ChunkingOptionsRequest = Field( validation_alias=AliasChoices("chunkingOptions", "chunking_options") ) + + +class IngestionResponse(_CamelModel): + doc_id: str + chunks_indexed: int + embedding_dimension: int + + +class IngestionStatusResponse(_CamelModel): + available: bool + opensearch_connected: bool = False + + +class SearchResultItem(_CamelModel): + """A single search result with content and metadata.""" + + doc_id: str + filename: str + content: str + chunk_index: int + page_number: int + score: float + headings: list[str] = [] + highlights: list[str] = [] + + +class SearchResponse(_CamelModel): + results: list[SearchResultItem] + total: int + query: str diff --git a/document-parser/domain/ports.py b/document-parser/domain/ports.py index 4c56396..1cd3fd5 100644 --- a/document-parser/domain/ports.py +++ b/document-parser/domain/ports.py @@ -6,7 +6,7 @@ Infrastructure adapters (local Docling, Docling Serve, etc.) implement these. from __future__ import annotations -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Protocol, runtime_checkable if TYPE_CHECKING: from domain.models import AnalysisJob, Document @@ -16,6 +16,7 @@ if TYPE_CHECKING: ConversionOptions, ConversionResult, ) + from domain.vector_schema import IndexedChunk, SearchResult class DocumentConverter(Protocol): @@ -79,3 +80,65 @@ class AnalysisRepository(Protocol): async def delete(self, job_id: str) -> bool: ... async def delete_by_document(self, document_id: str) -> int: ... + + +@runtime_checkable +class EmbeddingService(Protocol): + """Port for text-to-vector embedding. + + Implementations may call a local model, a remote microservice, etc. + """ + + async def embed(self, texts: list[str]) -> list[list[float]]: + """Generate embedding vectors for a batch of texts.""" + ... + + +@runtime_checkable +class VectorStore(Protocol): + """Port for vector storage and retrieval. + + Implementations (OpenSearch, pgvector, Qdrant, etc.) must satisfy this + contract. The port uses domain types from vector_schema — no infrastructure + details leak into the domain. + """ + + async def ensure_index(self, index_name: str, mapping: dict) -> None: + """Create the index if it does not exist. No-op if it already exists.""" + ... + + async def index_chunks(self, index_name: str, chunks: list[IndexedChunk]) -> int: + """Bulk-index a list of chunks. Returns the number of successfully indexed chunks.""" + ... + + async def search_similar( + self, + index_name: str, + embedding: list[float], + *, + k: int = 10, + doc_id: str | None = None, + ) -> list[SearchResult]: + """Find the k nearest chunks by embedding similarity. + + Args: + index_name: Target index. + embedding: Query vector. + k: Number of results to return. + doc_id: If provided, restrict search to chunks from this document. + """ + ... + + async def get_chunks( + self, + index_name: str, + doc_id: str, + *, + limit: int = 1000, + ) -> list[SearchResult]: + """Retrieve all indexed chunks for a given document, ordered by chunk_index.""" + ... + + async def delete_document(self, index_name: str, doc_id: str) -> int: + """Delete all chunks for a document from the index. Returns count deleted.""" + ... diff --git a/document-parser/domain/vector_schema.py b/document-parser/domain/vector_schema.py new file mode 100644 index 0000000..59512e6 --- /dev/null +++ b/document-parser/domain/vector_schema.py @@ -0,0 +1,177 @@ +"""Vector index schema — data contract for OpenSearch ingestion and inspection. + +This module defines the standard metadata schema for the vector index used by +the ingestion pipeline (0.4.0) and the inspection UI (0.5.0). + +Field usage by milestone: + ┌────────────┬────────────────────────┬───────────────────────────────┬──────────────┐ + │ Field │ 0.4.0 (write) │ 0.5.0 (read) │ Source │ + ├────────────┼────────────────────────┼───────────────────────────────┼──────────────┤ + │ content │ Full-text search │ Chunk panel display │ Docling std │ + │ embedding │ Indexed │ kNN semantic search │ Docling std │ + │ doc_items │ Indexed │ Element type filtering │ Docling std │ + │ headings │ Indexed │ Section hierarchy display │ Docling std │ + │ origin │ Indexed │ Document provenance │ Docling std │ + │ bboxes │ Written at ingestion │ Chunk↔bbox highlight │ Studio │ + │ page_number│ Written at ingestion │ Split view navigation │ Studio │ + │ chunk_index│ Written at ingestion │ Chunk ordering in panel │ Studio │ + │ chunk_type │ Written at ingestion │ Metadata panel │ Studio │ + │ doc_id │ Document linking │ Document list navigation │ Studio │ + │ filename │ "My Documents" list │ Display │ Studio │ + └────────────┴────────────────────────┴───────────────────────────────┴──────────────┘ +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +# -- Value objects for a single indexed chunk ---------------------------------- + +DEFAULT_EMBEDDING_DIMENSION = 384 # Granite Embedding 30M (sentence-transformers) +DEFAULT_INDEX_NAME = "docling-studio-chunks" + + +@dataclass(frozen=True) +class ChunkBboxEntry: + """Bounding box for a chunk region on a specific page.""" + + page: int + x: float + y: float + w: float + h: float + + +@dataclass(frozen=True) +class DocItemRef: + """Reference to a Docling DocItem (element in the document structure).""" + + self_ref: str + label: str # text, table, picture, list, etc. + + +@dataclass(frozen=True) +class ChunkOrigin: + """Provenance metadata — links a chunk back to its source document binary.""" + + binary_hash: str + filename: str + + +@dataclass(frozen=True) +class IndexedChunk: + """A single chunk ready to be indexed in the vector store. + + This is the domain-level representation of a document in the OpenSearch index. + It combines Docling-standard fields (content, embedding, doc_items, headings, + origin) with Docling Studio enriched fields (bboxes, page_number, chunk_index, + chunk_type, doc_id, filename). + """ + + doc_id: str + filename: str + content: str + embedding: list[float] + chunk_index: int + chunk_type: str # text, table, picture, list, etc. + page_number: int + bboxes: list[ChunkBboxEntry] = field(default_factory=list) + headings: list[str] = field(default_factory=list) + doc_items: list[DocItemRef] = field(default_factory=list) + origin: ChunkOrigin | None = None + + def to_dict(self) -> dict: + """Serialize to a dict matching the OpenSearch index mapping.""" + result: dict = { + "doc_id": self.doc_id, + "filename": self.filename, + "content": self.content, + "embedding": self.embedding, + "chunk_index": self.chunk_index, + "chunk_type": self.chunk_type, + "page_number": self.page_number, + "bboxes": [ + {"page": b.page, "x": b.x, "y": b.y, "w": b.w, "h": b.h} for b in self.bboxes + ], + "headings": self.headings, + "doc_items": [{"self_ref": d.self_ref, "label": d.label} for d in self.doc_items], + } + if self.origin: + result["origin"] = { + "binary_hash": self.origin.binary_hash, + "filename": self.origin.filename, + } + return result + + +# -- Search result ------------------------------------------------------------- + + +@dataclass(frozen=True) +class SearchResult: + """A chunk returned from a vector store query.""" + + chunk: IndexedChunk + score: float # similarity score (higher = more similar) + + +# -- Index mapping template ---------------------------------------------------- + + +def build_index_mapping(embedding_dimension: int = DEFAULT_EMBEDDING_DIMENSION) -> dict: + """Build the OpenSearch index mapping for the chunk index. + + Args: + embedding_dimension: Vector dimension for the knn_vector field. + Defaults to 384 (Granite Embedding 30M / all-MiniLM-L6-v2). + """ + return { + "settings": { + "index": { + "knn": True, + }, + }, + "mappings": { + "properties": { + "doc_id": {"type": "keyword"}, + "filename": {"type": "keyword"}, + "content": {"type": "text", "analyzer": "standard"}, + "embedding": { + "type": "knn_vector", + "dimension": embedding_dimension, + "method": { + "engine": "faiss", + "name": "hnsw", + }, + }, + "chunk_index": {"type": "integer"}, + "chunk_type": {"type": "keyword"}, + "page_number": {"type": "integer"}, + "bboxes": { + "type": "nested", + "properties": { + "page": {"type": "integer"}, + "x": {"type": "float"}, + "y": {"type": "float"}, + "w": {"type": "float"}, + "h": {"type": "float"}, + }, + }, + "headings": {"type": "text"}, + "doc_items": { + "type": "nested", + "properties": { + "self_ref": {"type": "keyword"}, + "label": {"type": "keyword"}, + }, + }, + "origin": { + "type": "object", + "properties": { + "binary_hash": {"type": "keyword"}, + "filename": {"type": "keyword"}, + }, + }, + }, + }, + } diff --git a/document-parser/infra/embedding_client.py b/document-parser/infra/embedding_client.py new file mode 100644 index 0000000..c0fc861 --- /dev/null +++ b/document-parser/infra/embedding_client.py @@ -0,0 +1,51 @@ +"""HTTP client adapter for the embedding microservice. + +Satisfies the ``EmbeddingService`` Protocol defined in ``domain.ports``. +Calls the embedding-service REST API (POST /embed). +""" + +from __future__ import annotations + +import logging + +import httpx + +logger = logging.getLogger(__name__) + +# Maximum texts per request to avoid payload / memory issues on the server. +_MAX_BATCH = 256 + + +class EmbeddingClient: + """Remote embedding adapter backed by the embedding-service microservice. + + Args: + base_url: Embedding service URL (e.g. ``http://localhost:8001``). + timeout: HTTP request timeout in seconds. + """ + + def __init__(self, base_url: str, *, timeout: float = 120.0) -> None: + self._base_url = base_url.rstrip("/") + self._timeout = timeout + + async def embed(self, texts: list[str]) -> list[list[float]]: + """Generate embeddings by calling the remote service. + + Automatically splits large batches into sub-batches of ``_MAX_BATCH``. + """ + if not texts: + return [] + + all_embeddings: list[list[float]] = [] + async with httpx.AsyncClient(timeout=self._timeout) as client: + for start in range(0, len(texts), _MAX_BATCH): + batch = texts[start : start + _MAX_BATCH] + resp = await client.post( + f"{self._base_url}/embed", + json={"texts": batch}, + ) + resp.raise_for_status() + data = resp.json() + all_embeddings.extend(data["embeddings"]) + + return all_embeddings diff --git a/document-parser/infra/opensearch_store.py b/document-parser/infra/opensearch_store.py new file mode 100644 index 0000000..c4a97f9 --- /dev/null +++ b/document-parser/infra/opensearch_store.py @@ -0,0 +1,204 @@ +"""OpenSearch adapter implementing the VectorStore port. + +Uses the opensearch-py client for kNN vector search, full-text search, +and document CRUD against an OpenSearch cluster. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from opensearchpy import AsyncOpenSearch, NotFoundError + +from domain.vector_schema import ( + ChunkBboxEntry, + ChunkOrigin, + DocItemRef, + IndexedChunk, + SearchResult, +) + +logger = logging.getLogger(__name__) + + +def _hit_to_indexed_chunk(hit: dict[str, Any]) -> IndexedChunk: + """Reconstruct an IndexedChunk from an OpenSearch _source document.""" + src = hit["_source"] + origin_raw = src.get("origin") + origin = ( + ChunkOrigin(binary_hash=origin_raw["binary_hash"], filename=origin_raw["filename"]) + if origin_raw + else None + ) + return IndexedChunk( + doc_id=src["doc_id"], + filename=src["filename"], + content=src["content"], + embedding=src.get("embedding", []), + chunk_index=src["chunk_index"], + chunk_type=src["chunk_type"], + page_number=src["page_number"], + bboxes=[ + ChunkBboxEntry(page=b["page"], x=b["x"], y=b["y"], w=b["w"], h=b["h"]) + for b in src.get("bboxes", []) + ], + headings=src.get("headings", []), + doc_items=[ + DocItemRef(self_ref=d["self_ref"], label=d["label"]) for d in src.get("doc_items", []) + ], + origin=origin, + ) + + +def _hit_to_result(hit: dict[str, Any]) -> SearchResult: + """Convert an OpenSearch hit to a SearchResult.""" + return SearchResult( + chunk=_hit_to_indexed_chunk(hit), + score=hit.get("_score", 0.0), + ) + + +class OpenSearchStore: + """Concrete VectorStore adapter backed by OpenSearch. + + Satisfies the ``VectorStore`` Protocol defined in ``domain.ports``. + + Args: + url: OpenSearch cluster URL (e.g. ``http://localhost:9200``). + verify_certs: Whether to verify TLS certificates. + """ + + def __init__(self, url: str, *, verify_certs: bool = False) -> None: + self._client = AsyncOpenSearch( + hosts=[url], + use_ssl=url.startswith("https"), + verify_certs=verify_certs, + ssl_show_warn=False, + ) + + # -- lifecycle ------------------------------------------------------------- + + async def close(self) -> None: + """Close the underlying HTTP connection pool.""" + await self._client.close() + + # -- VectorStore protocol methods ------------------------------------------ + + async def ensure_index(self, index_name: str, mapping: dict) -> None: + """Create the index if it does not exist. No-op if it already exists.""" + exists = await self._client.indices.exists(index=index_name) + if not exists: + await self._client.indices.create(index=index_name, body=mapping) + logger.info("Created OpenSearch index '%s'", index_name) + else: + logger.debug("Index '%s' already exists — skipping creation", index_name) + + async def index_chunks(self, index_name: str, chunks: list[IndexedChunk]) -> int: + """Bulk-index a list of chunks. Returns the number successfully indexed.""" + if not chunks: + return 0 + + body: list[dict[str, Any]] = [] + for chunk in chunks: + doc_id = f"{chunk.doc_id}_{chunk.chunk_index}" + body.append({"index": {"_index": index_name, "_id": doc_id}}) + body.append(chunk.to_dict()) + + resp = await self._client.bulk(body=body, refresh="wait_for") + + errors = sum(1 for item in resp["items"] if item["index"].get("error")) + indexed = len(chunks) - errors + if errors: + logger.warning("Bulk index to '%s': %d/%d failed", index_name, errors, len(chunks)) + return indexed + + async def search_similar( + self, + index_name: str, + embedding: list[float], + *, + k: int = 10, + doc_id: str | None = None, + ) -> list[SearchResult]: + """kNN search for the k nearest chunks by embedding similarity.""" + knn_query: dict[str, Any] = { + "knn": { + "embedding": { + "vector": embedding, + "k": k, + }, + }, + } + if doc_id: + knn_query["knn"]["embedding"]["filter"] = { + "term": {"doc_id": doc_id}, + } + + resp = await self._client.search( + index=index_name, + body={"size": k, "query": knn_query}, + _source_excludes=["embedding"], + ) + return [_hit_to_result(hit) for hit in resp["hits"]["hits"]] + + async def get_chunks( + self, + index_name: str, + doc_id: str, + *, + limit: int = 1000, + ) -> list[SearchResult]: + """Retrieve all indexed chunks for a document, ordered by chunk_index.""" + resp = await self._client.search( + index=index_name, + body={ + "size": limit, + "query": {"term": {"doc_id": doc_id}}, + "sort": [{"chunk_index": {"order": "asc"}}], + }, + _source_excludes=["embedding"], + ) + return [_hit_to_result(hit) for hit in resp["hits"]["hits"]] + + async def delete_document(self, index_name: str, doc_id: str) -> int: + """Delete all chunks for a document. Returns the number deleted.""" + try: + resp = await self._client.delete_by_query( + index=index_name, + body={"query": {"term": {"doc_id": doc_id}}}, + refresh=True, + ) + deleted: int = resp.get("deleted", 0) + return deleted + except NotFoundError: + return 0 + + # -- full-text search (bonus from spec) ------------------------------------ + + async def search_fulltext( + self, + index_name: str, + query_text: str, + *, + k: int = 10, + doc_id: str | None = None, + ) -> list[SearchResult]: + """Full-text search on the content field. + + This method is not part of the VectorStore protocol but is specified + in the issue acceptance criteria. + """ + must: list[dict[str, Any]] = [{"match": {"content": query_text}}] + if doc_id: + must.append({"term": {"doc_id": doc_id}}) + + resp = await self._client.search( + index=index_name, + body={ + "size": k, + "query": {"bool": {"must": must}}, + }, + _source_excludes=["embedding"], + ) + return [_hit_to_result(hit) for hit in resp["hits"]["hits"]] diff --git a/document-parser/infra/settings.py b/document-parser/infra/settings.py index 843ea59..50e465d 100644 --- a/document-parser/infra/settings.py +++ b/document-parser/infra/settings.py @@ -23,6 +23,9 @@ class Settings: max_file_size_mb: int = 50 # upload limit in MB (0 = unlimited) rate_limit_rpm: int = 100 # requests per minute per IP (0 = disabled) batch_page_size: int = 0 # 0 = disabled, > 0 = pages per batch + opensearch_url: str = "" # empty = disabled + embedding_url: str = "" # empty = disabled (e.g. http://localhost:8001) + embedding_dimension: int = 384 # Granite Embedding 30M / all-MiniLM-L6-v2 upload_dir: str = "./uploads" db_path: str = "./data/docling_studio.db" cors_origins: list[str] = field( @@ -51,6 +54,8 @@ class Settings: errors.append(f"rate_limit_rpm must be >= 0 (got {self.rate_limit_rpm})") if self.batch_page_size < 0: errors.append(f"batch_page_size must be >= 0 (got {self.batch_page_size})") + if self.embedding_dimension < 1: + errors.append(f"embedding_dimension must be >= 1 (got {self.embedding_dimension})") if self.default_table_mode not in ("accurate", "fast"): errors.append( f"default_table_mode must be 'accurate' or 'fast' (got '{self.default_table_mode}')" @@ -89,7 +94,10 @@ class Settings: max_file_size=int(os.environ.get("MAX_FILE_SIZE", "0")), max_file_size_mb=int(os.environ.get("MAX_FILE_SIZE_MB", "50")), rate_limit_rpm=int(os.environ.get("RATE_LIMIT_RPM", "100")), - batch_page_size=int(os.environ.get("BATCH_PAGE_SIZE", "0")), + batch_page_size=int(os.environ.get("BATCH_PAGE_SIZE", "10")), + opensearch_url=os.environ.get("OPENSEARCH_URL", ""), + embedding_url=os.environ.get("EMBEDDING_URL", ""), + embedding_dimension=int(os.environ.get("EMBEDDING_DIMENSION", "384")), upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"), db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"), cors_origins=[o.strip() for o in cors_raw.split(",")], diff --git a/document-parser/main.py b/document-parser/main.py index aa6573c..0f6a4a4 100644 --- a/document-parser/main.py +++ b/document-parser/main.py @@ -20,6 +20,7 @@ from fastapi.middleware.cors import CORSMiddleware from api.analyses import router as analyses_router from api.documents import router as documents_router +from api.ingestion import router as ingestion_router from api.schemas import HealthResponse from infra.rate_limiter import RateLimiterMiddleware from infra.settings import settings @@ -28,6 +29,7 @@ from persistence.database import get_connection, init_db from persistence.document_repo import SqliteDocumentRepository from services.analysis_service import AnalysisConfig, AnalysisService from services.document_service import DocumentConfig, DocumentService +from services.ingestion_service import IngestionConfig, IngestionService logging.basicConfig( level=logging.INFO, @@ -87,6 +89,28 @@ def _build_analysis_service( ) +def _build_ingestion_service() -> IngestionService | None: + """Build the ingestion service — only if embedding + opensearch are configured.""" + if not settings.embedding_url or not settings.opensearch_url: + logger.info("Ingestion disabled (EMBEDDING_URL or OPENSEARCH_URL not set)") + return None + + from infra.embedding_client import EmbeddingClient + from infra.opensearch_store import OpenSearchStore + + embedding = EmbeddingClient(settings.embedding_url) + vector_store = OpenSearchStore(settings.opensearch_url) + config = IngestionConfig( + embedding_dimension=settings.embedding_dimension, + ) + logger.info( + "Ingestion enabled (embedding=%s, opensearch=%s)", + settings.embedding_url, + settings.opensearch_url, + ) + return IngestionService(embedding, vector_store, config) + + def _build_document_service( document_repo: SqliteDocumentRepository, analysis_repo: SqliteAnalysisRepository, @@ -114,6 +138,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: document_repo, analysis_repo = _build_repos() app.state.analysis_service = _build_analysis_service(document_repo, analysis_repo) app.state.document_service = _build_document_service(document_repo, analysis_repo) + ingestion_service = _build_ingestion_service() + app.state.ingestion_service = ingestion_service + if ingestion_service is not None: + app.include_router(ingestion_router) + logger.info("Ingestion router mounted") logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine) yield @@ -128,7 +157,7 @@ app.add_middleware( CORSMiddleware, allow_origins=settings.cors_origins, allow_credentials=True, - allow_methods=["GET", "POST", "DELETE", "OPTIONS"], + allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"], allow_headers=["Content-Type", "Authorization"], ) if settings.rate_limit_rpm > 0: @@ -162,4 +191,5 @@ async def health() -> HealthResponse: database=db_status, max_page_count=settings.max_page_count if settings.max_page_count > 0 else None, max_file_size_mb=settings.max_file_size_mb if settings.max_file_size_mb > 0 else None, + ingestion_available=getattr(app.state, "ingestion_service", None) is not None, ) diff --git a/document-parser/requirements.txt b/document-parser/requirements.txt index 67cae25..81f72a7 100644 --- a/document-parser/requirements.txt +++ b/document-parser/requirements.txt @@ -7,3 +7,4 @@ pillow>=10.0.0,<11.0.0 aiosqlite>=0.20.0,<1.0.0 httpx>=0.27.0,<1.0.0 pypdfium2>=4.0.0,<5.0.0 +opensearch-py[async]>=2.6.0,<3.0.0 diff --git a/document-parser/services/analysis_service.py b/document-parser/services/analysis_service.py index b158272..809384e 100644 --- a/document-parser/services/analysis_service.py +++ b/document-parser/services/analysis_service.py @@ -160,6 +160,49 @@ class AnalysisService: return chunks + async def update_chunk_text(self, job_id: str, chunk_index: int, text: str) -> list[dict]: + """Update the text of a single chunk by index. Returns the full updated chunks list.""" + job = await self._analysis_repo.find_by_id(job_id) + if not job: + raise ValueError(f"Analysis not found: {job_id}") + if job.status != AnalysisStatus.COMPLETED: + raise ValueError(f"Analysis is not completed: {job_id}") + if not job.chunks_json: + raise ValueError(f"No chunks available: {job_id}") + + chunks = json.loads(job.chunks_json) + if chunk_index < 0 or chunk_index >= len(chunks): + raise ValueError(f"Chunk index out of range: {chunk_index}") + + chunks[chunk_index]["text"] = text + chunks[chunk_index]["modified"] = True + + chunks_json = json.dumps(chunks) + await self._analysis_repo.update_chunks(job_id, chunks_json) + + return chunks + + async def delete_chunk(self, job_id: str, chunk_index: int) -> list[dict]: + """Soft-delete a chunk by index. Returns the full updated chunks list.""" + job = await self._analysis_repo.find_by_id(job_id) + if not job: + raise ValueError(f"Analysis not found: {job_id}") + if job.status != AnalysisStatus.COMPLETED: + raise ValueError(f"Analysis is not completed: {job_id}") + if not job.chunks_json: + raise ValueError(f"No chunks available: {job_id}") + + chunks = json.loads(job.chunks_json) + if chunk_index < 0 or chunk_index >= len(chunks): + raise ValueError(f"Chunk index out of range: {chunk_index}") + + chunks[chunk_index]["deleted"] = True + + chunks_json = json.dumps(chunks) + await self._analysis_repo.update_chunks(job_id, chunks_json) + + return chunks + async def _run_batched_conversion( self, job_id: str, diff --git a/document-parser/services/ingestion_service.py b/document-parser/services/ingestion_service.py new file mode 100644 index 0000000..cce6b98 --- /dev/null +++ b/document-parser/services/ingestion_service.py @@ -0,0 +1,190 @@ +"""Ingestion service — orchestrates Docling → embedding → OpenSearch. + +Chains the full ingestion pipeline: +1. Convert document via Docling (reuse existing analysis) +2. Chunk with selected strategy +3. Embed all chunk texts via EmbeddingService +4. Index into OpenSearch via VectorStore + +Idempotent: re-ingesting a document deletes old chunks before re-indexing. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from domain.vector_schema import ( + ChunkBboxEntry, + ChunkOrigin, + IndexedChunk, + build_index_mapping, +) + +if TYPE_CHECKING: + from domain.ports import EmbeddingService, VectorStore + +logger = logging.getLogger(__name__) + + +@dataclass +class IngestionConfig: + """Configuration for the ingestion pipeline.""" + + index_name: str = "docling-studio-chunks" + embedding_dimension: int = 384 + + +@dataclass +class IngestionResult: + """Result of an ingestion pipeline run.""" + + doc_id: str + chunks_indexed: int + embedding_dimension: int + + +class IngestionService: + """Orchestrates the embedding + indexing pipeline.""" + + def __init__( + self, + embedding_service: EmbeddingService, + vector_store: VectorStore, + config: IngestionConfig | None = None, + ) -> None: + self._embedding = embedding_service + self._vector_store = vector_store + self._config = config or IngestionConfig() + + async def ensure_index(self) -> None: + """Ensure the vector index exists with the correct mapping.""" + mapping = build_index_mapping(self._config.embedding_dimension) + await self._vector_store.ensure_index(self._config.index_name, mapping) + + async def ingest( + self, + doc_id: str, + filename: str, + chunks_json: str, + *, + binary_hash: str | None = None, + ) -> IngestionResult: + """Run the embedding + indexing pipeline on pre-chunked data. + + This method is idempotent: it deletes any existing chunks for the + document before re-indexing. + + Args: + doc_id: Unique document identifier. + filename: Original filename. + chunks_json: JSON-serialized list of chunk dicts (from analysis). + binary_hash: Optional hash of the source file for provenance. + + Returns: + IngestionResult with the number of chunks indexed. + """ + await self.ensure_index() + + chunks_data: list[dict] = json.loads(chunks_json) + active_chunks = [c for c in chunks_data if not c.get("deleted")] + if not active_chunks: + logger.info("No active chunks for doc %s — skipping ingestion", doc_id) + return IngestionResult(doc_id=doc_id, chunks_indexed=0, embedding_dimension=0) + + # 1. Embed all chunk texts + texts = [c["text"] for c in active_chunks] + logger.info("Embedding %d chunks for doc %s", len(texts), doc_id) + embeddings = await self._embedding.embed(texts) + + # 2. Build IndexedChunk domain objects + origin = ( + ChunkOrigin(binary_hash=binary_hash or "", filename=filename) if binary_hash else None + ) + indexed_chunks: list[IndexedChunk] = [] + for i, (chunk_data, embedding) in enumerate(zip(active_chunks, embeddings, strict=True)): + bboxes = [ + ChunkBboxEntry( + page=b["page"], + x=b["bbox"][0] if b.get("bbox") else 0, + y=b["bbox"][1] if b.get("bbox") else 0, + w=(b["bbox"][2] - b["bbox"][0]) if b.get("bbox") and len(b["bbox"]) >= 4 else 0, + h=(b["bbox"][3] - b["bbox"][1]) if b.get("bbox") and len(b["bbox"]) >= 4 else 0, + ) + for b in chunk_data.get("bboxes", []) + ] + indexed_chunks.append( + IndexedChunk( + doc_id=doc_id, + filename=filename, + content=chunk_data["text"], + embedding=embedding, + chunk_index=i, + chunk_type=chunk_data.get("chunkType", "text"), + page_number=chunk_data.get("sourcePage", 0) or 0, + bboxes=bboxes, + headings=chunk_data.get("headings", []), + origin=origin, + ) + ) + + # 3. Delete old chunks (idempotent re-indexing) + deleted = await self._vector_store.delete_document(self._config.index_name, doc_id) + if deleted: + logger.info("Deleted %d old chunks for doc %s", deleted, doc_id) + + # 4. Index new chunks + indexed = await self._vector_store.index_chunks(self._config.index_name, indexed_chunks) + logger.info("Indexed %d/%d chunks for doc %s", indexed, len(indexed_chunks), doc_id) + + return IngestionResult( + doc_id=doc_id, + chunks_indexed=indexed, + embedding_dimension=len(embeddings[0]) if embeddings else 0, + ) + + async def delete_document(self, doc_id: str) -> int: + """Remove all indexed chunks for a document.""" + return await self._vector_store.delete_document(self._config.index_name, doc_id) + + async def search( + self, + query: str, + *, + k: int = 10, + doc_id: str | None = None, + ) -> list: + """Semantic search: embed the query then find nearest chunks.""" + embeddings = await self._embedding.embed([query]) + return await self._vector_store.search_similar( + self._config.index_name, + embeddings[0], + k=k, + doc_id=doc_id, + ) + + async def search_fulltext( + self, + query: str, + *, + k: int = 20, + doc_id: str | None = None, + ) -> list: + """Full-text keyword search in indexed chunks.""" + return await self._vector_store.search_fulltext( + self._config.index_name, + query, + k=k, + doc_id=doc_id, + ) + + async def ping(self) -> bool: + """Check if the OpenSearch cluster is reachable.""" + try: + info = await self._vector_store._client.info() + return bool(info) + except Exception: + logger.debug("OpenSearch ping failed", exc_info=True) + return False diff --git a/document-parser/tests/test_analysis_service.py b/document-parser/tests/test_analysis_service.py index f48f4cf..d79ae09 100644 --- a/document-parser/tests/test_analysis_service.py +++ b/document-parser/tests/test_analysis_service.py @@ -4,10 +4,12 @@ from __future__ import annotations import asyncio import functools +import json from unittest.mock import AsyncMock, MagicMock, patch import pytest +from domain.models import AnalysisStatus from domain.services import extract_html_body, merge_results from domain.value_objects import ConversionResult, PageDetail from services.analysis_service import AnalysisConfig, AnalysisService, _count_pdf_pages @@ -515,3 +517,141 @@ class TestBatchedConversion: assert result is None # Only first batch should have been converted assert converter.convert.call_count == 1 + + +class TestUpdateChunkText: + """Tests for AnalysisService.update_chunk_text.""" + + @pytest.mark.asyncio + async def test_update_chunk_text_success(self): + chunks = [ + {"text": "original", "headings": [], "sourcePage": 1, "tokenCount": 5, "bboxes": []}, + {"text": "second", "headings": [], "sourcePage": 2, "tokenCount": 3, "bboxes": []}, + ] + job = MagicMock() + job.status = AnalysisStatus.COMPLETED + job.chunks_json = json.dumps(chunks) + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + repo.update_chunks = AsyncMock(return_value=True) + + service = _make_service(analysis_repo=repo) + result = await service.update_chunk_text("j1", 0, "updated text") + + assert result[0]["text"] == "updated text" + assert result[0]["modified"] is True + assert result[1]["text"] == "second" + assert result[1].get("modified", False) is False + repo.update_chunks.assert_called_once() + + @pytest.mark.asyncio + async def test_update_chunk_text_not_found(self): + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=None) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="Analysis not found"): + await service.update_chunk_text("missing", 0, "text") + + @pytest.mark.asyncio + async def test_update_chunk_text_not_completed(self): + job = MagicMock() + job.status = AnalysisStatus.RUNNING + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="not completed"): + await service.update_chunk_text("j1", 0, "text") + + @pytest.mark.asyncio + async def test_update_chunk_text_index_out_of_range(self): + chunks = [ + {"text": "only one", "headings": [], "sourcePage": 1, "tokenCount": 5, "bboxes": []} + ] + job = MagicMock() + job.status = AnalysisStatus.COMPLETED + job.chunks_json = json.dumps(chunks) + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="out of range"): + await service.update_chunk_text("j1", 5, "text") + + @pytest.mark.asyncio + async def test_update_chunk_text_no_chunks(self): + job = MagicMock() + job.status = AnalysisStatus.COMPLETED + job.chunks_json = None + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="No chunks available"): + await service.update_chunk_text("j1", 0, "text") + + +class TestDeleteChunk: + """Tests for AnalysisService.delete_chunk.""" + + @pytest.mark.asyncio + async def test_delete_chunk_success(self): + chunks = [ + {"text": "chunk1", "headings": [], "sourcePage": 1, "tokenCount": 5, "bboxes": []}, + {"text": "chunk2", "headings": [], "sourcePage": 2, "tokenCount": 3, "bboxes": []}, + ] + job = MagicMock() + job.status = AnalysisStatus.COMPLETED + job.chunks_json = json.dumps(chunks) + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + repo.update_chunks = AsyncMock(return_value=True) + + service = _make_service(analysis_repo=repo) + result = await service.delete_chunk("j1", 0) + + assert result[0]["deleted"] is True + assert result[1].get("deleted", False) is False + repo.update_chunks.assert_called_once() + + @pytest.mark.asyncio + async def test_delete_chunk_not_found(self): + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=None) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="Analysis not found"): + await service.delete_chunk("missing", 0) + + @pytest.mark.asyncio + async def test_delete_chunk_index_out_of_range(self): + chunks = [{"text": "only", "headings": [], "sourcePage": 1, "tokenCount": 5, "bboxes": []}] + job = MagicMock() + job.status = AnalysisStatus.COMPLETED + job.chunks_json = json.dumps(chunks) + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="out of range"): + await service.delete_chunk("j1", 5) + + @pytest.mark.asyncio + async def test_delete_chunk_no_chunks(self): + job = MagicMock() + job.status = AnalysisStatus.COMPLETED + job.chunks_json = None + + repo = MagicMock() + repo.find_by_id = AsyncMock(return_value=job) + service = _make_service(analysis_repo=repo) + + with pytest.raises(ValueError, match="No chunks available"): + await service.delete_chunk("j1", 0) diff --git a/document-parser/tests/test_api_endpoints.py b/document-parser/tests/test_api_endpoints.py index 97c154e..2be220e 100644 --- a/document-parser/tests/test_api_endpoints.py +++ b/document-parser/tests/test_api_endpoints.py @@ -51,6 +51,23 @@ class TestHealthEndpoint: assert "maxFileSizeMb" in data assert data["maxFileSizeMb"] == 50 + def test_health_exposes_ingestion_available_false(self, client): + original = getattr(app.state, "ingestion_service", None) + app.state.ingestion_service = None + resp = client.get("/api/health") + app.state.ingestion_service = original + data = resp.json() + assert "ingestionAvailable" in data + assert data["ingestionAvailable"] is False + + def test_health_exposes_ingestion_available_true(self, client): + original = getattr(app.state, "ingestion_service", None) + app.state.ingestion_service = MagicMock() + resp = client.get("/api/health") + app.state.ingestion_service = original + data = resp.json() + assert data["ingestionAvailable"] is True + class TestDocumentEndpoints: def test_list_documents(self, client, mock_document_service): diff --git a/document-parser/tests/test_chunking.py b/document-parser/tests/test_chunking.py index 52e84c8..841ea05 100644 --- a/document-parser/tests/test_chunking.py +++ b/document-parser/tests/test_chunking.py @@ -278,6 +278,114 @@ class TestCreateAnalysisWithChunking: assert chunks[0]["text"] == "chunk1" +class TestUpdateChunkTextEndpoint: + def test_update_chunk_text_success(self, client, mock_analysis_service): + updated_chunks = [ + { + "text": "updated text", + "headings": ["H1"], + "sourcePage": 1, + "tokenCount": 10, + "bboxes": [], + "modified": True, + }, + { + "text": "chunk2", + "headings": [], + "sourcePage": 2, + "tokenCount": 20, + "bboxes": [], + "modified": False, + }, + ] + mock_analysis_service.update_chunk_text = AsyncMock(return_value=updated_chunks) + + resp = client.patch( + "/api/analyses/j1/chunks/0", + json={"text": "updated text"}, + ) + assert resp.status_code == 200 + data = resp.json() + assert len(data) == 2 + assert data[0]["text"] == "updated text" + assert data[0]["modified"] is True + assert data[1]["modified"] is False + + def test_update_chunk_text_invalid_index(self, client, mock_analysis_service): + mock_analysis_service.update_chunk_text = AsyncMock( + side_effect=ValueError("Chunk index out of range: 99"), + ) + resp = client.patch( + "/api/analyses/j1/chunks/99", + json={"text": "new"}, + ) + assert resp.status_code == 400 + + def test_update_chunk_text_not_completed(self, client, mock_analysis_service): + mock_analysis_service.update_chunk_text = AsyncMock( + side_effect=ValueError("Analysis is not completed: j1"), + ) + resp = client.patch( + "/api/analyses/j1/chunks/0", + json={"text": "new"}, + ) + assert resp.status_code == 400 + + def test_update_chunk_text_not_found(self, client, mock_analysis_service): + mock_analysis_service.update_chunk_text = AsyncMock( + side_effect=ValueError("Analysis not found: j1"), + ) + resp = client.patch( + "/api/analyses/j1/chunks/0", + json={"text": "new"}, + ) + assert resp.status_code == 400 + + +class TestDeleteChunkEndpoint: + def test_delete_chunk_success(self, client, mock_analysis_service): + updated_chunks = [ + { + "text": "chunk1", + "headings": [], + "sourcePage": 1, + "tokenCount": 10, + "bboxes": [], + "deleted": True, + }, + { + "text": "chunk2", + "headings": [], + "sourcePage": 2, + "tokenCount": 20, + "bboxes": [], + "deleted": False, + }, + ] + mock_analysis_service.delete_chunk = AsyncMock(return_value=updated_chunks) + + resp = client.delete("/api/analyses/j1/chunks/0") + assert resp.status_code == 200 + data = resp.json() + assert len(data) == 2 + assert data[0]["deleted"] is True + assert data[1]["deleted"] is False + + def test_delete_chunk_invalid_index(self, client, mock_analysis_service): + mock_analysis_service.delete_chunk = AsyncMock( + side_effect=ValueError("Chunk index out of range: 99"), + ) + resp = client.delete("/api/analyses/j1/chunks/99") + assert resp.status_code == 400 + + def test_delete_chunk_not_completed(self, client, mock_analysis_service): + mock_analysis_service.delete_chunk = AsyncMock( + side_effect=ValueError("Analysis is not completed: j1"), + ) + resp = client.delete("/api/analyses/j1/chunks/0") + assert resp.status_code == 400 + + class TestRechunkEndpoint: def test_rechunk_success(self, client, mock_analysis_service): mock_analysis_service.rechunk = AsyncMock( diff --git a/document-parser/tests/test_embedding_client.py b/document-parser/tests/test_embedding_client.py new file mode 100644 index 0000000..0a88eb6 --- /dev/null +++ b/document-parser/tests/test_embedding_client.py @@ -0,0 +1,112 @@ +"""Tests for the embedding client adapter (infra.embedding_client). + +Mock httpx to validate adapter logic without running the embedding service. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +from domain.ports import EmbeddingService +from infra.embedding_client import _MAX_BATCH, EmbeddingClient + +# -- Protocol satisfaction ----------------------------------------------------- + + +class TestProtocolSatisfaction: + def test_satisfies_embedding_service_protocol(self) -> None: + client = EmbeddingClient("http://localhost:8001") + assert isinstance(client, EmbeddingService) + + +# -- embed --------------------------------------------------------------------- + + +class TestEmbed: + async def test_returns_empty_for_empty_input(self) -> None: + client = EmbeddingClient("http://localhost:8001") + result = await client.embed([]) + assert result == [] + + async def test_calls_service_and_returns_embeddings(self) -> None: + client = EmbeddingClient("http://localhost:8001") + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = { + "embeddings": [[0.1, 0.2], [0.3, 0.4]], + "model": "all-MiniLM-L6-v2", + "dimension": 2, + } + + mock_http_client = AsyncMock() + mock_http_client.post.return_value = mock_response + mock_http_client.__aenter__ = AsyncMock(return_value=mock_http_client) + mock_http_client.__aexit__ = AsyncMock(return_value=False) + + with patch("infra.embedding_client.httpx.AsyncClient", return_value=mock_http_client): + result = await client.embed(["hello", "world"]) + + assert result == [[0.1, 0.2], [0.3, 0.4]] + mock_http_client.post.assert_awaited_once_with( + "http://localhost:8001/embed", + json={"texts": ["hello", "world"]}, + ) + + async def test_strips_trailing_slash_from_base_url(self) -> None: + client = EmbeddingClient("http://localhost:8001/") + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = {"embeddings": [[0.1]], "model": "m", "dimension": 1} + + mock_http_client = AsyncMock() + mock_http_client.post.return_value = mock_response + mock_http_client.__aenter__ = AsyncMock(return_value=mock_http_client) + mock_http_client.__aexit__ = AsyncMock(return_value=False) + + with patch("infra.embedding_client.httpx.AsyncClient", return_value=mock_http_client): + await client.embed(["test"]) + + mock_http_client.post.assert_awaited_once_with( + "http://localhost:8001/embed", + json={"texts": ["test"]}, + ) + + async def test_splits_large_batches(self) -> None: + client = EmbeddingClient("http://localhost:8001") + texts = [f"text_{i}" for i in range(_MAX_BATCH + 10)] + + call_count = 0 + + def make_response(batch_size: int) -> MagicMock: + resp = MagicMock() + resp.raise_for_status = MagicMock() + resp.json.return_value = { + "embeddings": [[0.1]] * batch_size, + "model": "m", + "dimension": 1, + } + return resp + + async def mock_post(url: str, json: dict) -> MagicMock: + nonlocal call_count + call_count += 1 + return make_response(len(json["texts"])) + + mock_http_client = AsyncMock() + mock_http_client.post = mock_post + mock_http_client.__aenter__ = AsyncMock(return_value=mock_http_client) + mock_http_client.__aexit__ = AsyncMock(return_value=False) + + with patch("infra.embedding_client.httpx.AsyncClient", return_value=mock_http_client): + result = await client.embed(texts) + + assert len(result) == _MAX_BATCH + 10 + assert call_count == 2 # _MAX_BATCH + 10 remaining + + +# -- max batch constant -------------------------------------------------------- + + +class TestMaxBatch: + def test_max_batch_is_256(self) -> None: + assert _MAX_BATCH == 256 diff --git a/document-parser/tests/test_ingestion_api.py b/document-parser/tests/test_ingestion_api.py new file mode 100644 index 0000000..a872d79 --- /dev/null +++ b/document-parser/tests/test_ingestion_api.py @@ -0,0 +1,187 @@ +"""Tests for the ingestion API endpoints (api.ingestion).""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from api.ingestion import router +from domain.models import AnalysisJob +from services.ingestion_service import IngestionResult + + +@pytest.fixture +def mock_ingestion_service() -> AsyncMock: + svc = AsyncMock() + svc.ingest.return_value = IngestionResult( + doc_id="doc-1", chunks_indexed=5, embedding_dimension=384 + ) + svc.delete_document.return_value = 3 + return svc + + +@pytest.fixture +def mock_analysis_service() -> AsyncMock: + svc = AsyncMock() + job = AnalysisJob(document_id="doc-1") + job.document_filename = "test.pdf" + job.mark_running() + job.mark_completed( + markdown="# Test", + html="

Test

", + pages_json="[]", + document_json='{"doc": true}', + chunks_json='[{"text": "hello"}]', + ) + svc.find_by_id.return_value = job + return svc + + +@pytest.fixture +def client(mock_ingestion_service: AsyncMock, mock_analysis_service: AsyncMock) -> TestClient: + app = FastAPI() + app.include_router(router) + app.state.ingestion_service = mock_ingestion_service + app.state.analysis_service = mock_analysis_service + return TestClient(app) + + +class TestIngestAnalysis: + def test_ingest_success(self, client: TestClient) -> None: + resp = client.post("/api/ingestion/job-1") + assert resp.status_code == 200 + data = resp.json() + assert data["docId"] == "doc-1" + assert data["chunksIndexed"] == 5 + assert data["embeddingDimension"] == 384 + + def test_ingest_not_found(self, client: TestClient, mock_analysis_service: AsyncMock) -> None: + mock_analysis_service.find_by_id.return_value = None + resp = client.post("/api/ingestion/missing") + assert resp.status_code == 404 + + def test_ingest_not_completed( + self, client: TestClient, mock_analysis_service: AsyncMock + ) -> None: + job = AnalysisJob(document_id="doc-1") + mock_analysis_service.find_by_id.return_value = job + resp = client.post("/api/ingestion/job-1") + assert resp.status_code == 400 + + def test_ingest_no_chunks(self, client: TestClient, mock_analysis_service: AsyncMock) -> None: + job = AnalysisJob(document_id="doc-1") + job.mark_running() + job.mark_completed(markdown="x", html="x", pages_json="[]") + mock_analysis_service.find_by_id.return_value = job + resp = client.post("/api/ingestion/job-1") + assert resp.status_code == 400 + + +class TestDeleteIngested: + def test_delete_success(self, client: TestClient) -> None: + resp = client.delete("/api/ingestion/doc-1") + assert resp.status_code == 204 + + +class TestIngestionStatus: + def test_available(self, client: TestClient) -> None: + resp = client.get("/api/ingestion/status") + assert resp.status_code == 200 + assert resp.json()["available"] is True + + def test_not_available(self) -> None: + app = FastAPI() + app.include_router(router) + app.state.ingestion_service = None + app.state.analysis_service = AsyncMock() + tc = TestClient(app) + resp = tc.get("/api/ingestion/status") + assert resp.status_code == 200 + assert resp.json()["available"] is False + + +class TestIngestionDisabled: + def test_router_not_mounted_returns_404(self) -> None: + """When ingestion service is None, router should not be mounted → 404.""" + app = FastAPI() + # Do NOT include ingestion router — simulates main.py conditional mount + app.state.ingestion_service = None + app.state.analysis_service = AsyncMock() + tc = TestClient(app) + resp = tc.post("/api/ingestion/job-1") + assert resp.status_code == 404 + + def test_status_still_returns_503_when_router_mounted_but_service_none(self) -> None: + """If router is mounted but service is None, endpoints return 503.""" + app = FastAPI() + app.include_router(router) + app.state.ingestion_service = None + app.state.analysis_service = AsyncMock() + tc = TestClient(app) + resp = tc.post("/api/ingestion/job-1") + assert resp.status_code == 503 + + +class TestStatusOpenSearch: + def test_status_with_opensearch_connected( + self, client: TestClient, mock_ingestion_service: AsyncMock + ) -> None: + mock_ingestion_service.ping.return_value = True + resp = client.get("/api/ingestion/status") + assert resp.status_code == 200 + data = resp.json() + assert data["available"] is True + assert data["opensearchConnected"] is True + + def test_status_with_opensearch_disconnected( + self, client: TestClient, mock_ingestion_service: AsyncMock + ) -> None: + mock_ingestion_service.ping.return_value = False + resp = client.get("/api/ingestion/status") + assert resp.status_code == 200 + data = resp.json() + assert data["available"] is True + assert data["opensearchConnected"] is False + + +class TestSearchEndpoint: + def test_search_success(self, client: TestClient, mock_ingestion_service: AsyncMock) -> None: + from domain.vector_schema import IndexedChunk, SearchResult + + chunk = IndexedChunk( + doc_id="doc-1", + filename="test.pdf", + content="hello world", + embedding=[], + chunk_index=0, + chunk_type="text", + page_number=1, + headings=["Intro"], + ) + mock_ingestion_service.search_fulltext.return_value = [ + SearchResult(chunk=chunk, score=0.95) + ] + resp = client.get("/api/ingestion/search", params={"q": "hello"}) + assert resp.status_code == 200 + data = resp.json() + assert data["total"] == 1 + assert data["query"] == "hello" + assert data["results"][0]["content"] == "hello world" + assert data["results"][0]["score"] == 0.95 + + def test_search_empty_query(self, client: TestClient) -> None: + resp = client.get("/api/ingestion/search", params={"q": ""}) + assert resp.status_code == 422 + + def test_search_with_doc_filter( + self, client: TestClient, mock_ingestion_service: AsyncMock + ) -> None: + mock_ingestion_service.search_fulltext.return_value = [] + resp = client.get("/api/ingestion/search", params={"q": "test", "doc_id": "doc-1"}) + assert resp.status_code == 200 + mock_ingestion_service.search_fulltext.assert_awaited_once_with( + "test", k=20, doc_id="doc-1" + ) diff --git a/document-parser/tests/test_ingestion_service.py b/document-parser/tests/test_ingestion_service.py new file mode 100644 index 0000000..dff801d --- /dev/null +++ b/document-parser/tests/test_ingestion_service.py @@ -0,0 +1,188 @@ +"""Tests for the ingestion service (services.ingestion_service).""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import pytest + +from services.ingestion_service import IngestionConfig, IngestionService + + +def _make_chunks_json(count: int = 3, *, with_deleted: bool = False) -> str: + chunks = [] + for i in range(count): + chunk = { + "text": f"chunk text {i}", + "headings": [f"Heading {i}"], + "sourcePage": i + 1, + "tokenCount": 10, + "bboxes": [{"page": i + 1, "bbox": [0.0, 0.0, 100.0, 50.0]}], + } + if with_deleted and i == count - 1: + chunk["deleted"] = True + chunks.append(chunk) + return json.dumps(chunks) + + +@pytest.fixture +def mock_embedding() -> AsyncMock: + svc = AsyncMock() + svc.embed.return_value = [[0.1, 0.2, 0.3]] * 3 + return svc + + +@pytest.fixture +def mock_vector_store() -> AsyncMock: + store = AsyncMock() + store.ensure_index.return_value = None + store.delete_document.return_value = 0 + store.index_chunks.return_value = 3 + return store + + +@pytest.fixture +def service(mock_embedding: AsyncMock, mock_vector_store: AsyncMock) -> IngestionService: + return IngestionService( + embedding_service=mock_embedding, + vector_store=mock_vector_store, + config=IngestionConfig(index_name="test-idx", embedding_dimension=3), + ) + + +class TestIngest: + async def test_full_pipeline( + self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock + ) -> None: + result = await service.ingest("doc-1", "test.pdf", _make_chunks_json(3)) + + assert result.doc_id == "doc-1" + assert result.chunks_indexed == 3 + mock_embedding.embed.assert_awaited_once() + texts = mock_embedding.embed.call_args[0][0] + assert len(texts) == 3 + mock_vector_store.ensure_index.assert_awaited_once() + mock_vector_store.delete_document.assert_awaited_once_with("test-idx", "doc-1") + mock_vector_store.index_chunks.assert_awaited_once() + indexed = mock_vector_store.index_chunks.call_args[0][1] + assert len(indexed) == 3 + assert indexed[0].doc_id == "doc-1" + assert indexed[0].filename == "test.pdf" + assert indexed[0].embedding == [0.1, 0.2, 0.3] + + async def test_skips_deleted_chunks( + self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock + ) -> None: + mock_embedding.embed.return_value = [[0.1, 0.2, 0.3]] * 2 + mock_vector_store.index_chunks.return_value = 2 + result = await service.ingest("doc-1", "test.pdf", _make_chunks_json(3, with_deleted=True)) + + assert result.chunks_indexed == 2 + texts = mock_embedding.embed.call_args[0][0] + assert len(texts) == 2 + + async def test_empty_chunks( + self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock + ) -> None: + result = await service.ingest("doc-1", "test.pdf", json.dumps([])) + assert result.chunks_indexed == 0 + mock_embedding.embed.assert_not_awaited() + + async def test_idempotent_deletes_old( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_vector_store.delete_document.return_value = 5 + await service.ingest("doc-1", "test.pdf", _make_chunks_json(3)) + mock_vector_store.delete_document.assert_awaited_once_with("test-idx", "doc-1") + + async def test_bbox_conversion( + self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock + ) -> None: + mock_embedding.embed.return_value = [[0.1, 0.2, 0.3]] + mock_vector_store.index_chunks.return_value = 1 + await service.ingest("doc-1", "test.pdf", _make_chunks_json(1)) + indexed = mock_vector_store.index_chunks.call_args[0][1] + bbox = indexed[0].bboxes[0] + assert bbox.x == 0.0 + assert bbox.y == 0.0 + assert bbox.w == 100.0 + assert bbox.h == 50.0 + + async def test_with_binary_hash( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_embedding = service._embedding + mock_embedding.embed.return_value = [[0.1]] * 1 + await service.ingest("doc-1", "test.pdf", _make_chunks_json(1), binary_hash="abc123") + indexed = mock_vector_store.index_chunks.call_args[0][1] + assert indexed[0].origin is not None + assert indexed[0].origin.binary_hash == "abc123" + + +class TestDeleteDocument: + async def test_delegates_to_vector_store( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_vector_store.delete_document.return_value = 3 + result = await service.delete_document("doc-1") + assert result == 3 + + +class TestSearch: + async def test_embeds_and_searches( + self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock + ) -> None: + mock_embedding.embed.return_value = [[0.5, 0.6, 0.7]] + mock_vector_store.search_similar.return_value = [] + await service.search("test query", k=5) + mock_embedding.embed.assert_awaited_once_with(["test query"]) + mock_vector_store.search_similar.assert_awaited_once() + + +class TestSearchFulltext: + async def test_delegates_to_vector_store( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_vector_store.search_fulltext.return_value = [] + await service.search_fulltext("hello world", k=5) + mock_vector_store.search_fulltext.assert_awaited_once_with( + "test-idx", "hello world", k=5, doc_id=None + ) + + async def test_filters_by_doc_id( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_vector_store.search_fulltext.return_value = [] + await service.search_fulltext("hello", doc_id="doc-1") + mock_vector_store.search_fulltext.assert_awaited_once_with( + "test-idx", "hello", k=20, doc_id="doc-1" + ) + + +class TestPing: + async def test_ping_success( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_vector_store._client = AsyncMock() + mock_vector_store._client.info.return_value = {"cluster_name": "test"} + result = await service.ping() + assert result is True + + async def test_ping_failure( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + mock_vector_store._client = AsyncMock() + mock_vector_store._client.info.side_effect = ConnectionError("down") + result = await service.ping() + assert result is False + + +class TestEnsureIndex: + async def test_calls_vector_store( + self, service: IngestionService, mock_vector_store: AsyncMock + ) -> None: + await service.ensure_index() + mock_vector_store.ensure_index.assert_awaited_once() + call_args = mock_vector_store.ensure_index.call_args + assert call_args[0][0] == "test-idx" diff --git a/document-parser/tests/test_opensearch_store.py b/document-parser/tests/test_opensearch_store.py new file mode 100644 index 0000000..e29bf0a --- /dev/null +++ b/document-parser/tests/test_opensearch_store.py @@ -0,0 +1,320 @@ +"""Tests for the OpenSearch adapter (infra.opensearch_store). + +These tests mock the AsyncOpenSearch client to validate adapter logic +without requiring a running OpenSearch instance. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from domain.ports import VectorStore +from domain.vector_schema import ( + ChunkBboxEntry, + ChunkOrigin, + DocItemRef, + IndexedChunk, + SearchResult, + build_index_mapping, +) +from infra.opensearch_store import OpenSearchStore, _hit_to_indexed_chunk, _hit_to_result + +# -- Fixtures ----------------------------------------------------------------- + + +def _make_chunk( + doc_id: str = "doc-1", + chunk_index: int = 0, + content: str = "hello world", + embedding: list[float] | None = None, +) -> IndexedChunk: + return IndexedChunk( + doc_id=doc_id, + filename="test.pdf", + content=content, + embedding=embedding or [0.1, 0.2, 0.3], + chunk_index=chunk_index, + chunk_type="text", + page_number=1, + bboxes=[ChunkBboxEntry(page=1, x=0.0, y=0.0, w=100.0, h=50.0)], + headings=["Chapter 1"], + doc_items=[DocItemRef(self_ref="#/texts/0", label="text")], + origin=ChunkOrigin(binary_hash="abc123", filename="test.pdf"), + ) + + +def _make_hit( + doc_id: str = "doc-1", + chunk_index: int = 0, + score: float = 0.95, + content: str = "hello world", +) -> dict: + return { + "_id": f"{doc_id}_{chunk_index}", + "_score": score, + "_source": { + "doc_id": doc_id, + "filename": "test.pdf", + "content": content, + "chunk_index": chunk_index, + "chunk_type": "text", + "page_number": 1, + "bboxes": [{"page": 1, "x": 0.0, "y": 0.0, "w": 100.0, "h": 50.0}], + "headings": ["Chapter 1"], + "doc_items": [{"self_ref": "#/texts/0", "label": "text"}], + "origin": {"binary_hash": "abc123", "filename": "test.pdf"}, + }, + } + + +@pytest.fixture +def store() -> OpenSearchStore: + return OpenSearchStore("http://localhost:9200") + + +@pytest.fixture +def mock_client(store: OpenSearchStore) -> AsyncMock: + client = AsyncMock() + store._client = client + return client + + +# -- Protocol satisfaction ----------------------------------------------------- + + +class TestProtocolSatisfaction: + def test_satisfies_vector_store_protocol(self) -> None: + """OpenSearchStore structurally satisfies VectorStore Protocol.""" + store = OpenSearchStore("http://localhost:9200") + assert isinstance(store, VectorStore) + + +# -- Hit deserialization ------------------------------------------------------- + + +class TestHitDeserialization: + def test_hit_to_indexed_chunk(self) -> None: + hit = _make_hit() + chunk = _hit_to_indexed_chunk(hit) + assert isinstance(chunk, IndexedChunk) + assert chunk.doc_id == "doc-1" + assert chunk.content == "hello world" + assert chunk.chunk_index == 0 + assert chunk.page_number == 1 + assert len(chunk.bboxes) == 1 + assert chunk.bboxes[0].w == 100.0 + assert len(chunk.doc_items) == 1 + assert chunk.doc_items[0].label == "text" + assert chunk.origin is not None + assert chunk.origin.binary_hash == "abc123" + + def test_hit_to_indexed_chunk_no_origin(self) -> None: + hit = _make_hit() + hit["_source"]["origin"] = None + chunk = _hit_to_indexed_chunk(hit) + assert chunk.origin is None + + def test_hit_to_indexed_chunk_missing_optional_fields(self) -> None: + hit = _make_hit() + del hit["_source"]["bboxes"] + del hit["_source"]["headings"] + del hit["_source"]["doc_items"] + del hit["_source"]["origin"] + chunk = _hit_to_indexed_chunk(hit) + assert chunk.bboxes == [] + assert chunk.headings == [] + assert chunk.doc_items == [] + assert chunk.origin is None + + def test_hit_to_result(self) -> None: + hit = _make_hit(score=0.87) + result = _hit_to_result(hit) + assert isinstance(result, SearchResult) + assert result.score == 0.87 + assert result.chunk.doc_id == "doc-1" + + +# -- ensure_index -------------------------------------------------------------- + + +class TestEnsureIndex: + async def test_creates_index_when_not_exists( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + mock_client.indices.exists.return_value = False + mapping = build_index_mapping() + await store.ensure_index("test-index", mapping) + mock_client.indices.create.assert_awaited_once_with(index="test-index", body=mapping) + + async def test_noop_when_index_exists( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + mock_client.indices.exists.return_value = True + await store.ensure_index("test-index", {}) + mock_client.indices.create.assert_not_awaited() + + +# -- index_chunks -------------------------------------------------------------- + + +class TestIndexChunks: + async def test_bulk_indexes_chunks( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + chunks = [_make_chunk(chunk_index=0), _make_chunk(chunk_index=1)] + mock_client.bulk.return_value = { + "errors": False, + "items": [ + {"index": {"_id": "doc-1_0", "status": 201}}, + {"index": {"_id": "doc-1_1", "status": 201}}, + ], + } + count = await store.index_chunks("test-index", chunks) + assert count == 2 + mock_client.bulk.assert_awaited_once() + + async def test_returns_zero_for_empty_list( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + count = await store.index_chunks("test-index", []) + assert count == 0 + mock_client.bulk.assert_not_awaited() + + async def test_counts_partial_failures( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + chunks = [_make_chunk(chunk_index=0), _make_chunk(chunk_index=1)] + mock_client.bulk.return_value = { + "errors": True, + "items": [ + {"index": {"_id": "doc-1_0", "status": 201}}, + {"index": {"_id": "doc-1_1", "error": {"reason": "mapping"}}}, + ], + } + count = await store.index_chunks("test-index", chunks) + assert count == 1 + + async def test_bulk_body_structure( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + chunk = _make_chunk(doc_id="d1", chunk_index=3) + mock_client.bulk.return_value = { + "errors": False, + "items": [{"index": {"_id": "d1_3", "status": 201}}], + } + await store.index_chunks("idx", [chunk]) + call_body = mock_client.bulk.call_args[1]["body"] + assert call_body[0] == {"index": {"_index": "idx", "_id": "d1_3"}} + assert call_body[1]["doc_id"] == "d1" + assert call_body[1]["chunk_index"] == 3 + + +# -- search_similar ------------------------------------------------------------ + + +class TestSearchSimilar: + async def test_knn_search(self, store: OpenSearchStore, mock_client: AsyncMock) -> None: + mock_client.search.return_value = {"hits": {"hits": [_make_hit(score=0.99)]}} + results = await store.search_similar("idx", [0.1, 0.2, 0.3], k=5) + assert len(results) == 1 + assert results[0].score == 0.99 + call_body = mock_client.search.call_args[1]["body"] + assert call_body["query"]["knn"]["embedding"]["vector"] == [0.1, 0.2, 0.3] + assert call_body["query"]["knn"]["embedding"]["k"] == 5 + + async def test_knn_search_with_doc_filter( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + mock_client.search.return_value = {"hits": {"hits": []}} + await store.search_similar("idx", [0.1], doc_id="doc-42") + call_body = mock_client.search.call_args[1]["body"] + assert call_body["query"]["knn"]["embedding"]["filter"] == {"term": {"doc_id": "doc-42"}} + + async def test_knn_search_no_filter_by_default( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + mock_client.search.return_value = {"hits": {"hits": []}} + await store.search_similar("idx", [0.1]) + call_body = mock_client.search.call_args[1]["body"] + assert "filter" not in call_body["query"]["knn"]["embedding"] + + +# -- get_chunks ---------------------------------------------------------------- + + +class TestGetChunks: + async def test_retrieves_by_doc_id( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + mock_client.search.return_value = { + "hits": {"hits": [_make_hit(chunk_index=0), _make_hit(chunk_index=1)]} + } + results = await store.get_chunks("idx", "doc-1") + assert len(results) == 2 + call_body = mock_client.search.call_args[1]["body"] + assert call_body["query"] == {"term": {"doc_id": "doc-1"}} + assert call_body["sort"] == [{"chunk_index": {"order": "asc"}}] + + async def test_respects_limit(self, store: OpenSearchStore, mock_client: AsyncMock) -> None: + mock_client.search.return_value = {"hits": {"hits": []}} + await store.get_chunks("idx", "doc-1", limit=50) + call_body = mock_client.search.call_args[1]["body"] + assert call_body["size"] == 50 + + +# -- delete_document ----------------------------------------------------------- + + +class TestDeleteDocument: + async def test_deletes_by_doc_id(self, store: OpenSearchStore, mock_client: AsyncMock) -> None: + mock_client.delete_by_query.return_value = {"deleted": 5} + count = await store.delete_document("idx", "doc-1") + assert count == 5 + call_body = mock_client.delete_by_query.call_args[1]["body"] + assert call_body["query"] == {"term": {"doc_id": "doc-1"}} + + async def test_returns_zero_on_not_found( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + from opensearchpy import NotFoundError + + mock_client.delete_by_query.side_effect = NotFoundError(404, "index_not_found") + count = await store.delete_document("idx", "doc-1") + assert count == 0 + + +# -- search_fulltext ----------------------------------------------------------- + + +class TestSearchFulltext: + async def test_fulltext_search(self, store: OpenSearchStore, mock_client: AsyncMock) -> None: + mock_client.search.return_value = { + "hits": {"hits": [_make_hit(content="matching text", score=1.5)]} + } + results = await store.search_fulltext("idx", "matching") + assert len(results) == 1 + assert results[0].chunk.content == "matching text" + call_body = mock_client.search.call_args[1]["body"] + assert {"match": {"content": "matching"}} in call_body["query"]["bool"]["must"] + + async def test_fulltext_search_with_doc_filter( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + mock_client.search.return_value = {"hits": {"hits": []}} + await store.search_fulltext("idx", "query", doc_id="doc-5") + call_body = mock_client.search.call_args[1]["body"] + must_clauses = call_body["query"]["bool"]["must"] + assert {"term": {"doc_id": "doc-5"}} in must_clauses + + +# -- close --------------------------------------------------------------------- + + +class TestClose: + async def test_close_delegates_to_client( + self, store: OpenSearchStore, mock_client: AsyncMock + ) -> None: + await store.close() + mock_client.close.assert_awaited_once() diff --git a/document-parser/tests/test_vector_schema.py b/document-parser/tests/test_vector_schema.py new file mode 100644 index 0000000..960fcb4 --- /dev/null +++ b/document-parser/tests/test_vector_schema.py @@ -0,0 +1,228 @@ +"""Tests for vector index schema — value objects and OpenSearch mapping.""" + +from __future__ import annotations + +import pytest + +from domain.vector_schema import ( + DEFAULT_EMBEDDING_DIMENSION, + DEFAULT_INDEX_NAME, + ChunkBboxEntry, + ChunkOrigin, + DocItemRef, + IndexedChunk, + SearchResult, + build_index_mapping, +) + + +class TestChunkBboxEntry: + def test_construction(self): + bbox = ChunkBboxEntry(page=1, x=10.0, y=20.0, w=100.0, h=50.0) + assert bbox.page == 1 + assert bbox.x == 10.0 + assert bbox.w == 100.0 + + def test_frozen(self): + bbox = ChunkBboxEntry(page=1, x=0, y=0, w=10, h=10) + with pytest.raises(AttributeError): + bbox.page = 2 # type: ignore[misc] + + +class TestDocItemRef: + def test_construction(self): + ref = DocItemRef(self_ref="#/texts/0", label="text") + assert ref.self_ref == "#/texts/0" + assert ref.label == "text" + + +class TestChunkOrigin: + def test_construction(self): + origin = ChunkOrigin(binary_hash="abc123", filename="doc.pdf") + assert origin.binary_hash == "abc123" + assert origin.filename == "doc.pdf" + + +class TestIndexedChunk: + def _make_chunk(self, **overrides) -> IndexedChunk: + defaults = { + "doc_id": "doc-1", + "filename": "test.pdf", + "content": "Hello world", + "embedding": [0.1] * 384, + "chunk_index": 0, + "chunk_type": "text", + "page_number": 1, + } + defaults.update(overrides) + return IndexedChunk(**defaults) + + def test_minimal_chunk(self): + chunk = self._make_chunk() + assert chunk.doc_id == "doc-1" + assert chunk.content == "Hello world" + assert chunk.bboxes == [] + assert chunk.headings == [] + assert chunk.doc_items == [] + assert chunk.origin is None + + def test_full_chunk(self): + chunk = self._make_chunk( + bboxes=[ChunkBboxEntry(page=1, x=10, y=20, w=100, h=50)], + headings=["Chapter 1", "Section A"], + doc_items=[DocItemRef(self_ref="#/texts/0", label="text")], + origin=ChunkOrigin(binary_hash="abc", filename="test.pdf"), + ) + assert len(chunk.bboxes) == 1 + assert chunk.headings == ["Chapter 1", "Section A"] + assert chunk.doc_items[0].label == "text" + assert chunk.origin.binary_hash == "abc" + + def test_to_dict_minimal(self): + chunk = self._make_chunk() + d = chunk.to_dict() + assert d["doc_id"] == "doc-1" + assert d["filename"] == "test.pdf" + assert d["content"] == "Hello world" + assert d["embedding"] == [0.1] * 384 + assert d["chunk_index"] == 0 + assert d["chunk_type"] == "text" + assert d["page_number"] == 1 + assert d["bboxes"] == [] + assert d["headings"] == [] + assert d["doc_items"] == [] + assert "origin" not in d + + def test_to_dict_full(self): + chunk = self._make_chunk( + bboxes=[ChunkBboxEntry(page=1, x=10.5, y=20.0, w=100.0, h=50.0)], + headings=["H1"], + doc_items=[DocItemRef(self_ref="#/texts/0", label="text")], + origin=ChunkOrigin(binary_hash="abc", filename="test.pdf"), + ) + d = chunk.to_dict() + assert d["bboxes"] == [{"page": 1, "x": 10.5, "y": 20.0, "w": 100.0, "h": 50.0}] + assert d["headings"] == ["H1"] + assert d["doc_items"] == [{"self_ref": "#/texts/0", "label": "text"}] + assert d["origin"] == {"binary_hash": "abc", "filename": "test.pdf"} + + def test_frozen(self): + chunk = self._make_chunk() + with pytest.raises(AttributeError): + chunk.content = "modified" # type: ignore[misc] + + +class TestBuildIndexMapping: + def test_default_dimension(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + assert props["embedding"]["dimension"] == 384 + assert props["embedding"]["type"] == "knn_vector" + assert props["embedding"]["method"]["engine"] == "faiss" + assert props["embedding"]["method"]["name"] == "hnsw" + + def test_custom_dimension(self): + mapping = build_index_mapping(embedding_dimension=768) + assert mapping["mappings"]["properties"]["embedding"]["dimension"] == 768 + + def test_knn_enabled(self): + mapping = build_index_mapping() + assert mapping["settings"]["index"]["knn"] is True + + def test_all_fields_present(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + expected_fields = { + "doc_id", + "filename", + "content", + "embedding", + "chunk_index", + "chunk_type", + "page_number", + "bboxes", + "headings", + "doc_items", + "origin", + } + assert set(props.keys()) == expected_fields + + def test_bboxes_nested_type(self): + mapping = build_index_mapping() + bboxes = mapping["mappings"]["properties"]["bboxes"] + assert bboxes["type"] == "nested" + assert "page" in bboxes["properties"] + assert "x" in bboxes["properties"] + assert "y" in bboxes["properties"] + assert "w" in bboxes["properties"] + assert "h" in bboxes["properties"] + + def test_doc_items_nested_type(self): + mapping = build_index_mapping() + doc_items = mapping["mappings"]["properties"]["doc_items"] + assert doc_items["type"] == "nested" + assert "self_ref" in doc_items["properties"] + assert "label" in doc_items["properties"] + + def test_origin_object_type(self): + mapping = build_index_mapping() + origin = mapping["mappings"]["properties"]["origin"] + assert origin["type"] == "object" + assert "binary_hash" in origin["properties"] + assert "filename" in origin["properties"] + + def test_content_uses_standard_analyzer(self): + mapping = build_index_mapping() + content = mapping["mappings"]["properties"]["content"] + assert content["type"] == "text" + assert content["analyzer"] == "standard" + + def test_keyword_fields(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + for field_name in ("doc_id", "filename", "chunk_type"): + assert props[field_name]["type"] == "keyword", f"{field_name} should be keyword" + + def test_integer_fields(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + for field_name in ("chunk_index", "page_number"): + assert props[field_name]["type"] == "integer", f"{field_name} should be integer" + + +class TestSearchResult: + def test_construction(self): + chunk = IndexedChunk( + doc_id="doc-1", + filename="test.pdf", + content="Hello", + embedding=[0.1] * 384, + chunk_index=0, + chunk_type="text", + page_number=1, + ) + result = SearchResult(chunk=chunk, score=0.95) + assert result.chunk.content == "Hello" + assert result.score == 0.95 + + def test_frozen(self): + chunk = IndexedChunk( + doc_id="d", + filename="f", + content="c", + embedding=[], + chunk_index=0, + chunk_type="text", + page_number=1, + ) + result = SearchResult(chunk=chunk, score=0.5) + with pytest.raises(AttributeError): + result.score = 0.9 # type: ignore[misc] + + +class TestConstants: + def test_default_embedding_dimension(self): + assert DEFAULT_EMBEDDING_DIMENSION == 384 + + def test_default_index_name(self): + assert DEFAULT_INDEX_NAME == "docling-studio-chunks" diff --git a/document-parser/tests/test_vector_store_port.py b/document-parser/tests/test_vector_store_port.py new file mode 100644 index 0000000..eea70d7 --- /dev/null +++ b/document-parser/tests/test_vector_store_port.py @@ -0,0 +1,113 @@ +"""Tests for VectorStore port — verify the protocol contract is implementable.""" + +from __future__ import annotations + +import pytest + +from domain.ports import VectorStore +from domain.vector_schema import IndexedChunk, SearchResult + + +class FakeVectorStore: + """Minimal concrete implementation to verify the protocol is implementable.""" + + async def ensure_index(self, index_name: str, mapping: dict) -> None: + pass + + async def index_chunks(self, index_name: str, chunks: list[IndexedChunk]) -> int: + return len(chunks) + + async def search_similar( + self, + index_name: str, + embedding: list[float], + *, + k: int = 10, + doc_id: str | None = None, + ) -> list[SearchResult]: + return [] + + async def get_chunks( + self, + index_name: str, + doc_id: str, + *, + limit: int = 1000, + ) -> list[SearchResult]: + return [] + + async def delete_document(self, index_name: str, doc_id: str) -> int: + return 0 + + +class TestVectorStorePort: + def test_fake_satisfies_protocol(self): + """A class implementing all methods is accepted as a VectorStore.""" + store: VectorStore = FakeVectorStore() + assert store is not None + + @pytest.mark.asyncio + async def test_ensure_index(self): + store = FakeVectorStore() + await store.ensure_index("test-index", {"mappings": {}}) + + @pytest.mark.asyncio + async def test_index_chunks(self): + store = FakeVectorStore() + chunk = IndexedChunk( + doc_id="d1", + filename="test.pdf", + content="Hello", + embedding=[0.1] * 384, + chunk_index=0, + chunk_type="text", + page_number=1, + ) + count = await store.index_chunks("test-index", [chunk]) + assert count == 1 + + @pytest.mark.asyncio + async def test_search_similar(self): + store = FakeVectorStore() + results = await store.search_similar("test-index", [0.1] * 384, k=5) + assert results == [] + + @pytest.mark.asyncio + async def test_search_similar_with_doc_filter(self): + store = FakeVectorStore() + results = await store.search_similar("test-index", [0.1] * 384, k=5, doc_id="d1") + assert results == [] + + @pytest.mark.asyncio + async def test_get_chunks(self): + store = FakeVectorStore() + results = await store.get_chunks("test-index", "d1") + assert results == [] + + @pytest.mark.asyncio + async def test_get_chunks_with_limit(self): + store = FakeVectorStore() + results = await store.get_chunks("test-index", "d1", limit=50) + assert results == [] + + @pytest.mark.asyncio + async def test_delete_document(self): + store = FakeVectorStore() + count = await store.delete_document("test-index", "d1") + assert count == 0 + + def test_protocol_methods_list(self): + """Verify the protocol exposes the expected methods.""" + expected = { + "ensure_index", + "index_chunks", + "search_similar", + "get_chunks", + "delete_document", + } + protocol_methods = { + name + for name in dir(VectorStore) + if not name.startswith("_") and callable(getattr(VectorStore, name, None)) + } + assert expected.issubset(protocol_methods) diff --git a/e2e/api/src/test/resources/ingestion/ingest-and-verify.feature b/e2e/api/src/test/resources/ingestion/ingest-and-verify.feature new file mode 100644 index 0000000..46ad687 --- /dev/null +++ b/e2e/api/src/test/resources/ingestion/ingest-and-verify.feature @@ -0,0 +1,59 @@ +@e2e @ingestion +Feature: Ingestion pipeline — PDF → chunks → embeddings → OpenSearch + + Background: + * url baseUrl + + Scenario: Upload PDF, analyze with chunking, ingest into OpenSearch, verify + + # Step 1: Check ingestion is available + Given path '/api/ingestion/status' + When method GET + Then status 200 + And match response.available == true + + # Step 2: Upload a PDF + Given path '/api/documents/upload' + And multipart file file = { read: 'classpath:common/data/generated/medium.pdf', filename: 'medium.pdf', contentType: 'application/pdf' } + When method POST + Then status 200 + * def docId = response.id + + # Step 3: Create analysis with chunking + Given path '/api/analyses' + And request { documentId: '#(docId)', pipelineOptions: { doOcr: true, tableMode: 'fast' }, chunkingOptions: { chunkerType: 'hybrid', maxTokens: 256 } } + When method POST + Then status 200 + * def jobId = response.id + + # Step 4: Poll until completed + Given path '/api/analyses', jobId + And retry until response.status == 'COMPLETED' || response.status == 'FAILED' + When method GET + Then status 200 + And match response.status == 'COMPLETED' + And match response.chunksJson == '#string' + + # Step 5: Trigger ingestion + Given path '/api/ingestion', jobId + When method POST + Then status 200 + And match response.docId == docId + And match response.chunksIndexed == '#number' + And assert response.chunksIndexed > 0 + And match response.embeddingDimension == '#number' + And assert response.embeddingDimension > 0 + + # Step 6: Cleanup — delete ingested data + Given path '/api/ingestion', docId + When method DELETE + Then status 204 + + # Step 7: Cleanup — delete analysis and document + Given path '/api/analyses', jobId + When method DELETE + Then status 204 + + Given path '/api/documents', docId + When method DELETE + Then status 204 diff --git a/e2e/ui/src/test/resources/analyses/analysis.feature b/e2e/ui/src/test/resources/analyses/analysis.feature index 42fc8dc..9b6824c 100644 --- a/e2e/ui/src/test/resources/analyses/analysis.feature +++ b/e2e/ui/src/test/resources/analyses/analysis.feature @@ -18,7 +18,7 @@ Feature: UI — Launch an analysis and verify results * waitFor('[data-e2e=doc-item].selected') # Verify we are in Configure mode (first toggle button is active) - * waitFor('[data-e2e=toggle-btn].active') + * waitFor('[data-e2e~=configure-btn].active') # Click Run / Exécuter * waitFor('[data-e2e=run-btn]') diff --git a/e2e/ui/src/test/resources/analyses/pipeline-options.feature b/e2e/ui/src/test/resources/analyses/pipeline-options.feature index 1d33609..add592d 100644 --- a/e2e/ui/src/test/resources/analyses/pipeline-options.feature +++ b/e2e/ui/src/test/resources/analyses/pipeline-options.feature @@ -51,10 +51,9 @@ Feature: UI — Pipeline configuration options * select('[data-e2e=config-select]', 'fast') # Switch to Verify mode and back - * def toggleBtns = locateAll('[data-e2e=toggle-btn]') - * toggleBtns[1].click() - * waitFor('[data-e2e=toggle-btn].active') - * toggleBtns[0].click() + * click('[data-e2e~=verify-btn]') + * waitFor('[data-e2e~=verify-btn].active') + * click('[data-e2e~=configure-btn]') * waitFor('[data-e2e=config-select]') # Verify table mode is still fast diff --git a/e2e/ui/src/test/resources/analyses/rechunk.feature b/e2e/ui/src/test/resources/analyses/rechunk.feature index e409343..33ff945 100644 --- a/e2e/ui/src/test/resources/analyses/rechunk.feature +++ b/e2e/ui/src/test/resources/analyses/rechunk.feature @@ -25,9 +25,9 @@ Feature: UI — Rechunk an analysis with different parameters * call read('classpath:common/helpers/ui-wait-analysis.feature') * waitFor('[data-e2e=result-tabs]') - # Now Préparer toggle should be enabled — click the last one - * def toggleBtns = locateAll('[data-e2e=toggle-btn]') - * toggleBtns[karate.sizeOf(toggleBtns) - 1].click() + # Switch to Prepare tab — use dedicated selector (avoids race with feature flag load) + * waitFor('[data-e2e~=prepare-btn]') + * click('[data-e2e~=prepare-btn]') # Wait for chunk panel to load * waitFor('[data-e2e=chunk-panel]') diff --git a/e2e/ui/src/test/resources/workflows/full-ui-path.feature b/e2e/ui/src/test/resources/workflows/full-ui-path.feature index 6790a53..e2a67ac 100644 --- a/e2e/ui/src/test/resources/workflows/full-ui-path.feature +++ b/e2e/ui/src/test/resources/workflows/full-ui-path.feature @@ -22,7 +22,7 @@ Feature: UI — Full happy path via browser * waitFor('[data-e2e=doc-item].selected') # Step 5: Verify Configure mode is active - * waitFor('[data-e2e=toggle-btn].active') + * waitFor('[data-e2e~=configure-btn].active') # Step 6: Run the analysis * click('[data-e2e=run-btn]') @@ -46,8 +46,8 @@ Feature: UI — Full happy path via browser * match text('[data-e2e=raw-content]') != '' # Step 9: Switch to Préparer mode and rechunk - * def toggleBtns = locateAll('[data-e2e=toggle-btn]') - * toggleBtns[karate.sizeOf(toggleBtns) - 1].click() + * waitFor('[data-e2e~=prepare-btn]') + * click('[data-e2e~=prepare-btn]') * waitFor('[data-e2e=chunk-panel]') # Expand config if needed @@ -66,8 +66,7 @@ Feature: UI — Full happy path via browser * assert karate.sizeOf(locateAll('[data-e2e=chunk-card]')) > 0 # Step 10: Delete the document via UI - * def toggleBtns2 = locateAll('[data-e2e=toggle-btn]') - * toggleBtns2[0].click() + * click('[data-e2e~=configure-btn]') * waitFor('[data-e2e=doc-item]') # Hover and delete diff --git a/embedding-service/Dockerfile b/embedding-service/Dockerfile new file mode 100644 index 0000000..7311e03 --- /dev/null +++ b/embedding-service/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* + +# Install dependencies first (cache layer) +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Pre-download default model into the image +ARG EMBEDDING_MODEL=all-MiniLM-L6-v2 +RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('${EMBEDDING_MODEL}')" + +COPY main.py . + +EXPOSE 8001 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001"] diff --git a/embedding-service/main.py b/embedding-service/main.py new file mode 100644 index 0000000..497a5a2 --- /dev/null +++ b/embedding-service/main.py @@ -0,0 +1,89 @@ +"""Embedding microservice — exposes sentence-transformers models via REST API. + +POST /embed {"texts": ["...", "..."]} → {"embeddings": [[...], [...]], "model": "...", "dimension": N} +GET /health → {"status": "ok", "model": "...", "dimension": N} +""" + +from __future__ import annotations + +import logging +import os +import time + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field +from sentence_transformers import SentenceTransformer + +logger = logging.getLogger(__name__) + +MODEL_NAME = os.environ.get("EMBEDDING_MODEL", "all-MiniLM-L6-v2") +BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", "64")) + +app = FastAPI(title="Docling Studio — Embedding Service", version="0.4.0") + +# Load model at startup (downloaded / cached in HF cache dir) +model: SentenceTransformer | None = None + + +@app.on_event("startup") +async def _load_model() -> None: + global model # noqa: PLW0603 + logger.info("Loading sentence-transformers model '%s' …", MODEL_NAME) + t0 = time.monotonic() + model = SentenceTransformer(MODEL_NAME) + elapsed = time.monotonic() - t0 + dim = model.get_sentence_embedding_dimension() + logger.info("Model loaded in %.1fs — dimension=%d", elapsed, dim) + + +# -- Schemas ------------------------------------------------------------------- + + +class EmbedRequest(BaseModel): + texts: list[str] = Field(..., min_length=1, description="Texts to embed") + + +class EmbedResponse(BaseModel): + embeddings: list[list[float]] + model: str + dimension: int + + +class HealthResponse(BaseModel): + status: str + model: str + dimension: int + + +# -- Endpoints ----------------------------------------------------------------- + + +@app.post("/embed", response_model=EmbedResponse) +async def embed(request: EmbedRequest) -> EmbedResponse: + """Generate embeddings for a batch of texts.""" + if model is None: + raise HTTPException(status_code=503, detail="Model not loaded yet") + + vectors = model.encode( + request.texts, + batch_size=BATCH_SIZE, + show_progress_bar=False, + normalize_embeddings=True, + ) + return EmbedResponse( + embeddings=vectors.tolist(), + model=MODEL_NAME, + dimension=model.get_sentence_embedding_dimension(), + ) + + +@app.get("/health", response_model=HealthResponse) +async def health() -> HealthResponse: + """Health check — verifies the model is loaded.""" + if model is None: + raise HTTPException(status_code=503, detail="Model not loaded yet") + return HealthResponse( + status="ok", + model=MODEL_NAME, + dimension=model.get_sentence_embedding_dimension(), + ) diff --git a/embedding-service/requirements.txt b/embedding-service/requirements.txt new file mode 100644 index 0000000..abe998e --- /dev/null +++ b/embedding-service/requirements.txt @@ -0,0 +1,3 @@ +fastapi>=0.115.0,<1.0.0 +uvicorn[standard]>=0.32.0,<1.0.0 +sentence-transformers>=3.0.0,<4.0.0 diff --git a/embedding-service/test_main.py b/embedding-service/test_main.py new file mode 100644 index 0000000..e90d01b --- /dev/null +++ b/embedding-service/test_main.py @@ -0,0 +1,64 @@ +"""Tests for the embedding microservice API.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +from fastapi.testclient import TestClient + +import main + + +@pytest.fixture(autouse=True) +def _mock_model() -> None: + """Inject a mock SentenceTransformer model for all tests.""" + mock = MagicMock() + mock.get_sentence_embedding_dimension.return_value = 3 + mock.encode.return_value = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]) + main.model = mock + yield + main.model = None + + +@pytest.fixture +def client() -> TestClient: + return TestClient(main.app) + + +class TestEmbed: + def test_embed_returns_vectors(self, client: TestClient) -> None: + resp = client.post("/embed", json={"texts": ["hello", "world"]}) + assert resp.status_code == 200 + data = resp.json() + assert len(data["embeddings"]) == 2 + assert data["dimension"] == 3 + assert data["model"] == main.MODEL_NAME + + def test_embed_empty_texts_rejected(self, client: TestClient) -> None: + resp = client.post("/embed", json={"texts": []}) + assert resp.status_code == 422 + + def test_embed_missing_texts(self, client: TestClient) -> None: + resp = client.post("/embed", json={}) + assert resp.status_code == 422 + + def test_embed_model_not_loaded(self, client: TestClient) -> None: + main.model = None + resp = client.post("/embed", json={"texts": ["test"]}) + assert resp.status_code == 503 + + +class TestHealth: + def test_health_ok(self, client: TestClient) -> None: + resp = client.get("/health") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "ok" + assert data["dimension"] == 3 + + def test_health_model_not_loaded(self, client: TestClient) -> None: + main.model = None + resp = client.get("/health") + assert resp.status_code == 503 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 469c71b..b6e3d75 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "docling-studio", - "version": "0.3.0", + "version": "0.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "docling-studio", - "version": "0.3.0", + "version": "0.3.1", "dependencies": { "dompurify": "^3.3.3", "marked": "^17.0.4", diff --git a/frontend/package.json b/frontend/package.json index 5c7dcff..3c7c581 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "docling-studio", - "version": "0.3.1", + "version": "0.4.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/app/router/index.ts b/frontend/src/app/router/index.ts index e78265e..19f8141 100644 --- a/frontend/src/app/router/index.ts +++ b/frontend/src/app/router/index.ts @@ -22,6 +22,11 @@ const routes: RouteRecordRaw[] = [ name: 'documents', component: () => import('../../pages/DocumentsPage.vue'), }, + { + path: '/search', + name: 'search', + component: () => import('../../pages/SearchPage.vue'), + }, { path: '/settings', name: 'settings', diff --git a/frontend/src/features/analysis/store.ts b/frontend/src/features/analysis/store.ts index 06bb1b9..146c89b 100644 --- a/frontend/src/features/analysis/store.ts +++ b/frontend/src/features/analysis/store.ts @@ -111,6 +111,15 @@ export const useAnalysisStore = defineStore('analysis', () => { } } + function updateChunks(chunks: Chunk[]): void { + if (currentAnalysis.value) { + currentAnalysis.value = { + ...currentAnalysis.value, + chunksJson: JSON.stringify(chunks), + } + } + } + async function select(id: string): Promise { try { currentAnalysis.value = await api.fetchAnalysis(id) @@ -142,6 +151,7 @@ export const useAnalysisStore = defineStore('analysis', () => { load, run, select, + updateChunks, remove, stopPolling, } diff --git a/frontend/src/features/chunking/api.test.ts b/frontend/src/features/chunking/api.test.ts index 4fbd990..fce35d3 100644 --- a/frontend/src/features/chunking/api.test.ts +++ b/frontend/src/features/chunking/api.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { rechunkAnalysis } from './api' +import { rechunkAnalysis, updateChunkText, deleteChunk } from './api' vi.mock('../../shared/api/http', () => ({ apiFetch: vi.fn(), @@ -25,4 +25,33 @@ describe('chunking API', () => { }) expect(result).toEqual(chunks) }) + + it('updateChunkText sends PATCH to chunk endpoint', async () => { + const chunks = [ + { text: 'updated', headings: [], sourcePage: 1, tokenCount: 10, bboxes: [], modified: true }, + ] + apiFetch.mockResolvedValue(chunks) + + const result = await updateChunkText('job-1', 0, 'updated') + + expect(apiFetch).toHaveBeenCalledWith('/api/analyses/job-1/chunks/0', { + method: 'PATCH', + body: JSON.stringify({ text: 'updated' }), + }) + expect(result).toEqual(chunks) + }) + + it('deleteChunk sends DELETE to chunk endpoint', async () => { + const chunks = [ + { text: 'chunk1', headings: [], sourcePage: 1, tokenCount: 10, bboxes: [], deleted: true }, + ] + apiFetch.mockResolvedValue(chunks) + + const result = await deleteChunk('job-1', 0) + + expect(apiFetch).toHaveBeenCalledWith('/api/analyses/job-1/chunks/0', { + method: 'DELETE', + }) + expect(result).toEqual(chunks) + }) }) diff --git a/frontend/src/features/chunking/api.ts b/frontend/src/features/chunking/api.ts index 5a79601..7729829 100644 --- a/frontend/src/features/chunking/api.ts +++ b/frontend/src/features/chunking/api.ts @@ -7,3 +7,16 @@ export function rechunkAnalysis(jobId: string, chunkingOptions: ChunkingOptions) body: JSON.stringify({ chunkingOptions }), }) } + +export function updateChunkText(jobId: string, chunkIndex: number, text: string): Promise { + return apiFetch(`/api/analyses/${jobId}/chunks/${chunkIndex}`, { + method: 'PATCH', + body: JSON.stringify({ text }), + }) +} + +export function deleteChunk(jobId: string, chunkIndex: number): Promise { + return apiFetch(`/api/analyses/${jobId}/chunks/${chunkIndex}`, { + method: 'DELETE', + }) +} diff --git a/frontend/src/features/chunking/store.test.ts b/frontend/src/features/chunking/store.test.ts index e7296f4..9f55991 100644 --- a/frontend/src/features/chunking/store.test.ts +++ b/frontend/src/features/chunking/store.test.ts @@ -4,6 +4,8 @@ import { useChunkingStore } from './store' vi.mock('./api', () => ({ rechunkAnalysis: vi.fn(), + updateChunkText: vi.fn(), + deleteChunk: vi.fn(), })) import * as api from './api' @@ -17,6 +19,8 @@ describe('useChunkingStore', () => { it('starts with default state', () => { const store = useChunkingStore() expect(store.rechunking).toBe(false) + expect(store.saving).toBe(false) + expect(store.deleting).toBe(false) expect(store.error).toBeNull() }) @@ -62,4 +66,88 @@ describe('useChunkingStore', () => { expect(store.rechunking).toBe(false) expect(store.error).toBe('fail') }) + + it('updateChunkText calls API and returns chunks', async () => { + const chunks = [ + { text: 'updated', headings: [], sourcePage: 1, tokenCount: 5, bboxes: [], modified: true }, + ] + vi.mocked(api.updateChunkText).mockResolvedValue(chunks) + + const store = useChunkingStore() + const result = await store.updateChunkText('j1', 0, 'updated') + + expect(api.updateChunkText).toHaveBeenCalledWith('j1', 0, 'updated') + expect(result).toEqual(chunks) + expect(store.saving).toBe(false) + }) + + it('updateChunkText sets saving during execution', async () => { + let resolve: (v: any) => void + vi.mocked(api.updateChunkText).mockImplementation( + () => + new Promise((r) => { + resolve = r + }), + ) + + const store = useChunkingStore() + const promise = store.updateChunkText('j1', 0, 'updated') + + expect(store.saving).toBe(true) + resolve!([]) + await promise + expect(store.saving).toBe(false) + }) + + it('updateChunkText handles errors', async () => { + vi.mocked(api.updateChunkText).mockRejectedValue(new Error('save failed')) + vi.spyOn(console, 'error').mockImplementation(() => {}) + + const store = useChunkingStore() + await expect(store.updateChunkText('j1', 0, 'text')).rejects.toThrow('save failed') + expect(store.saving).toBe(false) + expect(store.error).toBe('save failed') + }) + + it('deleteChunk calls API and returns chunks', async () => { + const chunks = [ + { text: 'chunk1', headings: [], sourcePage: 1, tokenCount: 5, bboxes: [], deleted: true }, + ] + vi.mocked(api.deleteChunk).mockResolvedValue(chunks) + + const store = useChunkingStore() + const result = await store.deleteChunk('j1', 0) + + expect(api.deleteChunk).toHaveBeenCalledWith('j1', 0) + expect(result).toEqual(chunks) + expect(store.deleting).toBe(false) + }) + + it('deleteChunk sets deleting during execution', async () => { + let resolve: (v: any) => void + vi.mocked(api.deleteChunk).mockImplementation( + () => + new Promise((r) => { + resolve = r + }), + ) + + const store = useChunkingStore() + const promise = store.deleteChunk('j1', 0) + + expect(store.deleting).toBe(true) + resolve!([]) + await promise + expect(store.deleting).toBe(false) + }) + + it('deleteChunk handles errors', async () => { + vi.mocked(api.deleteChunk).mockRejectedValue(new Error('delete failed')) + vi.spyOn(console, 'error').mockImplementation(() => {}) + + const store = useChunkingStore() + await expect(store.deleteChunk('j1', 0)).rejects.toThrow('delete failed') + expect(store.deleting).toBe(false) + expect(store.error).toBe('delete failed') + }) }) diff --git a/frontend/src/features/chunking/store.ts b/frontend/src/features/chunking/store.ts index 86feaa2..b34b3c2 100644 --- a/frontend/src/features/chunking/store.ts +++ b/frontend/src/features/chunking/store.ts @@ -5,6 +5,8 @@ import * as api from './api' export const useChunkingStore = defineStore('chunking', () => { const rechunking = ref(false) + const saving = ref(false) + const deleting = ref(false) const error = ref(null) async function rechunk(jobId: string, chunkingOptions: ChunkingOptions): Promise { @@ -21,5 +23,37 @@ export const useChunkingStore = defineStore('chunking', () => { } } - return { rechunking, error, rechunk } + async function updateChunkText( + jobId: string, + chunkIndex: number, + text: string, + ): Promise { + saving.value = true + error.value = null + try { + return await api.updateChunkText(jobId, chunkIndex, text) + } catch (e) { + error.value = (e as Error).message || 'Failed to update chunk' + console.error('Failed to update chunk', e) + throw e + } finally { + saving.value = false + } + } + + async function deleteChunk(jobId: string, chunkIndex: number): Promise { + deleting.value = true + error.value = null + try { + return await api.deleteChunk(jobId, chunkIndex) + } catch (e) { + error.value = (e as Error).message || 'Failed to delete chunk' + console.error('Failed to delete chunk', e) + throw e + } finally { + deleting.value = false + } + } + + return { rechunking, saving, deleting, error, rechunk, updateChunkText, deleteChunk } }) diff --git a/frontend/src/features/chunking/ui/ChunkPanel.vue b/frontend/src/features/chunking/ui/ChunkPanel.vue index de70ad3..b55e2fb 100644 --- a/frontend/src/features/chunking/ui/ChunkPanel.vue +++ b/frontend/src/features/chunking/ui/ChunkPanel.vue @@ -84,7 +84,7 @@
- {{ pagination.totalItems.value }} {{ t('chunking.chunks') }} + {{ activeChunks.length }} {{ t('chunking.chunks') }}
p.{{ chunk.sourcePage }} + + {{ t('chunking.modified') }} + + +
{{ h }}
-
{{ chunk.text }}
+ + +
+