diff --git a/document-parser/api/analyses.py b/document-parser/api/analyses.py index 3a978fe..4fbf988 100644 --- a/document-parser/api/analyses.py +++ b/document-parser/api/analyses.py @@ -3,19 +3,22 @@ from __future__ import annotations import logging +from typing import Annotated -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request from api.schemas import AnalysisResponse, CreateAnalysisRequest +from services.analysis_service import AnalysisService logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/analyses", tags=["analyses"]) -def _get_service(): - """Lazy import to avoid circular dependency at module level.""" - from main import analysis_service - return analysis_service +def _get_service(request: Request) -> AnalysisService: + return request.app.state.analysis_service + + +ServiceDep = Annotated[AnalysisService, Depends(_get_service)] def _to_response(job) -> AnalysisResponse: @@ -35,7 +38,7 @@ def _to_response(job) -> AnalysisResponse: @router.post("", response_model=AnalysisResponse) -async def create_analysis(body: CreateAnalysisRequest): +async def create_analysis(body: CreateAnalysisRequest, service: ServiceDep): """Create a new analysis job for a document.""" if not body.documentId or not body.documentId.strip(): raise HTTPException(status_code=400, detail="documentId is required") @@ -45,7 +48,7 @@ async def create_analysis(body: CreateAnalysisRequest): pipeline_opts = body.pipelineOptions.model_dump() try: - job = await _get_service().create(body.documentId, pipeline_options=pipeline_opts) + job = await service.create(body.documentId, pipeline_options=pipeline_opts) except ValueError as e: raise HTTPException(status_code=404, detail=str(e)) from e @@ -53,24 +56,24 @@ async def create_analysis(body: CreateAnalysisRequest): @router.get("", response_model=list[AnalysisResponse]) -async def list_analyses(): +async def list_analyses(service: ServiceDep): """List all analysis jobs.""" - jobs = await _get_service().find_all() + jobs = await service.find_all() return [_to_response(j) for j in jobs] @router.get("/{job_id}", response_model=AnalysisResponse) -async def get_analysis(job_id: str): +async def get_analysis(job_id: str, service: ServiceDep): """Get a single analysis job.""" - job = await _get_service().find_by_id(job_id) + job = await service.find_by_id(job_id) if not job: raise HTTPException(status_code=404, detail="Analysis not found") return _to_response(job) @router.delete("/{job_id}", status_code=204) -async def delete_analysis(job_id: str): +async def delete_analysis(job_id: str, service: ServiceDep): """Delete an analysis job.""" - deleted = await _get_service().delete(job_id) + deleted = await service.delete(job_id) if not deleted: raise HTTPException(status_code=404, detail="Analysis not found") diff --git a/document-parser/domain/parsing.py b/document-parser/domain/parsing.py deleted file mode 100644 index 4a44e94..0000000 --- a/document-parser/domain/parsing.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Backward-compatible re-exports for domain.parsing. - -After the hexagonal architecture refactoring: -- Value objects moved to domain.value_objects -- Docling implementation moved to infra.local_converter - -This module re-exports the public names so existing code and tests -that import from domain.parsing continue to work. -""" - -from __future__ import annotations - -from domain.value_objects import ( # noqa: F401 - ConversionOptions, - ConversionResult, - PageDetail, - PageElement, -) -from infra.local_converter import ( - _build_docling_converter, - _convert_sync, - _extract_pages_detail as extract_pages_detail, # noqa: F401 - _get_default_converter as get_default_converter, # noqa: F401 -) - - -def build_converter(options: ConversionOptions | None = None): - """Build a Docling DocumentConverter (backward-compatible signature).""" - return _build_docling_converter(options or ConversionOptions()) - - -def convert_document(file_path: str, options: ConversionOptions | None = None) -> ConversionResult: - """Convert a document synchronously (backward-compatible signature).""" - return _convert_sync(file_path, options or ConversionOptions()) diff --git a/document-parser/infra/serve_converter.py b/document-parser/infra/serve_converter.py index 42c1a5c..34fc4ef 100644 --- a/document-parser/infra/serve_converter.py +++ b/document-parser/infra/serve_converter.py @@ -1,12 +1,18 @@ """Remote Docling Serve converter — delegates conversion via HTTP. This adapter implements the DocumentConverter port by calling a remote -Docling Serve instance's REST API. It supports both synchronous and -asynchronous conversion endpoints. +Docling Serve instance's REST API (v1). + +API contract based on docling-serve source code: +- Options are sent as individual multipart form fields (not a JSON blob) +- Response contains document.md_content, document.html_content, document.json_content +- json_content is a serialized DoclingDocument with texts[], tables[], pictures[] +- Bounding boxes use {l, t, r, b, coord_origin} format """ from __future__ import annotations +import json import logging import mimetypes from pathlib import Path @@ -22,12 +28,28 @@ from domain.value_objects import ( logger = logging.getLogger(__name__) -# Docling Serve API base path _API_PREFIX = "/v1" - -# Default timeout for HTTP requests (seconds) _DEFAULT_TIMEOUT = 600.0 +# Docling Serve label → our element type +_LABEL_MAP = { + "table": "table", + "picture": "picture", + "figure": "picture", + "title": "title", + "section_header": "section_header", + "list_item": "list", + "formula": "formula", + "code": "code", + "caption": "text", + "footnote": "text", + "page_header": "text", + "page_footer": "text", + "paragraph": "text", + "text": "text", + "reference": "text", +} + class ServeConverter: """Adapter that delegates document conversion to a remote Docling Serve instance.""" @@ -48,21 +70,6 @@ class ServeConverter: headers["X-Api-Key"] = self._api_key return headers - def _build_conversion_options(self, options: ConversionOptions) -> dict: - """Map our ConversionOptions to Docling Serve's expected format.""" - opts: dict = { - "to_formats": ["md", "html"], - "do_ocr": options.do_ocr, - "do_table_structure": options.do_table_structure, - "table_mode": options.table_mode, - "do_code_enrichment": options.do_code_enrichment, - "do_formula_enrichment": options.do_formula_enrichment, - "do_picture_classification": options.do_picture_classification, - "do_picture_description": options.do_picture_description, - "images_scale": options.images_scale, - } - return opts - async def convert( self, file_path: str, options: ConversionOptions, ) -> ConversionResult: @@ -70,25 +77,20 @@ class ServeConverter: path = Path(file_path) content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" - conversion_opts = self._build_conversion_options(options) - + form_data = _build_form_data(options) url = f"{self._base_url}{_API_PREFIX}/convert/file" async with httpx.AsyncClient(timeout=self._timeout) as client: with open(path, "rb") as f: - files = {"files": (path.name, f, content_type)} - data = {"options": _serialize_options(conversion_opts)} - - logger.info("Sending conversion request to %s", url) response = await client.post( url, - files=files, - data=data, + files={"files": (path.name, f, content_type)}, + data=form_data, headers=self._headers(), ) - response.raise_for_status() - result_data = response.json() + response.raise_for_status() + result_data = response.json() return _parse_response(result_data) @@ -102,40 +104,46 @@ class ServeConverter: ) return resp.status_code == 200 except httpx.HTTPError: + logger.warning("Docling Serve health check failed at %s", self._base_url, exc_info=True) return False -def _serialize_options(opts: dict) -> str: - """Serialize conversion options to JSON string for multipart form.""" - import json - return json.dumps(opts) +def _build_form_data(options: ConversionOptions) -> dict[str, str]: + """Build individual form fields matching Docling Serve's FormDepends pattern. + + Docling Serve uses FormDepends to flatten ConvertDocumentsRequestOptions + into individual form fields (not a JSON blob). + """ + return { + "to_formats": '["md","html","json"]', + "do_ocr": str(options.do_ocr).lower(), + "do_table_structure": str(options.do_table_structure).lower(), + "table_mode": options.table_mode, + "do_code_enrichment": str(options.do_code_enrichment).lower(), + "do_formula_enrichment": str(options.do_formula_enrichment).lower(), + "do_picture_classification": str(options.do_picture_classification).lower(), + "do_picture_description": str(options.do_picture_description).lower(), + "include_images": str(options.generate_picture_images).lower(), + "images_scale": str(options.images_scale), + } def _parse_response(data: dict) -> ConversionResult: - """Parse Docling Serve JSON response into our domain ConversionResult. + """Parse Docling Serve v1 ConvertDocumentResponse into our domain ConversionResult.""" + document = data.get("document", {}) - Docling Serve returns a DoclingDocument structure. The response format - contains document content and page-level information with bounding boxes. - """ - document = data.get("document", data) + content_md = document.get("md_content") or "" + content_html = document.get("html_content") or "" - # Extract markdown and HTML content - content_md = "" - content_html = "" + # json_content contains the full DoclingDocument with pages, elements, bboxes + json_content = document.get("json_content") + if isinstance(json_content, str): + json_content = json.loads(json_content) - # Docling Serve may return content in different formats - if "md_content" in document: - content_md = document["md_content"] - elif "export_to_markdown" in document: - content_md = document["export_to_markdown"] + pages: list[PageDetail] = [] + if json_content: + pages = _extract_pages_from_docling_document(json_content) - if "html_content" in document: - content_html = document["html_content"] - elif "export_to_html" in document: - content_html = document["export_to_html"] - - # Parse pages - pages = _extract_pages(document) page_count = len(pages) if pages else 1 return ConversionResult( @@ -146,13 +154,19 @@ def _parse_response(data: dict) -> ConversionResult: ) -def _extract_pages(document: dict) -> list[PageDetail]: - """Extract page details with elements from Docling Serve response.""" +def _extract_pages_from_docling_document(doc: dict) -> list[PageDetail]: + """Extract pages with elements from a serialized DoclingDocument. + + DoclingDocument structure: + - pages: {page_no: {size: {width, height}}} + - texts: [{label, text, prov: [{page_no, bbox: {l,t,r,b,coord_origin}}]}] + - tables: [{label, prov: [...], data: {...}}] + - pictures: [{label, prov: [...]}] + """ pages_dict: dict[int, PageDetail] = {} - # Extract page dimensions from pages metadata - raw_pages = document.get("pages", {}) - for page_key, page_data in raw_pages.items(): + # Build page dimensions + for page_key, page_data in doc.get("pages", {}).items(): page_no = int(page_key) size = page_data.get("size", {}) pages_dict[page_no] = PageDetail( @@ -161,69 +175,55 @@ def _extract_pages(document: dict) -> list[PageDetail]: height=size.get("height", 792.0), ) - # Extract elements from the document body - body = document.get("body", document.get("main_text", [])) - if isinstance(body, list): - for item in body: - _process_serve_item(item, pages_dict, document) + # Process all element arrays + for item in doc.get("texts", []): + _add_element(item, pages_dict) + + for item in doc.get("tables", []): + _add_element(item, pages_dict) + + for item in doc.get("pictures", []): + _add_element(item, pages_dict) return sorted(pages_dict.values(), key=lambda p: p.page_number) -def _process_serve_item( - item: dict, pages: dict[int, PageDetail], document: dict, -) -> None: - """Process a single item from Docling Serve response body.""" - prov_list = item.get("prov", []) - if not prov_list: - return +def _add_element(item: dict, pages: dict[int, PageDetail]) -> None: + """Add an element from a DoclingDocument array to the correct page.""" + label = item.get("label", "text") + element_type = _LABEL_MAP.get(label, "text") + content = item.get("text", "") or "" - item_type = _map_item_type(item) - content = item.get("text", "") - level = item.get("level", 0) - - for prov in prov_list: - page_no = prov.get("page_no", prov.get("page", 1)) + for prov in item.get("prov", []): + page_no = prov.get("page_no", 1) if page_no not in pages: pages[page_no] = PageDetail( page_number=page_no, width=612.0, height=792.0, ) bbox_data = prov.get("bbox", {}) - if isinstance(bbox_data, dict): - bbox = [ - bbox_data.get("l", 0.0), - bbox_data.get("t", 0.0), - bbox_data.get("r", 0.0), - bbox_data.get("b", 0.0), - ] - elif isinstance(bbox_data, list) and len(bbox_data) == 4: - bbox = [float(v) for v in bbox_data] - else: - bbox = [0.0, 0.0, 0.0, 0.0] + bbox = _extract_bbox(bbox_data, pages[page_no].height) pages[page_no].elements.append( - PageElement(type=item_type, bbox=bbox, content=content, level=level) + PageElement(type=element_type, bbox=bbox, content=content, level=0) ) -def _map_item_type(item: dict) -> str: - """Map Docling Serve item type to our element type string.""" - item_type = item.get("type", item.get("obj_type", "text")) - type_mapping = { - "table": "table", - "picture": "picture", - "figure": "picture", - "title": "title", - "section_header": "section_header", - "section-header": "section_header", - "list_item": "list", - "list": "list", - "formula": "formula", - "equation": "formula", - "code": "code", - "floating": "floating", - "text": "text", - "paragraph": "text", - } - return type_mapping.get(item_type.lower(), "text") if item_type else "text" +def _extract_bbox(bbox_data: dict, page_height: float) -> list[float]: + """Extract and normalize bbox to TOPLEFT [l, t, r, b] format.""" + if not isinstance(bbox_data, dict): + return [0.0, 0.0, 0.0, 0.0] + + l = bbox_data.get("l", 0.0) + t = bbox_data.get("t", 0.0) + r = bbox_data.get("r", 0.0) + b = bbox_data.get("b", 0.0) + coord_origin = bbox_data.get("coord_origin", "TOPLEFT") + + if coord_origin == "BOTTOMLEFT": + # Convert: top = page_height - old_top, bottom = page_height - old_bottom + new_t = page_height - b + new_b = page_height - t + t, b = new_t, new_b + + return [l, t, r, b] diff --git a/document-parser/main.py b/document-parser/main.py index fda1072..2a8e305 100644 --- a/document-parser/main.py +++ b/document-parser/main.py @@ -14,13 +14,14 @@ from __future__ import annotations import logging from contextlib import asynccontextmanager -from fastapi import FastAPI +from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from api.analyses import router as analyses_router from api.documents import router as documents_router from infra.settings import Settings from persistence.database import init_db +from services.analysis_service import AnalysisService logging.basicConfig( level=logging.INFO, @@ -50,8 +51,7 @@ def _build_converter(): return LocalConverter() -def _build_analysis_service(): - from services.analysis_service import AnalysisService +def _build_analysis_service() -> AnalysisService: converter = _build_converter() return AnalysisService( converter=converter, @@ -59,10 +59,6 @@ def _build_analysis_service(): ) -# Singleton service instance — imported by API routers -analysis_service = _build_analysis_service() - - # --------------------------------------------------------------------------- # FastAPI app # --------------------------------------------------------------------------- @@ -70,6 +66,7 @@ analysis_service = _build_analysis_service() @asynccontextmanager async def lifespan(app: FastAPI): await init_db() + app.state.analysis_service = _build_analysis_service() logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine) yield @@ -92,6 +89,11 @@ app.include_router(documents_router) app.include_router(analyses_router) +def get_analysis_service(request: Request) -> AnalysisService: + """FastAPI dependency — retrieve the AnalysisService from app.state.""" + return request.app.state.analysis_service + + @app.get("/health") def health(): """Health check endpoint.""" diff --git a/document-parser/tests/test_api_endpoints.py b/document-parser/tests/test_api_endpoints.py index d7970cb..173dd30 100644 --- a/document-parser/tests/test_api_endpoints.py +++ b/document-parser/tests/test_api_endpoints.py @@ -1,6 +1,6 @@ """Tests for FastAPI API endpoints using TestClient.""" -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient @@ -14,6 +14,16 @@ def client(): return TestClient(app, raise_server_exceptions=False) +@pytest.fixture +def mock_analysis_service(client): + """Inject a mock AnalysisService into app.state for the duration of the test.""" + mock_svc = MagicMock() + original = getattr(app.state, "analysis_service", None) + app.state.analysis_service = mock_svc + yield mock_svc + app.state.analysis_service = original + + class TestHealthEndpoint: def test_health(self, client): resp = client.get("/health") @@ -104,11 +114,10 @@ class TestDocumentEndpoints: class TestAnalysisEndpoints: - @patch("main.analysis_service.find_all", new_callable=AsyncMock) - def test_list_analyses(self, mock_find_all, client): - mock_find_all.return_value = [ + def test_list_analyses(self, client, mock_analysis_service): + mock_analysis_service.find_all = AsyncMock(return_value=[ AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf"), - ] + ]) resp = client.get("/api/analyses") assert resp.status_code == 200 @@ -119,11 +128,10 @@ class TestAnalysisEndpoints: assert data[0]["documentFilename"] == "test.pdf" assert data[0]["status"] == "PENDING" - @patch("main.analysis_service.find_by_id", new_callable=AsyncMock) - def test_get_analysis(self, mock_find, client): + def test_get_analysis(self, client, mock_analysis_service): job = AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf") job.mark_running() - mock_find.return_value = job + mock_analysis_service.find_by_id = AsyncMock(return_value=job) resp = client.get("/api/analyses/j1") assert resp.status_code == 200 @@ -131,31 +139,28 @@ class TestAnalysisEndpoints: assert data["status"] == "RUNNING" assert data["startedAt"] is not None - @patch("main.analysis_service.find_by_id", new_callable=AsyncMock) - def test_get_analysis_not_found(self, mock_find, client): - mock_find.return_value = None + def test_get_analysis_not_found(self, client, mock_analysis_service): + mock_analysis_service.find_by_id = AsyncMock(return_value=None) resp = client.get("/api/analyses/missing") assert resp.status_code == 404 - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_create_analysis(self, mock_create, client): - mock_create.return_value = AnalysisJob( + def test_create_analysis(self, client, mock_analysis_service): + mock_analysis_service.create = AsyncMock(return_value=AnalysisJob( id="j1", document_id="d1", document_filename="test.pdf", - ) + )) resp = client.post("/api/analyses", json={"documentId": "d1"}) assert resp.status_code == 200 data = resp.json() assert data["id"] == "j1" assert data["documentId"] == "d1" - mock_create.assert_called_once_with("d1", pipeline_options=None) + mock_analysis_service.create.assert_called_once_with("d1", pipeline_options=None) - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_create_analysis_with_pipeline_options(self, mock_create, client): - mock_create.return_value = AnalysisJob( + def test_create_analysis_with_pipeline_options(self, client, mock_analysis_service): + mock_analysis_service.create = AsyncMock(return_value=AnalysisJob( id="j2", document_id="d1", document_filename="test.pdf", - ) + )) resp = client.post("/api/analyses", json={ "documentId": "d1", @@ -176,7 +181,7 @@ class TestAnalysisEndpoints: data = resp.json() assert data["id"] == "j2" - call_kwargs = mock_create.call_args + call_kwargs = mock_analysis_service.create.call_args opts = call_kwargs.kwargs["pipeline_options"] assert opts["do_ocr"] is False assert opts["table_mode"] == "fast" @@ -184,12 +189,11 @@ class TestAnalysisEndpoints: assert opts["generate_picture_images"] is True assert opts["images_scale"] == 2.0 - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_create_analysis_with_partial_pipeline_options(self, mock_create, client): + def test_create_analysis_with_partial_pipeline_options(self, client, mock_analysis_service): """Pipeline options should use defaults for unspecified fields.""" - mock_create.return_value = AnalysisJob( + mock_analysis_service.create = AsyncMock(return_value=AnalysisJob( id="j3", document_id="d1", document_filename="test.pdf", - ) + )) resp = client.post("/api/analyses", json={ "documentId": "d1", @@ -197,38 +201,35 @@ class TestAnalysisEndpoints: }) assert resp.status_code == 200 - opts = mock_create.call_args.kwargs["pipeline_options"] + opts = mock_analysis_service.create.call_args.kwargs["pipeline_options"] assert opts["do_ocr"] is False # Defaults assert opts["do_table_structure"] is True assert opts["table_mode"] == "accurate" assert opts["do_code_enrichment"] is False - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_create_analysis_document_not_found(self, mock_create, client): - mock_create.side_effect = ValueError("Document not found: d99") + def test_create_analysis_document_not_found(self, client, mock_analysis_service): + mock_analysis_service.create = AsyncMock(side_effect=ValueError("Document not found: d99")) resp = client.post("/api/analyses", json={"documentId": "d99"}) assert resp.status_code == 404 - def test_create_analysis_empty_document_id(self, client): + def test_create_analysis_empty_document_id(self, client, mock_analysis_service): resp = client.post("/api/analyses", json={"documentId": ""}) assert resp.status_code == 400 - def test_create_analysis_whitespace_document_id(self, client): + def test_create_analysis_whitespace_document_id(self, client, mock_analysis_service): resp = client.post("/api/analyses", json={"documentId": " "}) assert resp.status_code == 400 - @patch("main.analysis_service.delete", new_callable=AsyncMock) - def test_delete_analysis(self, mock_delete, client): - mock_delete.return_value = True + def test_delete_analysis(self, client, mock_analysis_service): + mock_analysis_service.delete = AsyncMock(return_value=True) resp = client.delete("/api/analyses/j1") assert resp.status_code == 204 - @patch("main.analysis_service.delete", new_callable=AsyncMock) - def test_delete_analysis_not_found(self, mock_delete, client): - mock_delete.return_value = False + def test_delete_analysis_not_found(self, client, mock_analysis_service): + mock_analysis_service.delete = AsyncMock(return_value=False) resp = client.delete("/api/analyses/missing") assert resp.status_code == 404 diff --git a/document-parser/tests/test_pipeline_options.py b/document-parser/tests/test_pipeline_options.py index 41509c5..39ec84b 100644 --- a/document-parser/tests/test_pipeline_options.py +++ b/document-parser/tests/test_pipeline_options.py @@ -11,10 +11,10 @@ from docling.datamodel.pipeline_options import ( TableFormerMode, ) -from domain.parsing import ( - ConversionOptions, - build_converter, - convert_document, +from domain.value_objects import ConversionOptions +from infra.local_converter import ( + _build_docling_converter as build_converter, + _convert_sync as convert_document, ) # --------------------------------------------------------------------------- @@ -30,7 +30,7 @@ class TestBuildConverter: return fmt_opt.pipeline_options def test_defaults(self): - conv = build_converter() + conv = build_converter(ConversionOptions()) opts = self._get_pipeline_options(conv) assert opts.do_ocr is True assert opts.do_table_structure is True @@ -143,7 +143,7 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_get_default.return_value = mock_conv - convert_document("/tmp/test.pdf") + convert_document("/tmp/test.pdf", ConversionOptions()) mock_get_default.assert_called_once() mock_build.assert_not_called() @@ -457,30 +457,37 @@ class TestAnalysisEndpointPipelineOptions: @pytest.fixture def client(self): from fastapi.testclient import TestClient - from main import app return TestClient(app, raise_server_exceptions=False) - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_no_pipeline_options_sends_none(self, mock_create, client): + @pytest.fixture + def mock_svc(self, client): + from main import app + from unittest.mock import MagicMock + mock = MagicMock() + original = getattr(app.state, "analysis_service", None) + app.state.analysis_service = mock + yield mock + app.state.analysis_service = original + + def test_no_pipeline_options_sends_none(self, client, mock_svc): from domain.models import AnalysisJob - mock_create.return_value = AnalysisJob(id="j1", document_id="d1") + mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1")) client.post("/api/analyses", json={"documentId": "d1"}) - mock_create.assert_called_once_with("d1", pipeline_options=None) + mock_svc.create.assert_called_once_with("d1", pipeline_options=None) - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_empty_pipeline_options_object_uses_defaults(self, mock_create, client): + def test_empty_pipeline_options_object_uses_defaults(self, client, mock_svc): from domain.models import AnalysisJob - mock_create.return_value = AnalysisJob(id="j1", document_id="d1") + mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1")) client.post("/api/analyses", json={ "documentId": "d1", "pipelineOptions": {}, }) - opts = mock_create.call_args.kwargs["pipeline_options"] + opts = mock_svc.create.call_args.kwargs["pipeline_options"] assert opts["do_ocr"] is True assert opts["do_table_structure"] is True assert opts["table_mode"] == "accurate" @@ -488,20 +495,18 @@ class TestAnalysisEndpointPipelineOptions: assert opts["do_formula_enrichment"] is False assert opts["images_scale"] == 1.0 - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_partial_pipeline_options_merges_with_defaults(self, mock_create, client): + def test_partial_pipeline_options_merges_with_defaults(self, client, mock_svc): from domain.models import AnalysisJob - mock_create.return_value = AnalysisJob(id="j1", document_id="d1") + mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1")) client.post("/api/analyses", json={ "documentId": "d1", "pipelineOptions": {"do_ocr": False, "images_scale": 1.5}, }) - opts = mock_create.call_args.kwargs["pipeline_options"] + opts = mock_svc.create.call_args.kwargs["pipeline_options"] assert opts["do_ocr"] is False assert opts["images_scale"] == 1.5 - # All other fields should have defaults assert opts["do_table_structure"] is True assert opts["table_mode"] == "accurate" assert opts["do_code_enrichment"] is False @@ -511,10 +516,9 @@ class TestAnalysisEndpointPipelineOptions: assert opts["generate_picture_images"] is False assert opts["generate_page_images"] is False - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_full_pipeline_options(self, mock_create, client): + def test_full_pipeline_options(self, client, mock_svc): from domain.models import AnalysisJob - mock_create.return_value = AnalysisJob(id="j1", document_id="d1") + mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1")) payload = { "documentId": "d1", @@ -535,25 +539,22 @@ class TestAnalysisEndpointPipelineOptions: resp = client.post("/api/analyses", json=payload) assert resp.status_code == 200 - opts = mock_create.call_args.kwargs["pipeline_options"] + opts = mock_svc.create.call_args.kwargs["pipeline_options"] assert opts == payload["pipelineOptions"] - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_invalid_pipeline_option_type_rejected(self, mock_create, client): + def test_invalid_pipeline_option_type_rejected(self, client, mock_svc): resp = client.post("/api/analyses", json={ "documentId": "d1", "pipelineOptions": {"do_ocr": "not-a-bool"}, }) assert resp.status_code == 422 - @patch("main.analysis_service.create", new_callable=AsyncMock) - def test_unknown_pipeline_option_ignored(self, mock_create, client): + def test_unknown_pipeline_option_ignored(self, client, mock_svc): from domain.models import AnalysisJob - mock_create.return_value = AnalysisJob(id="j1", document_id="d1") + mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1")) resp = client.post("/api/analyses", json={ "documentId": "d1", "pipelineOptions": {"do_ocr": True, "unknown_field": True}, }) - # Pydantic ignores extra fields by default assert resp.status_code == 200 diff --git a/document-parser/tests/test_serve_converter.py b/document-parser/tests/test_serve_converter.py index c982c1a..31f73c0 100644 --- a/document-parser/tests/test_serve_converter.py +++ b/document-parser/tests/test_serve_converter.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from unittest.mock import AsyncMock, MagicMock, mock_open, patch +from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest @@ -11,79 +11,92 @@ import pytest from domain.value_objects import ConversionOptions, ConversionResult, PageDetail, PageElement from infra.serve_converter import ( ServeConverter, - _extract_pages, - _map_item_type, + _build_form_data, + _extract_bbox, + _extract_pages_from_docling_document, _parse_response, ) +# --------------------------------------------------------------------------- +# Unit tests — form data building +# --------------------------------------------------------------------------- + +class TestBuildFormData: + def test_default_options(self): + data = _build_form_data(ConversionOptions()) + assert data["do_ocr"] == "true" + assert data["do_table_structure"] == "true" + assert data["table_mode"] == "accurate" + assert data["do_code_enrichment"] == "false" + assert data["do_formula_enrichment"] == "false" + assert data["do_picture_classification"] == "false" + assert data["do_picture_description"] == "false" + assert data["include_images"] == "false" + assert data["images_scale"] == "1.0" + assert '"json"' in data["to_formats"] + + def test_custom_options(self): + opts = ConversionOptions( + do_ocr=False, table_mode="fast", images_scale=2.0, + generate_picture_images=True, + ) + data = _build_form_data(opts) + assert data["do_ocr"] == "false" + assert data["table_mode"] == "fast" + assert data["images_scale"] == "2.0" + assert data["include_images"] == "true" + + # --------------------------------------------------------------------------- # Unit tests — response parsing # --------------------------------------------------------------------------- class TestParseResponse: - """Verify _parse_response correctly maps Docling Serve JSON to ConversionResult.""" - def test_minimal_response(self): data = { "document": { "md_content": "# Hello", "html_content": "

Hello

", - "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "json_content": { + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "texts": [], + "tables": [], + "pictures": [], + }, } } - result = _parse_response(data) - assert isinstance(result, ConversionResult) assert result.content_markdown == "# Hello" assert result.content_html == "

Hello

" assert result.page_count == 1 - assert len(result.pages) == 1 assert result.pages[0].width == 612.0 - assert result.pages[0].height == 792.0 - def test_multi_page_response(self): + def test_response_with_elements(self): data = { "document": { - "md_content": "text", - "html_content": "

text

", - "pages": { - "1": {"size": {"width": 612.0, "height": 792.0}}, - "2": {"size": {"width": 595.0, "height": 842.0}}, + "md_content": "# Title\nText", + "html_content": "

Title

Text

", + "json_content": { + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "texts": [ + { + "label": "title", + "text": "Title", + "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40, "coord_origin": "TOPLEFT"}}], + }, + { + "label": "paragraph", + "text": "Text", + "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 50, "r": 200, "b": 70, "coord_origin": "TOPLEFT"}}], + }, + ], + "tables": [], + "pictures": [], }, } } - - result = _parse_response(data) - assert result.page_count == 2 - assert result.pages[0].page_number == 1 - assert result.pages[1].page_number == 2 - assert result.pages[1].width == 595.0 # A4 - - def test_response_with_body_elements(self): - data = { - "document": { - "md_content": "# Title\nSome text", - "html_content": "

Title

Some text

", - "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, - "body": [ - { - "type": "title", - "text": "Title", - "level": 1, - "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40}}], - }, - { - "type": "text", - "text": "Some text", - "level": 0, - "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 50, "r": 200, "b": 70}}], - }, - ], - } - } - result = _parse_response(data) assert len(result.pages[0].elements) == 2 assert result.pages[0].elements[0].type == "title" @@ -91,57 +104,121 @@ class TestParseResponse: assert result.pages[0].elements[0].bbox == [10, 20, 200, 40] assert result.pages[0].elements[1].type == "text" - def test_empty_response(self): - data = {"document": {"pages": {}}} - result = _parse_response(data) - assert result.content_markdown == "" - assert result.content_html == "" - assert result.page_count == 1 # fallback minimum - - def test_bbox_as_list(self): + def test_multi_page(self): data = { "document": { "md_content": "", "html_content": "", - "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, - "body": [ - { - "type": "text", - "text": "hello", - "prov": [{"page_no": 1, "bbox": [10.0, 20.0, 200.0, 40.0]}], + "json_content": { + "pages": { + "1": {"size": {"width": 612.0, "height": 792.0}}, + "2": {"size": {"width": 595.0, "height": 842.0}}, }, - ], + "texts": [], "tables": [], "pictures": [], + }, } } result = _parse_response(data) - assert result.pages[0].elements[0].bbox == [10.0, 20.0, 200.0, 40.0] + assert result.page_count == 2 + assert result.pages[1].width == 595.0 + + def test_no_json_content(self): + data = { + "document": { + "md_content": "text", + "html_content": "

text

", + } + } + result = _parse_response(data) + assert result.content_markdown == "text" + assert result.pages == [] + assert result.page_count == 1 + + def test_json_content_as_string(self): + json_doc = { + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "texts": [], "tables": [], "pictures": [], + } + data = { + "document": { + "md_content": "", + "html_content": "", + "json_content": json.dumps(json_doc), + } + } + result = _parse_response(data) + assert result.page_count == 1 + + def test_tables_and_pictures(self): + data = { + "document": { + "md_content": "", + "html_content": "", + "json_content": { + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "texts": [], + "tables": [ + {"label": "table", "text": "", "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 10, "r": 300, "b": 200, "coord_origin": "TOPLEFT"}}]}, + ], + "pictures": [ + {"label": "picture", "text": "", "prov": [{"page_no": 1, "bbox": {"l": 50, "t": 300, "r": 250, "b": 500, "coord_origin": "TOPLEFT"}}]}, + ], + }, + } + } + result = _parse_response(data) + types = [e.type for e in result.pages[0].elements] + assert "table" in types + assert "picture" in types # --------------------------------------------------------------------------- -# Unit tests — item type mapping +# Unit tests — bbox extraction # --------------------------------------------------------------------------- -class TestMapItemType: - @pytest.mark.parametrize("input_type,expected", [ - ("table", "table"), - ("picture", "picture"), - ("figure", "picture"), - ("title", "title"), - ("section_header", "section_header"), - ("section-header", "section_header"), - ("list_item", "list"), - ("formula", "formula"), - ("equation", "formula"), - ("code", "code"), - ("text", "text"), - ("paragraph", "text"), - ("unknown_type", "text"), - ]) - def test_type_mapping(self, input_type, expected): - assert _map_item_type({"type": input_type}) == expected +class TestExtractBbox: + def test_topleft_passthrough(self): + bbox = _extract_bbox({"l": 10, "t": 20, "r": 100, "b": 50, "coord_origin": "TOPLEFT"}, 792.0) + assert bbox == [10, 20, 100, 50] - def test_missing_type_defaults_to_text(self): - assert _map_item_type({}) == "text" + def test_bottomleft_conversion(self): + bbox = _extract_bbox({"l": 10, "t": 742, "r": 100, "b": 772, "coord_origin": "BOTTOMLEFT"}, 792.0) + # new_t = 792 - 772 = 20, new_b = 792 - 742 = 50 + assert bbox == [10, 20, 100, 50] + + def test_missing_coord_origin_defaults_topleft(self): + bbox = _extract_bbox({"l": 10, "t": 20, "r": 100, "b": 50}, 792.0) + assert bbox == [10, 20, 100, 50] + + def test_empty_dict(self): + bbox = _extract_bbox({}, 792.0) + assert bbox == [0.0, 0.0, 0.0, 0.0] + + def test_non_dict_returns_zeros(self): + bbox = _extract_bbox("invalid", 792.0) + assert bbox == [0.0, 0.0, 0.0, 0.0] + + +# --------------------------------------------------------------------------- +# Unit tests — label mapping +# --------------------------------------------------------------------------- + +class TestLabelMapping: + def test_known_labels(self): + from infra.serve_converter import _LABEL_MAP + assert _LABEL_MAP["table"] == "table" + assert _LABEL_MAP["picture"] == "picture" + assert _LABEL_MAP["figure"] == "picture" + assert _LABEL_MAP["title"] == "title" + assert _LABEL_MAP["section_header"] == "section_header" + assert _LABEL_MAP["list_item"] == "list" + assert _LABEL_MAP["formula"] == "formula" + assert _LABEL_MAP["code"] == "code" + assert _LABEL_MAP["paragraph"] == "text" + + def test_unknown_label_defaults_to_text(self): + from infra.serve_converter import _LABEL_MAP + assert _LABEL_MAP.get("unknown_thing", "text") == "text" # --------------------------------------------------------------------------- @@ -157,17 +234,6 @@ class TestServeConverter: conv = ServeConverter(base_url="http://localhost:5001") assert conv._headers() == {} - def test_build_conversion_options(self): - conv = ServeConverter(base_url="http://localhost:5001") - opts = ConversionOptions(do_ocr=False, table_mode="fast", images_scale=2.0) - - result = conv._build_conversion_options(opts) - - assert result["do_ocr"] is False - assert result["table_mode"] == "fast" - assert result["images_scale"] == 2.0 - assert result["to_formats"] == ["md", "html"] - def test_base_url_trailing_slash_stripped(self): conv = ServeConverter(base_url="http://localhost:5001/") assert conv._base_url == "http://localhost:5001" @@ -180,7 +246,6 @@ class TestServeConverter: class TestServeConverterConvert: @pytest.mark.asyncio async def test_successful_conversion(self, tmp_path): - # Create a temp file to "upload" test_file = tmp_path / "test.pdf" test_file.write_bytes(b"%PDF-1.4 fake content") @@ -188,14 +253,14 @@ class TestServeConverterConvert: "document": { "md_content": "# Converted", "html_content": "

Converted

", - "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, - "body": [ - { - "type": "title", - "text": "Converted", - "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40}}], - }, - ], + "json_content": { + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "texts": [ + {"label": "title", "text": "Converted", "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40, "coord_origin": "TOPLEFT"}}]}, + ], + "tables": [], + "pictures": [], + }, } } @@ -218,11 +283,13 @@ class TestServeConverterConvert: assert result.content_markdown == "# Converted" assert result.page_count == 1 assert len(result.pages[0].elements) == 1 + assert result.pages[0].elements[0].type == "title" - # Verify the HTTP call - mock_client.post.assert_called_once() + # Verify form fields sent individually (not as JSON blob) call_kwargs = mock_client.post.call_args - assert "/v1/convert/file" in call_kwargs[0][0] + sent_data = call_kwargs.kwargs.get("data", {}) + assert "do_ocr" in sent_data + assert sent_data["do_ocr"] == "true" @pytest.mark.asyncio async def test_http_error_raises(self, tmp_path): @@ -281,26 +348,26 @@ class TestConverterWiring: def test_local_engine_builds_local_converter(self): from infra.local_converter import LocalConverter from infra.settings import Settings - from main import _build_converter with patch("main.settings", Settings(conversion_engine="local")): + from main import _build_converter converter = _build_converter() assert isinstance(converter, LocalConverter) def test_remote_engine_builds_serve_converter(self): from infra.settings import Settings - from main import _build_converter with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001")): + from main import _build_converter converter = _build_converter() assert isinstance(converter, ServeConverter) assert converter._base_url == "http://serve:5001" def test_remote_engine_passes_api_key(self): from infra.settings import Settings - from main import _build_converter with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001", docling_serve_api_key="my-key")): + from main import _build_converter converter = _build_converter() assert isinstance(converter, ServeConverter) assert converter._api_key == "my-key" diff --git a/frontend/src/features/document/store.ts b/frontend/src/features/document/store.ts index 84e1369..cff9f48 100644 --- a/frontend/src/features/document/store.ts +++ b/frontend/src/features/document/store.ts @@ -3,6 +3,8 @@ import { ref } from 'vue' import type { Document } from '../../shared/types' import * as api from './api' +const MAX_FILE_SIZE = 50 * 1024 * 1024 // 50 MB + export const useDocumentStore = defineStore('document', () => { const documents = ref([]) const selectedId = ref(null) @@ -24,6 +26,10 @@ export const useDocumentStore = defineStore('document', () => { } async function upload(file: File): Promise { + if (file.size > MAX_FILE_SIZE) { + error.value = 'File too large (max 50 MB)' + throw new Error(error.value) + } uploading.value = true error.value = null try {