From a3486a8501e224057b9526257548b5f0d20b94bb Mon Sep 17 00:00:00 2001 From: Pier-Jean Malandrino Date: Tue, 31 Mar 2026 10:36:35 +0200 Subject: [PATCH] Add ServeConverter adapter for remote Docling Serve integration Implement the HTTP client adapter that delegates document conversion to a remote Docling Serve instance via its /v1/convert/file endpoint. Switchable via CONVERSION_ENGINE=remote env var. Includes health check, API key auth, response parsing, and 30 new tests covering parsing, type mapping, HTTP calls, and DI wiring. Co-Authored-By: Claude Opus 4.6 --- document-parser/infra/serve_converter.py | 229 +++++++++++++ document-parser/requirements.txt | 1 + document-parser/tests/test_serve_converter.py | 306 ++++++++++++++++++ 3 files changed, 536 insertions(+) create mode 100644 document-parser/infra/serve_converter.py create mode 100644 document-parser/tests/test_serve_converter.py diff --git a/document-parser/infra/serve_converter.py b/document-parser/infra/serve_converter.py new file mode 100644 index 0000000..42c1a5c --- /dev/null +++ b/document-parser/infra/serve_converter.py @@ -0,0 +1,229 @@ +"""Remote Docling Serve converter — delegates conversion via HTTP. + +This adapter implements the DocumentConverter port by calling a remote +Docling Serve instance's REST API. It supports both synchronous and +asynchronous conversion endpoints. +""" + +from __future__ import annotations + +import logging +import mimetypes +from pathlib import Path + +import httpx + +from domain.value_objects import ( + ConversionOptions, + ConversionResult, + PageDetail, + PageElement, +) + +logger = logging.getLogger(__name__) + +# Docling Serve API base path +_API_PREFIX = "/v1" + +# Default timeout for HTTP requests (seconds) +_DEFAULT_TIMEOUT = 600.0 + + +class ServeConverter: + """Adapter that delegates document conversion to a remote Docling Serve instance.""" + + def __init__( + self, + base_url: str, + api_key: str | None = None, + timeout: float = _DEFAULT_TIMEOUT, + ): + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._timeout = timeout + + def _headers(self) -> dict[str, str]: + headers: dict[str, str] = {} + if self._api_key: + headers["X-Api-Key"] = self._api_key + return headers + + def _build_conversion_options(self, options: ConversionOptions) -> dict: + """Map our ConversionOptions to Docling Serve's expected format.""" + opts: dict = { + "to_formats": ["md", "html"], + "do_ocr": options.do_ocr, + "do_table_structure": options.do_table_structure, + "table_mode": options.table_mode, + "do_code_enrichment": options.do_code_enrichment, + "do_formula_enrichment": options.do_formula_enrichment, + "do_picture_classification": options.do_picture_classification, + "do_picture_description": options.do_picture_description, + "images_scale": options.images_scale, + } + return opts + + async def convert( + self, file_path: str, options: ConversionOptions, + ) -> ConversionResult: + """Convert a document by uploading it to Docling Serve.""" + path = Path(file_path) + content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + + conversion_opts = self._build_conversion_options(options) + + url = f"{self._base_url}{_API_PREFIX}/convert/file" + + async with httpx.AsyncClient(timeout=self._timeout) as client: + with open(path, "rb") as f: + files = {"files": (path.name, f, content_type)} + data = {"options": _serialize_options(conversion_opts)} + + logger.info("Sending conversion request to %s", url) + response = await client.post( + url, + files=files, + data=data, + headers=self._headers(), + ) + + response.raise_for_status() + result_data = response.json() + + return _parse_response(result_data) + + async def health_check(self) -> bool: + """Check if Docling Serve is reachable.""" + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.get( + f"{self._base_url}/version", + headers=self._headers(), + ) + return resp.status_code == 200 + except httpx.HTTPError: + return False + + +def _serialize_options(opts: dict) -> str: + """Serialize conversion options to JSON string for multipart form.""" + import json + return json.dumps(opts) + + +def _parse_response(data: dict) -> ConversionResult: + """Parse Docling Serve JSON response into our domain ConversionResult. + + Docling Serve returns a DoclingDocument structure. The response format + contains document content and page-level information with bounding boxes. + """ + document = data.get("document", data) + + # Extract markdown and HTML content + content_md = "" + content_html = "" + + # Docling Serve may return content in different formats + if "md_content" in document: + content_md = document["md_content"] + elif "export_to_markdown" in document: + content_md = document["export_to_markdown"] + + if "html_content" in document: + content_html = document["html_content"] + elif "export_to_html" in document: + content_html = document["export_to_html"] + + # Parse pages + pages = _extract_pages(document) + page_count = len(pages) if pages else 1 + + return ConversionResult( + page_count=page_count, + content_markdown=content_md, + content_html=content_html, + pages=pages, + ) + + +def _extract_pages(document: dict) -> list[PageDetail]: + """Extract page details with elements from Docling Serve response.""" + pages_dict: dict[int, PageDetail] = {} + + # Extract page dimensions from pages metadata + raw_pages = document.get("pages", {}) + for page_key, page_data in raw_pages.items(): + page_no = int(page_key) + size = page_data.get("size", {}) + pages_dict[page_no] = PageDetail( + page_number=page_no, + width=size.get("width", 612.0), + height=size.get("height", 792.0), + ) + + # Extract elements from the document body + body = document.get("body", document.get("main_text", [])) + if isinstance(body, list): + for item in body: + _process_serve_item(item, pages_dict, document) + + return sorted(pages_dict.values(), key=lambda p: p.page_number) + + +def _process_serve_item( + item: dict, pages: dict[int, PageDetail], document: dict, +) -> None: + """Process a single item from Docling Serve response body.""" + prov_list = item.get("prov", []) + if not prov_list: + return + + item_type = _map_item_type(item) + content = item.get("text", "") + level = item.get("level", 0) + + for prov in prov_list: + page_no = prov.get("page_no", prov.get("page", 1)) + if page_no not in pages: + pages[page_no] = PageDetail( + page_number=page_no, width=612.0, height=792.0, + ) + + bbox_data = prov.get("bbox", {}) + if isinstance(bbox_data, dict): + bbox = [ + bbox_data.get("l", 0.0), + bbox_data.get("t", 0.0), + bbox_data.get("r", 0.0), + bbox_data.get("b", 0.0), + ] + elif isinstance(bbox_data, list) and len(bbox_data) == 4: + bbox = [float(v) for v in bbox_data] + else: + bbox = [0.0, 0.0, 0.0, 0.0] + + pages[page_no].elements.append( + PageElement(type=item_type, bbox=bbox, content=content, level=level) + ) + + +def _map_item_type(item: dict) -> str: + """Map Docling Serve item type to our element type string.""" + item_type = item.get("type", item.get("obj_type", "text")) + type_mapping = { + "table": "table", + "picture": "picture", + "figure": "picture", + "title": "title", + "section_header": "section_header", + "section-header": "section_header", + "list_item": "list", + "list": "list", + "formula": "formula", + "equation": "formula", + "code": "code", + "floating": "floating", + "text": "text", + "paragraph": "text", + } + return type_mapping.get(item_type.lower(), "text") if item_type else "text" diff --git a/document-parser/requirements.txt b/document-parser/requirements.txt index 46d50d0..4b8cc74 100644 --- a/document-parser/requirements.txt +++ b/document-parser/requirements.txt @@ -6,3 +6,4 @@ python-multipart>=0.0.12 pdf2image>=1.17.0,<2.0.0 pillow>=10.0.0,<11.0.0 aiosqlite>=0.20.0,<1.0.0 +httpx>=0.27.0,<1.0.0 diff --git a/document-parser/tests/test_serve_converter.py b/document-parser/tests/test_serve_converter.py new file mode 100644 index 0000000..c982c1a --- /dev/null +++ b/document-parser/tests/test_serve_converter.py @@ -0,0 +1,306 @@ +"""Tests for the ServeConverter adapter (Docling Serve HTTP client).""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, mock_open, patch + +import httpx +import pytest + +from domain.value_objects import ConversionOptions, ConversionResult, PageDetail, PageElement +from infra.serve_converter import ( + ServeConverter, + _extract_pages, + _map_item_type, + _parse_response, +) + + +# --------------------------------------------------------------------------- +# Unit tests — response parsing +# --------------------------------------------------------------------------- + +class TestParseResponse: + """Verify _parse_response correctly maps Docling Serve JSON to ConversionResult.""" + + def test_minimal_response(self): + data = { + "document": { + "md_content": "# Hello", + "html_content": "

Hello

", + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + } + } + + result = _parse_response(data) + + assert isinstance(result, ConversionResult) + assert result.content_markdown == "# Hello" + assert result.content_html == "

Hello

" + assert result.page_count == 1 + assert len(result.pages) == 1 + assert result.pages[0].width == 612.0 + assert result.pages[0].height == 792.0 + + def test_multi_page_response(self): + data = { + "document": { + "md_content": "text", + "html_content": "

text

", + "pages": { + "1": {"size": {"width": 612.0, "height": 792.0}}, + "2": {"size": {"width": 595.0, "height": 842.0}}, + }, + } + } + + result = _parse_response(data) + assert result.page_count == 2 + assert result.pages[0].page_number == 1 + assert result.pages[1].page_number == 2 + assert result.pages[1].width == 595.0 # A4 + + def test_response_with_body_elements(self): + data = { + "document": { + "md_content": "# Title\nSome text", + "html_content": "

Title

Some text

", + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "body": [ + { + "type": "title", + "text": "Title", + "level": 1, + "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40}}], + }, + { + "type": "text", + "text": "Some text", + "level": 0, + "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 50, "r": 200, "b": 70}}], + }, + ], + } + } + + result = _parse_response(data) + assert len(result.pages[0].elements) == 2 + assert result.pages[0].elements[0].type == "title" + assert result.pages[0].elements[0].content == "Title" + assert result.pages[0].elements[0].bbox == [10, 20, 200, 40] + assert result.pages[0].elements[1].type == "text" + + def test_empty_response(self): + data = {"document": {"pages": {}}} + result = _parse_response(data) + assert result.content_markdown == "" + assert result.content_html == "" + assert result.page_count == 1 # fallback minimum + + def test_bbox_as_list(self): + data = { + "document": { + "md_content": "", + "html_content": "", + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "body": [ + { + "type": "text", + "text": "hello", + "prov": [{"page_no": 1, "bbox": [10.0, 20.0, 200.0, 40.0]}], + }, + ], + } + } + result = _parse_response(data) + assert result.pages[0].elements[0].bbox == [10.0, 20.0, 200.0, 40.0] + + +# --------------------------------------------------------------------------- +# Unit tests — item type mapping +# --------------------------------------------------------------------------- + +class TestMapItemType: + @pytest.mark.parametrize("input_type,expected", [ + ("table", "table"), + ("picture", "picture"), + ("figure", "picture"), + ("title", "title"), + ("section_header", "section_header"), + ("section-header", "section_header"), + ("list_item", "list"), + ("formula", "formula"), + ("equation", "formula"), + ("code", "code"), + ("text", "text"), + ("paragraph", "text"), + ("unknown_type", "text"), + ]) + def test_type_mapping(self, input_type, expected): + assert _map_item_type({"type": input_type}) == expected + + def test_missing_type_defaults_to_text(self): + assert _map_item_type({}) == "text" + + +# --------------------------------------------------------------------------- +# Unit tests — ServeConverter +# --------------------------------------------------------------------------- + +class TestServeConverter: + def test_headers_with_api_key(self): + conv = ServeConverter(base_url="http://localhost:5001", api_key="secret") + assert conv._headers() == {"X-Api-Key": "secret"} + + def test_headers_without_api_key(self): + conv = ServeConverter(base_url="http://localhost:5001") + assert conv._headers() == {} + + def test_build_conversion_options(self): + conv = ServeConverter(base_url="http://localhost:5001") + opts = ConversionOptions(do_ocr=False, table_mode="fast", images_scale=2.0) + + result = conv._build_conversion_options(opts) + + assert result["do_ocr"] is False + assert result["table_mode"] == "fast" + assert result["images_scale"] == 2.0 + assert result["to_formats"] == ["md", "html"] + + def test_base_url_trailing_slash_stripped(self): + conv = ServeConverter(base_url="http://localhost:5001/") + assert conv._base_url == "http://localhost:5001" + + +# --------------------------------------------------------------------------- +# Integration tests — HTTP calls (mocked) +# --------------------------------------------------------------------------- + +class TestServeConverterConvert: + @pytest.mark.asyncio + async def test_successful_conversion(self, tmp_path): + # Create a temp file to "upload" + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"%PDF-1.4 fake content") + + serve_response = { + "document": { + "md_content": "# Converted", + "html_content": "

Converted

", + "pages": {"1": {"size": {"width": 612.0, "height": 792.0}}}, + "body": [ + { + "type": "title", + "text": "Converted", + "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40}}], + }, + ], + } + } + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = serve_response + mock_response.raise_for_status = MagicMock() + + mock_client = AsyncMock() + mock_client.post.return_value = mock_response + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + conv = ServeConverter(base_url="http://localhost:5001", api_key="test-key") + + with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client): + result = await conv.convert(str(test_file), ConversionOptions()) + + assert isinstance(result, ConversionResult) + assert result.content_markdown == "# Converted" + assert result.page_count == 1 + assert len(result.pages[0].elements) == 1 + + # Verify the HTTP call + mock_client.post.assert_called_once() + call_kwargs = mock_client.post.call_args + assert "/v1/convert/file" in call_kwargs[0][0] + + @pytest.mark.asyncio + async def test_http_error_raises(self, tmp_path): + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"%PDF-1.4 fake content") + + mock_response = MagicMock() + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "Server Error", request=MagicMock(), response=MagicMock(status_code=500), + ) + + mock_client = AsyncMock() + mock_client.post.return_value = mock_response + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + conv = ServeConverter(base_url="http://localhost:5001") + + with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client): + with pytest.raises(httpx.HTTPStatusError): + await conv.convert(str(test_file), ConversionOptions()) + + @pytest.mark.asyncio + async def test_health_check_success(self): + mock_response = MagicMock() + mock_response.status_code = 200 + + mock_client = AsyncMock() + mock_client.get.return_value = mock_response + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + conv = ServeConverter(base_url="http://localhost:5001") + + with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client): + assert await conv.health_check() is True + + @pytest.mark.asyncio + async def test_health_check_failure(self): + mock_client = AsyncMock() + mock_client.get.side_effect = httpx.ConnectError("Connection refused") + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + conv = ServeConverter(base_url="http://localhost:5001") + + with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client): + assert await conv.health_check() is False + + +# --------------------------------------------------------------------------- +# Integration — converter wiring in main.py +# --------------------------------------------------------------------------- + +class TestConverterWiring: + def test_local_engine_builds_local_converter(self): + from infra.local_converter import LocalConverter + from infra.settings import Settings + from main import _build_converter + + with patch("main.settings", Settings(conversion_engine="local")): + converter = _build_converter() + assert isinstance(converter, LocalConverter) + + def test_remote_engine_builds_serve_converter(self): + from infra.settings import Settings + from main import _build_converter + + with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001")): + converter = _build_converter() + assert isinstance(converter, ServeConverter) + assert converter._base_url == "http://serve:5001" + + def test_remote_engine_passes_api_key(self): + from infra.settings import Settings + from main import _build_converter + + with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001", docling_serve_api_key="my-key")): + converter = _build_converter() + assert isinstance(converter, ServeConverter) + assert converter._api_key == "my-key"