Add ServeConverter adapter for remote Docling Serve integration
Implement the HTTP client adapter that delegates document conversion to a remote Docling Serve instance via its /v1/convert/file endpoint. Switchable via CONVERSION_ENGINE=remote env var. Includes health check, API key auth, response parsing, and 30 new tests covering parsing, type mapping, HTTP calls, and DI wiring. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Pier-Jean Malandrino
parent
3743ed4ca8
commit
a3486a8501
3 changed files with 536 additions and 0 deletions
229
document-parser/infra/serve_converter.py
Normal file
229
document-parser/infra/serve_converter.py
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
"""Remote Docling Serve converter — delegates conversion via HTTP.
|
||||
|
||||
This adapter implements the DocumentConverter port by calling a remote
|
||||
Docling Serve instance's REST API. It supports both synchronous and
|
||||
asynchronous conversion endpoints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from domain.value_objects import (
|
||||
ConversionOptions,
|
||||
ConversionResult,
|
||||
PageDetail,
|
||||
PageElement,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Docling Serve API base path
|
||||
_API_PREFIX = "/v1"
|
||||
|
||||
# Default timeout for HTTP requests (seconds)
|
||||
_DEFAULT_TIMEOUT = 600.0
|
||||
|
||||
|
||||
class ServeConverter:
|
||||
"""Adapter that delegates document conversion to a remote Docling Serve instance."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str | None = None,
|
||||
timeout: float = _DEFAULT_TIMEOUT,
|
||||
):
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._api_key = api_key
|
||||
self._timeout = timeout
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
headers: dict[str, str] = {}
|
||||
if self._api_key:
|
||||
headers["X-Api-Key"] = self._api_key
|
||||
return headers
|
||||
|
||||
def _build_conversion_options(self, options: ConversionOptions) -> dict:
|
||||
"""Map our ConversionOptions to Docling Serve's expected format."""
|
||||
opts: dict = {
|
||||
"to_formats": ["md", "html"],
|
||||
"do_ocr": options.do_ocr,
|
||||
"do_table_structure": options.do_table_structure,
|
||||
"table_mode": options.table_mode,
|
||||
"do_code_enrichment": options.do_code_enrichment,
|
||||
"do_formula_enrichment": options.do_formula_enrichment,
|
||||
"do_picture_classification": options.do_picture_classification,
|
||||
"do_picture_description": options.do_picture_description,
|
||||
"images_scale": options.images_scale,
|
||||
}
|
||||
return opts
|
||||
|
||||
async def convert(
|
||||
self, file_path: str, options: ConversionOptions,
|
||||
) -> ConversionResult:
|
||||
"""Convert a document by uploading it to Docling Serve."""
|
||||
path = Path(file_path)
|
||||
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||||
|
||||
conversion_opts = self._build_conversion_options(options)
|
||||
|
||||
url = f"{self._base_url}{_API_PREFIX}/convert/file"
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
with open(path, "rb") as f:
|
||||
files = {"files": (path.name, f, content_type)}
|
||||
data = {"options": _serialize_options(conversion_opts)}
|
||||
|
||||
logger.info("Sending conversion request to %s", url)
|
||||
response = await client.post(
|
||||
url,
|
||||
files=files,
|
||||
data=data,
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result_data = response.json()
|
||||
|
||||
return _parse_response(result_data)
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
"""Check if Docling Serve is reachable."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
resp = await client.get(
|
||||
f"{self._base_url}/version",
|
||||
headers=self._headers(),
|
||||
)
|
||||
return resp.status_code == 200
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
|
||||
def _serialize_options(opts: dict) -> str:
|
||||
"""Serialize conversion options to JSON string for multipart form."""
|
||||
import json
|
||||
return json.dumps(opts)
|
||||
|
||||
|
||||
def _parse_response(data: dict) -> ConversionResult:
|
||||
"""Parse Docling Serve JSON response into our domain ConversionResult.
|
||||
|
||||
Docling Serve returns a DoclingDocument structure. The response format
|
||||
contains document content and page-level information with bounding boxes.
|
||||
"""
|
||||
document = data.get("document", data)
|
||||
|
||||
# Extract markdown and HTML content
|
||||
content_md = ""
|
||||
content_html = ""
|
||||
|
||||
# Docling Serve may return content in different formats
|
||||
if "md_content" in document:
|
||||
content_md = document["md_content"]
|
||||
elif "export_to_markdown" in document:
|
||||
content_md = document["export_to_markdown"]
|
||||
|
||||
if "html_content" in document:
|
||||
content_html = document["html_content"]
|
||||
elif "export_to_html" in document:
|
||||
content_html = document["export_to_html"]
|
||||
|
||||
# Parse pages
|
||||
pages = _extract_pages(document)
|
||||
page_count = len(pages) if pages else 1
|
||||
|
||||
return ConversionResult(
|
||||
page_count=page_count,
|
||||
content_markdown=content_md,
|
||||
content_html=content_html,
|
||||
pages=pages,
|
||||
)
|
||||
|
||||
|
||||
def _extract_pages(document: dict) -> list[PageDetail]:
|
||||
"""Extract page details with elements from Docling Serve response."""
|
||||
pages_dict: dict[int, PageDetail] = {}
|
||||
|
||||
# Extract page dimensions from pages metadata
|
||||
raw_pages = document.get("pages", {})
|
||||
for page_key, page_data in raw_pages.items():
|
||||
page_no = int(page_key)
|
||||
size = page_data.get("size", {})
|
||||
pages_dict[page_no] = PageDetail(
|
||||
page_number=page_no,
|
||||
width=size.get("width", 612.0),
|
||||
height=size.get("height", 792.0),
|
||||
)
|
||||
|
||||
# Extract elements from the document body
|
||||
body = document.get("body", document.get("main_text", []))
|
||||
if isinstance(body, list):
|
||||
for item in body:
|
||||
_process_serve_item(item, pages_dict, document)
|
||||
|
||||
return sorted(pages_dict.values(), key=lambda p: p.page_number)
|
||||
|
||||
|
||||
def _process_serve_item(
|
||||
item: dict, pages: dict[int, PageDetail], document: dict,
|
||||
) -> None:
|
||||
"""Process a single item from Docling Serve response body."""
|
||||
prov_list = item.get("prov", [])
|
||||
if not prov_list:
|
||||
return
|
||||
|
||||
item_type = _map_item_type(item)
|
||||
content = item.get("text", "")
|
||||
level = item.get("level", 0)
|
||||
|
||||
for prov in prov_list:
|
||||
page_no = prov.get("page_no", prov.get("page", 1))
|
||||
if page_no not in pages:
|
||||
pages[page_no] = PageDetail(
|
||||
page_number=page_no, width=612.0, height=792.0,
|
||||
)
|
||||
|
||||
bbox_data = prov.get("bbox", {})
|
||||
if isinstance(bbox_data, dict):
|
||||
bbox = [
|
||||
bbox_data.get("l", 0.0),
|
||||
bbox_data.get("t", 0.0),
|
||||
bbox_data.get("r", 0.0),
|
||||
bbox_data.get("b", 0.0),
|
||||
]
|
||||
elif isinstance(bbox_data, list) and len(bbox_data) == 4:
|
||||
bbox = [float(v) for v in bbox_data]
|
||||
else:
|
||||
bbox = [0.0, 0.0, 0.0, 0.0]
|
||||
|
||||
pages[page_no].elements.append(
|
||||
PageElement(type=item_type, bbox=bbox, content=content, level=level)
|
||||
)
|
||||
|
||||
|
||||
def _map_item_type(item: dict) -> str:
|
||||
"""Map Docling Serve item type to our element type string."""
|
||||
item_type = item.get("type", item.get("obj_type", "text"))
|
||||
type_mapping = {
|
||||
"table": "table",
|
||||
"picture": "picture",
|
||||
"figure": "picture",
|
||||
"title": "title",
|
||||
"section_header": "section_header",
|
||||
"section-header": "section_header",
|
||||
"list_item": "list",
|
||||
"list": "list",
|
||||
"formula": "formula",
|
||||
"equation": "formula",
|
||||
"code": "code",
|
||||
"floating": "floating",
|
||||
"text": "text",
|
||||
"paragraph": "text",
|
||||
}
|
||||
return type_mapping.get(item_type.lower(), "text") if item_type else "text"
|
||||
|
|
@ -6,3 +6,4 @@ python-multipart>=0.0.12
|
|||
pdf2image>=1.17.0,<2.0.0
|
||||
pillow>=10.0.0,<11.0.0
|
||||
aiosqlite>=0.20.0,<1.0.0
|
||||
httpx>=0.27.0,<1.0.0
|
||||
|
|
|
|||
306
document-parser/tests/test_serve_converter.py
Normal file
306
document-parser/tests/test_serve_converter.py
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
"""Tests for the ServeConverter adapter (Docling Serve HTTP client)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from domain.value_objects import ConversionOptions, ConversionResult, PageDetail, PageElement
|
||||
from infra.serve_converter import (
|
||||
ServeConverter,
|
||||
_extract_pages,
|
||||
_map_item_type,
|
||||
_parse_response,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests — response parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestParseResponse:
|
||||
"""Verify _parse_response correctly maps Docling Serve JSON to ConversionResult."""
|
||||
|
||||
def test_minimal_response(self):
|
||||
data = {
|
||||
"document": {
|
||||
"md_content": "# Hello",
|
||||
"html_content": "<h1>Hello</h1>",
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
}
|
||||
}
|
||||
|
||||
result = _parse_response(data)
|
||||
|
||||
assert isinstance(result, ConversionResult)
|
||||
assert result.content_markdown == "# Hello"
|
||||
assert result.content_html == "<h1>Hello</h1>"
|
||||
assert result.page_count == 1
|
||||
assert len(result.pages) == 1
|
||||
assert result.pages[0].width == 612.0
|
||||
assert result.pages[0].height == 792.0
|
||||
|
||||
def test_multi_page_response(self):
|
||||
data = {
|
||||
"document": {
|
||||
"md_content": "text",
|
||||
"html_content": "<p>text</p>",
|
||||
"pages": {
|
||||
"1": {"size": {"width": 612.0, "height": 792.0}},
|
||||
"2": {"size": {"width": 595.0, "height": 842.0}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
result = _parse_response(data)
|
||||
assert result.page_count == 2
|
||||
assert result.pages[0].page_number == 1
|
||||
assert result.pages[1].page_number == 2
|
||||
assert result.pages[1].width == 595.0 # A4
|
||||
|
||||
def test_response_with_body_elements(self):
|
||||
data = {
|
||||
"document": {
|
||||
"md_content": "# Title\nSome text",
|
||||
"html_content": "<h1>Title</h1><p>Some text</p>",
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
"body": [
|
||||
{
|
||||
"type": "title",
|
||||
"text": "Title",
|
||||
"level": 1,
|
||||
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40}}],
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Some text",
|
||||
"level": 0,
|
||||
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 50, "r": 200, "b": 70}}],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
result = _parse_response(data)
|
||||
assert len(result.pages[0].elements) == 2
|
||||
assert result.pages[0].elements[0].type == "title"
|
||||
assert result.pages[0].elements[0].content == "Title"
|
||||
assert result.pages[0].elements[0].bbox == [10, 20, 200, 40]
|
||||
assert result.pages[0].elements[1].type == "text"
|
||||
|
||||
def test_empty_response(self):
|
||||
data = {"document": {"pages": {}}}
|
||||
result = _parse_response(data)
|
||||
assert result.content_markdown == ""
|
||||
assert result.content_html == ""
|
||||
assert result.page_count == 1 # fallback minimum
|
||||
|
||||
def test_bbox_as_list(self):
|
||||
data = {
|
||||
"document": {
|
||||
"md_content": "",
|
||||
"html_content": "",
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
"body": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "hello",
|
||||
"prov": [{"page_no": 1, "bbox": [10.0, 20.0, 200.0, 40.0]}],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
result = _parse_response(data)
|
||||
assert result.pages[0].elements[0].bbox == [10.0, 20.0, 200.0, 40.0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests — item type mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMapItemType:
|
||||
@pytest.mark.parametrize("input_type,expected", [
|
||||
("table", "table"),
|
||||
("picture", "picture"),
|
||||
("figure", "picture"),
|
||||
("title", "title"),
|
||||
("section_header", "section_header"),
|
||||
("section-header", "section_header"),
|
||||
("list_item", "list"),
|
||||
("formula", "formula"),
|
||||
("equation", "formula"),
|
||||
("code", "code"),
|
||||
("text", "text"),
|
||||
("paragraph", "text"),
|
||||
("unknown_type", "text"),
|
||||
])
|
||||
def test_type_mapping(self, input_type, expected):
|
||||
assert _map_item_type({"type": input_type}) == expected
|
||||
|
||||
def test_missing_type_defaults_to_text(self):
|
||||
assert _map_item_type({}) == "text"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests — ServeConverter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestServeConverter:
|
||||
def test_headers_with_api_key(self):
|
||||
conv = ServeConverter(base_url="http://localhost:5001", api_key="secret")
|
||||
assert conv._headers() == {"X-Api-Key": "secret"}
|
||||
|
||||
def test_headers_without_api_key(self):
|
||||
conv = ServeConverter(base_url="http://localhost:5001")
|
||||
assert conv._headers() == {}
|
||||
|
||||
def test_build_conversion_options(self):
|
||||
conv = ServeConverter(base_url="http://localhost:5001")
|
||||
opts = ConversionOptions(do_ocr=False, table_mode="fast", images_scale=2.0)
|
||||
|
||||
result = conv._build_conversion_options(opts)
|
||||
|
||||
assert result["do_ocr"] is False
|
||||
assert result["table_mode"] == "fast"
|
||||
assert result["images_scale"] == 2.0
|
||||
assert result["to_formats"] == ["md", "html"]
|
||||
|
||||
def test_base_url_trailing_slash_stripped(self):
|
||||
conv = ServeConverter(base_url="http://localhost:5001/")
|
||||
assert conv._base_url == "http://localhost:5001"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration tests — HTTP calls (mocked)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestServeConverterConvert:
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_conversion(self, tmp_path):
|
||||
# Create a temp file to "upload"
|
||||
test_file = tmp_path / "test.pdf"
|
||||
test_file.write_bytes(b"%PDF-1.4 fake content")
|
||||
|
||||
serve_response = {
|
||||
"document": {
|
||||
"md_content": "# Converted",
|
||||
"html_content": "<h1>Converted</h1>",
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
"body": [
|
||||
{
|
||||
"type": "title",
|
||||
"text": "Converted",
|
||||
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40}}],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = serve_response
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post.return_value = mock_response
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
conv = ServeConverter(base_url="http://localhost:5001", api_key="test-key")
|
||||
|
||||
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||
result = await conv.convert(str(test_file), ConversionOptions())
|
||||
|
||||
assert isinstance(result, ConversionResult)
|
||||
assert result.content_markdown == "# Converted"
|
||||
assert result.page_count == 1
|
||||
assert len(result.pages[0].elements) == 1
|
||||
|
||||
# Verify the HTTP call
|
||||
mock_client.post.assert_called_once()
|
||||
call_kwargs = mock_client.post.call_args
|
||||
assert "/v1/convert/file" in call_kwargs[0][0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_error_raises(self, tmp_path):
|
||||
test_file = tmp_path / "test.pdf"
|
||||
test_file.write_bytes(b"%PDF-1.4 fake content")
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||
"Server Error", request=MagicMock(), response=MagicMock(status_code=500),
|
||||
)
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post.return_value = mock_response
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
conv = ServeConverter(base_url="http://localhost:5001")
|
||||
|
||||
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
await conv.convert(str(test_file), ConversionOptions())
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_success(self):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get.return_value = mock_response
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
conv = ServeConverter(base_url="http://localhost:5001")
|
||||
|
||||
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||
assert await conv.health_check() is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_check_failure(self):
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get.side_effect = httpx.ConnectError("Connection refused")
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
conv = ServeConverter(base_url="http://localhost:5001")
|
||||
|
||||
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||
assert await conv.health_check() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration — converter wiring in main.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestConverterWiring:
|
||||
def test_local_engine_builds_local_converter(self):
|
||||
from infra.local_converter import LocalConverter
|
||||
from infra.settings import Settings
|
||||
from main import _build_converter
|
||||
|
||||
with patch("main.settings", Settings(conversion_engine="local")):
|
||||
converter = _build_converter()
|
||||
assert isinstance(converter, LocalConverter)
|
||||
|
||||
def test_remote_engine_builds_serve_converter(self):
|
||||
from infra.settings import Settings
|
||||
from main import _build_converter
|
||||
|
||||
with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001")):
|
||||
converter = _build_converter()
|
||||
assert isinstance(converter, ServeConverter)
|
||||
assert converter._base_url == "http://serve:5001"
|
||||
|
||||
def test_remote_engine_passes_api_key(self):
|
||||
from infra.settings import Settings
|
||||
from main import _build_converter
|
||||
|
||||
with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001", docling_serve_api_key="my-key")):
|
||||
converter = _build_converter()
|
||||
assert isinstance(converter, ServeConverter)
|
||||
assert converter._api_key == "my-key"
|
||||
Loading…
Reference in a new issue