diff --git a/.env.example b/.env.example index 0821ea4..1e0a95d 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,8 @@ +# Docker Compose build target: "local" or "remote" (selects Dockerfile target) +# CONVERSION_MODE=local + # Conversion engine: "local" (in-process Docling) or "remote" (Docling Serve) +# Set automatically by the Docker target — override only for local dev # CONVERSION_ENGINE=local # Docling Serve settings (remote mode only) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a50c6ed..b58235a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,5 +55,5 @@ jobs: build-args: APP_VERSION=${{ steps.version.outputs.value }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.target }} + cache-to: type=gha,mode=max,scope=${{ matrix.target }} diff --git a/README.md b/README.md index fda0a1a..cee5b1f 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ document-parser/ ├── main.py # FastAPI app, CORS, lifespan ├── domain/ # Pure domain — no HTTP, no DB │ ├── models.py # Document, AnalysisJob dataclasses -│ ├── parsing.py # Docling conversion & page extraction -│ └── bbox.py # Bounding box coordinate normalization +│ ├── ports.py # Abstract protocols (converter, chunker) +│ └── value_objects.py # ConversionResult, PageDetail, ChunkResult ├── api/ # HTTP layer (FastAPI routers) │ ├── schemas.py # Pydantic DTOs (camelCase serialization) │ ├── documents.py # /api/documents endpoints diff --git a/docs/architecture.md b/docs/architecture.md index 9cfcade..f51795f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -68,7 +68,7 @@ document-parser/ | Layer | Role | Depends on | |-------|------|------------| -| **domain** | Dataclasses, bbox math, Docling conversion | Nothing (pure Python) | +| **domain** | Dataclasses, value objects, ports | Nothing (pure Python) | | **persistence** | SQLite CRUD, aiosqlite | domain (models) | | **services** | Orchestrate use cases, call Docling | domain + persistence | | **api** | HTTP endpoints, Pydantic DTOs, error handling | services | diff --git a/docs/bbox-pipeline.md b/docs/bbox-pipeline.md index b0be9e1..4f2e0fe 100644 --- a/docs/bbox-pipeline.md +++ b/docs/bbox-pipeline.md @@ -38,7 +38,7 @@ The frontend converts PDF points to CSS pixels, then the canvas renders at `devi ## Transformation 1 — `to_topleft_list()` -**File:** `document-parser/domain/bbox.py` +**File:** `document-parser/infra/bbox.py` Normalizes any Docling bbox to `[left, top, right, bottom]` in TOPLEFT coordinates. diff --git a/document-parser/domain/bbox.py b/document-parser/infra/bbox.py similarity index 100% rename from document-parser/domain/bbox.py rename to document-parser/infra/bbox.py diff --git a/document-parser/infra/local_chunker.py b/document-parser/infra/local_chunker.py index 500b691..1b59406 100644 --- a/document-parser/infra/local_chunker.py +++ b/document-parser/infra/local_chunker.py @@ -15,7 +15,7 @@ from docling_core.transforms.chunker import HierarchicalChunker from docling_core.transforms.chunker.hybrid_chunker import HybridChunker from docling_core.types.doc.document import DoclingDocument -from domain.bbox import EMPTY_BBOX, to_topleft_list +from infra.bbox import EMPTY_BBOX, to_topleft_list from domain.value_objects import ChunkBbox, ChunkingOptions, ChunkResult logger = logging.getLogger(__name__) diff --git a/document-parser/infra/local_converter.py b/document-parser/infra/local_converter.py index 9e0b23a..765f954 100644 --- a/document-parser/infra/local_converter.py +++ b/document-parser/infra/local_converter.py @@ -35,7 +35,7 @@ from docling_core.types.doc import ( TitleItem, ) -from domain.bbox import to_topleft_list +from infra.bbox import to_topleft_list from domain.value_objects import ( ConversionOptions, ConversionResult, diff --git a/document-parser/infra/serve_converter.py b/document-parser/infra/serve_converter.py index 410e497..a6a0502 100644 --- a/document-parser/infra/serve_converter.py +++ b/document-parser/infra/serve_converter.py @@ -18,6 +18,7 @@ import mimetypes from pathlib import Path import httpx +from docling_core.types.doc.base import BoundingBox, CoordOrigin from domain.value_objects import ( ConversionOptions, @@ -25,6 +26,7 @@ from domain.value_objects import ( PageDetail, PageElement, ) +from infra.bbox import to_topleft_list logger = logging.getLogger(__name__) @@ -126,6 +128,7 @@ def _build_form_data(options: ConversionOptions) -> dict[str, str | list[str]]: "do_picture_classification": str(options.do_picture_classification).lower(), "do_picture_description": str(options.do_picture_description).lower(), "include_images": str(options.generate_picture_images).lower(), + "generate_page_images": str(options.generate_page_images).lower(), "images_scale": str(options.images_scale), } @@ -152,11 +155,14 @@ def _parse_response(data: dict) -> ConversionResult: page_count = len(pages) if pages else 1 + document_json = json.dumps(json_content) if json_content else None + return ConversionResult( page_count=page_count, content_markdown=content_md, content_html=content_html, pages=pages, + document_json=document_json, ) @@ -218,21 +224,22 @@ def _add_element(item: dict, pages: dict[int, PageDetail]) -> None: def _extract_bbox(bbox_data: dict, page_height: float) -> list[float]: - """Extract and normalize bbox to TOPLEFT [l, t, r, b] format.""" + """Extract and normalize bbox to TOPLEFT [l, t, r, b] format. + + Delegates to the canonical to_topleft_list function via a docling-core + BoundingBox, ensuring consistent coordinate handling across all converters. + """ if not isinstance(bbox_data, dict): return [0.0, 0.0, 0.0, 0.0] - left = bbox_data.get("l", 0.0) - top = bbox_data.get("t", 0.0) - right = bbox_data.get("r", 0.0) - bottom = bbox_data.get("b", 0.0) - coord_origin = bbox_data.get("coord_origin", "TOPLEFT") + origin_str = bbox_data.get("coord_origin", "TOPLEFT") + origin = CoordOrigin.BOTTOMLEFT if origin_str == "BOTTOMLEFT" else CoordOrigin.TOPLEFT - if coord_origin == "BOTTOMLEFT": - # In BOTTOMLEFT: top has higher y, bottom has lower y - # In TOPLEFT: flip both — new_top = page_height - old_top - new_top = page_height - top - new_bottom = page_height - bottom - top, bottom = new_top, new_bottom - - return [left, top, right, bottom] + bbox = BoundingBox( + l=bbox_data.get("l", 0.0), + t=bbox_data.get("t", 0.0), + r=bbox_data.get("r", 0.0), + b=bbox_data.get("b", 0.0), + coord_origin=origin, + ) + return to_topleft_list(bbox, page_height) diff --git a/document-parser/persistence/analysis_repo.py b/document-parser/persistence/analysis_repo.py index 0652e5b..9d6a7a7 100644 --- a/document-parser/persistence/analysis_repo.py +++ b/document-parser/persistence/analysis_repo.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime from domain.models import AnalysisJob, AnalysisStatus from persistence.database import get_connection @@ -29,7 +29,7 @@ def _row_to_job(row) -> AnalysisJob: error_message=row["error_message"], started_at=_parse_dt(row["started_at"]), completed_at=_parse_dt(row["completed_at"]), - created_at=_parse_dt(row["created_at"]) or datetime.now(), + created_at=_parse_dt(row["created_at"]) or datetime.now(UTC), document_filename=row["filename"] if "filename" in keys else None, ) diff --git a/document-parser/persistence/document_repo.py b/document-parser/persistence/document_repo.py index 623e36a..04ac473 100644 --- a/document-parser/persistence/document_repo.py +++ b/document-parser/persistence/document_repo.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime from domain.models import Document from persistence.database import get_connection @@ -12,6 +12,8 @@ def _row_to_document(row) -> Document: created = row["created_at"] if isinstance(created, str): created = datetime.fromisoformat(created) + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) return Document( id=row["id"], filename=row["filename"], diff --git a/document-parser/requirements-local.txt b/document-parser/requirements-local.txt index 82cf543..fc2f3c7 100644 --- a/document-parser/requirements-local.txt +++ b/document-parser/requirements-local.txt @@ -1,3 +1,2 @@ -r requirements.txt docling>=2.80.0,<3.0.0 -docling-core>=2.0.0,<3.0.0 diff --git a/document-parser/requirements.txt b/document-parser/requirements.txt index 24aeedb..92dcf4c 100644 --- a/document-parser/requirements.txt +++ b/document-parser/requirements.txt @@ -1,3 +1,4 @@ +docling-core>=2.0.0,<3.0.0 fastapi>=0.115.0,<1.0.0 uvicorn[standard]>=0.32.0,<1.0.0 python-multipart>=0.0.12 diff --git a/document-parser/services/document_service.py b/document-parser/services/document_service.py index cba3d5c..bee24fb 100644 --- a/document-parser/services/document_service.py +++ b/document-parser/services/document_service.py @@ -32,7 +32,7 @@ async def upload(filename: str, content_type: str, file_content: bytes) -> Docum os.makedirs(UPLOAD_DIR, exist_ok=True) - ext = os.path.splitext(filename)[1] or ".pdf" + ext = ".pdf" # Content already validated as PDF safe_name = f"{uuid.uuid4()}{ext}" file_path = os.path.join(UPLOAD_DIR, safe_name) diff --git a/document-parser/tests/test_bbox.py b/document-parser/tests/test_bbox.py index f201a73..8d3694f 100644 --- a/document-parser/tests/test_bbox.py +++ b/document-parser/tests/test_bbox.py @@ -8,7 +8,7 @@ misaligned overlays in the UI. import pytest from docling_core.types.doc.base import BoundingBox, CoordOrigin -from domain.bbox import EMPTY_BBOX, to_topleft_list +from infra.bbox import EMPTY_BBOX, to_topleft_list # --------------------------------------------------------------------------- # Standard conversions diff --git a/document-parser/tests/test_serve_converter.py b/document-parser/tests/test_serve_converter.py index dd4d4ab..6b0b9cb 100644 --- a/document-parser/tests/test_serve_converter.py +++ b/document-parser/tests/test_serve_converter.py @@ -32,6 +32,7 @@ class TestBuildFormData: assert data["do_picture_classification"] == "false" assert data["do_picture_description"] == "false" assert data["include_images"] == "false" + assert data["generate_page_images"] == "false" assert data["images_scale"] == "1.0" assert set(data["to_formats"]) == {"md", "html", "json"} @@ -74,6 +75,7 @@ class TestParseResponse: assert result.content_html == "