Compare commits

...

5 commits

Author SHA1 Message Date
Pier-Jean Malandrino
3bdc4cec50 feat: enable chunking in remote (Docling Serve) mode
Hybrid approach: reuse LocalChunker to chunk the DoclingDocument JSON
returned by Serve, so chunking works identically in both local and
remote modes without calling Serve's chunk endpoint.

Backend:
- _build_chunker() always returns LocalChunker (remove engine guard)
- Use docling-core[chunking] extra for required dependencies
- Skip client-side batching in remote mode (Serve manages its own
  resources, and batching discards document_json needed for chunking)
- Fix Serve form fields: remove generate_page_images (not a Serve
  field), use repeated form keys for to_formats and page_range
- Log Serve error response body on 4xx/5xx for diagnosis
- Fix FastAPI 204 DELETE routes missing response_model=None

Frontend:
- Update chunking feature flag to enable Prepare UI in remote mode

Closes #51
2026-04-16 15:11:14 +02:00
Pier-Jean Malandrino
0bffe6e7d4
Merge pull request #183 from scub-france/feat/centralize-magic-numbers-arch-tests
refactor: centralize magic numbers + arch tests (#168, #177)
2026-04-16 10:51:48 +02:00
Pier-Jean Malandrino
987d43735d fix(ci): install pytestarch in backend tests job (#177)
CI was missing pytestarch dependency, causing test_architecture.py to fail
at collection time. Switch to requirements-test.txt which includes all
test dependencies.
2026-04-16 08:32:07 +00:00
Pier-Jean Malandrino
7a76d2efbd feat: add hexagonal architecture tests with pytestarch (#177)
- Create tests/test_architecture.py with 20 automated rules:
  inter-layer dependency checks (domain, services, api, infra, persistence),
  external dependency constraints (fastapi, sqlalchemy, httpx, opensearchpy),
  and port convention enforcement (Protocol only in domain.ports)
- Add requirements-test.txt with pytestarch dependency
- Fix persistence.database importing infra.settings (read DB_PATH from env directly)
2026-04-16 08:32:07 +00:00
Pier-Jean Malandrino
f2436290c5 refactor: centralize magic numbers for page dimensions, limits, and timeout (#168)
- Move DEFAULT_PAGE_WIDTH/HEIGHT to domain/value_objects.py and import in both converters
- Add opensearch_default_limit to Settings (configurable via OPENSEARCH_DEFAULT_LIMIT env var)
- Pass settings.conversion_timeout to ServeConverter, removing independent _DEFAULT_TIMEOUT
- Update OpenSearchStore to accept default_limit from Settings via constructor
2026-04-16 08:32:07 +00:00
21 changed files with 399 additions and 47 deletions

View file

@ -29,7 +29,7 @@ jobs:
with: with:
python-version: "3.12" python-version: "3.12"
cache: pip cache: pip
cache-dependency-path: document-parser/requirements.txt cache-dependency-path: document-parser/requirements-test.txt
- name: Install system dependencies - name: Install system dependencies
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends poppler-utils run: sudo apt-get update && sudo apt-get install -y --no-install-recommends poppler-utils
@ -37,8 +37,8 @@ jobs:
- name: Install Python dependencies - name: Install Python dependencies
run: | run: |
pip install --upgrade pip pip install --upgrade pip
pip install -r requirements.txt pip install -r requirements-test.txt
pip install pytest pytest-asyncio httpx ruff pip install httpx ruff
- name: Lint - name: Lint
run: ruff check . run: ruff check .

View file

@ -155,7 +155,7 @@ async def delete_chunk(job_id: str, chunk_index: int, service: ServiceDep) -> li
] ]
@router.delete("/{job_id}", status_code=204) @router.delete("/{job_id}", status_code=204, response_model=None)
async def delete_analysis(job_id: str, service: ServiceDep) -> None: async def delete_analysis(job_id: str, service: ServiceDep) -> None:
"""Delete an analysis job.""" """Delete an analysis job."""
deleted = await service.delete(job_id) deleted = await service.delete(job_id)

View file

@ -85,7 +85,7 @@ async def get_document(doc_id: str, service: ServiceDep) -> DocumentResponse:
return _to_response(doc) return _to_response(doc)
@router.delete("/{doc_id}", status_code=204) @router.delete("/{doc_id}", status_code=204, response_model=None)
async def delete_document(doc_id: str, service: ServiceDep) -> None: async def delete_document(doc_id: str, service: ServiceDep) -> None:
"""Delete a document and its file.""" """Delete a document and its file."""
deleted = await service.delete(doc_id) deleted = await service.delete(doc_id)

View file

@ -74,7 +74,7 @@ async def ingest_analysis(
) )
@router.delete("/{doc_id}", status_code=204) @router.delete("/{doc_id}", status_code=204, response_model=None)
async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None: async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None:
"""Delete all indexed chunks for a document.""" """Delete all indexed chunks for a document."""
await ingestion.delete_document(doc_id) await ingestion.delete_document(doc_id)

View file

@ -8,6 +8,10 @@ from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass, field
# US Letter page dimensions (points) — fallback when page size is unknown
DEFAULT_PAGE_WIDTH: float = 612.0
DEFAULT_PAGE_HEIGHT: float = 792.0
@dataclass(frozen=True) @dataclass(frozen=True)
class PageElement: class PageElement:

View file

@ -36,6 +36,8 @@ from docling_core.types.doc import (
) )
from domain.value_objects import ( from domain.value_objects import (
DEFAULT_PAGE_HEIGHT,
DEFAULT_PAGE_WIDTH,
ConversionOptions, ConversionOptions,
ConversionResult, ConversionResult,
PageDetail, PageDetail,
@ -50,10 +52,6 @@ logger = logging.getLogger(__name__)
# Uses a timeout to prevent a frozen conversion from blocking all others. # Uses a timeout to prevent a frozen conversion from blocking all others.
_converter_lock = threading.Lock() _converter_lock = threading.Lock()
# US Letter page dimensions (points) — fallback when page size is unknown
_DEFAULT_PAGE_WIDTH = 612.0
_DEFAULT_PAGE_HEIGHT = 792.0
# Default converter (lazy-init on first request) # Default converter (lazy-init on first request)
_default_converter: DoclingConverter | None = None _default_converter: DoclingConverter | None = None
@ -175,11 +173,11 @@ def _process_content_item(
logger.warning( logger.warning(
"Page %d not found in document metadata — using US Letter fallback (%sx%s pt)", "Page %d not found in document metadata — using US Letter fallback (%sx%s pt)",
page_no, page_no,
_DEFAULT_PAGE_WIDTH, DEFAULT_PAGE_WIDTH,
_DEFAULT_PAGE_HEIGHT, DEFAULT_PAGE_HEIGHT,
) )
pages[page_no] = PageDetail( pages[page_no] = PageDetail(
page_number=page_no, width=_DEFAULT_PAGE_WIDTH, height=_DEFAULT_PAGE_HEIGHT page_number=page_no, width=DEFAULT_PAGE_WIDTH, height=DEFAULT_PAGE_HEIGHT
) )
page_height = pages[page_no].height page_height = pages[page_no].height
@ -248,10 +246,10 @@ def _convert_sync(
pages_detail = [ pages_detail = [
PageDetail( PageDetail(
page_number=i + 1, page_number=i + 1,
width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else _DEFAULT_PAGE_WIDTH, width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else DEFAULT_PAGE_WIDTH,
height=doc.pages[i + 1].size.height height=doc.pages[i + 1].size.height
if (i + 1) in doc.pages if (i + 1) in doc.pages
else _DEFAULT_PAGE_HEIGHT, else DEFAULT_PAGE_HEIGHT,
) )
for i in range(page_count) for i in range(page_count)
] ]

View file

@ -69,13 +69,14 @@ class OpenSearchStore:
verify_certs: Whether to verify TLS certificates. verify_certs: Whether to verify TLS certificates.
""" """
def __init__(self, url: str, *, verify_certs: bool = False) -> None: def __init__(self, url: str, *, verify_certs: bool = False, default_limit: int = 1000) -> None:
self._client = AsyncOpenSearch( self._client = AsyncOpenSearch(
hosts=[url], hosts=[url],
use_ssl=url.startswith("https"), use_ssl=url.startswith("https"),
verify_certs=verify_certs, verify_certs=verify_certs,
ssl_show_warn=False, ssl_show_warn=False,
) )
self._default_limit = default_limit
# -- lifecycle ------------------------------------------------------------- # -- lifecycle -------------------------------------------------------------
@ -147,9 +148,11 @@ class OpenSearchStore:
index_name: str, index_name: str,
doc_id: str, doc_id: str,
*, *,
limit: int = 1000, limit: int | None = None,
) -> list[SearchResult]: ) -> list[SearchResult]:
"""Retrieve all indexed chunks for a document, ordered by chunk_index.""" """Retrieve all indexed chunks for a document, ordered by chunk_index."""
if limit is None:
limit = self._default_limit
resp = await self._client.search( resp = await self._client.search(
index=index_name, index=index_name,
body={ body={

View file

@ -21,6 +21,8 @@ import httpx
from docling_core.types.doc.base import BoundingBox, CoordOrigin from docling_core.types.doc.base import BoundingBox, CoordOrigin
from domain.value_objects import ( from domain.value_objects import (
DEFAULT_PAGE_HEIGHT,
DEFAULT_PAGE_WIDTH,
ConversionOptions, ConversionOptions,
ConversionResult, ConversionResult,
PageDetail, PageDetail,
@ -31,7 +33,6 @@ from infra.bbox import to_topleft_list
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_API_PREFIX = "/v1" _API_PREFIX = "/v1"
_DEFAULT_TIMEOUT = 600.0
# Docling Serve label → our element type # Docling Serve label → our element type
_LABEL_MAP = { _LABEL_MAP = {
@ -60,7 +61,7 @@ class ServeConverter:
self, self,
base_url: str, base_url: str,
api_key: str | None = None, api_key: str | None = None,
timeout: float = _DEFAULT_TIMEOUT, timeout: float = 600.0,
): ):
self._base_url = base_url.rstrip("/") self._base_url = base_url.rstrip("/")
self._api_key = api_key self._api_key = api_key
@ -95,6 +96,13 @@ class ServeConverter:
headers=self._headers(), headers=self._headers(),
) )
if response.status_code >= 400:
logger.error(
"Docling Serve error %d: %s (form_data=%s)",
response.status_code,
response.text[:500],
{k: v for k, v in form_data.items()},
)
response.raise_for_status() response.raise_for_status()
result_data = response.json() result_data = response.json()
@ -121,8 +129,12 @@ def _build_form_data(
) -> dict[str, str | list[str]]: ) -> dict[str, str | list[str]]:
"""Build form fields matching Docling Serve's multipart form contract. """Build form fields matching Docling Serve's multipart form contract.
Array fields (to_formats) are sent as lists httpx encodes them as Serve uses FastAPI's ``Form()`` parsing — list/tuple fields are sent
repeated form keys (to_formats=md&to_formats=html&to_formats=json). as **repeated form keys** (httpx encodes Python lists this way
automatically: ``to_formats=md&to_formats=html&to_formats=json``).
Note: ``generate_page_images`` is a PdfPipelineOptions field, NOT a
ConvertDocumentsOptions field sending it causes a 422.
""" """
data: dict[str, str | list[str]] = { data: dict[str, str | list[str]] = {
"to_formats": ["md", "html", "json"], "to_formats": ["md", "html", "json"],
@ -134,11 +146,12 @@ def _build_form_data(
"do_picture_classification": str(options.do_picture_classification).lower(), "do_picture_classification": str(options.do_picture_classification).lower(),
"do_picture_description": str(options.do_picture_description).lower(), "do_picture_description": str(options.do_picture_description).lower(),
"include_images": str(options.generate_picture_images).lower(), "include_images": str(options.generate_picture_images).lower(),
"generate_page_images": str(options.generate_page_images).lower(),
"images_scale": str(options.images_scale), "images_scale": str(options.images_scale),
} }
if page_range is not None: if page_range is not None:
data["page_range"] = f"{page_range[0]}-{page_range[1]}" # Serve expects page_range as two repeated form fields:
# page_range=1&page_range=10
data["page_range"] = [str(page_range[0]), str(page_range[1])]
return data return data
@ -192,8 +205,8 @@ def _extract_pages_from_docling_document(doc: dict) -> list[PageDetail]:
size = page_data.get("size", {}) size = page_data.get("size", {})
pages_dict[page_no] = PageDetail( pages_dict[page_no] = PageDetail(
page_number=page_no, page_number=page_no,
width=size.get("width", 612.0), width=size.get("width", DEFAULT_PAGE_WIDTH),
height=size.get("height", 792.0), height=size.get("height", DEFAULT_PAGE_HEIGHT),
) )
# Process all element arrays # Process all element arrays
@ -220,8 +233,8 @@ def _add_element(item: dict, pages: dict[int, PageDetail]) -> None:
if page_no not in pages: if page_no not in pages:
pages[page_no] = PageDetail( pages[page_no] = PageDetail(
page_number=page_no, page_number=page_no,
width=612.0, width=DEFAULT_PAGE_WIDTH,
height=792.0, height=DEFAULT_PAGE_HEIGHT,
) )
bbox_data = prov.get("bbox", {}) bbox_data = prov.get("bbox", {})

View file

@ -25,6 +25,7 @@ class Settings:
batch_page_size: int = 0 # 0 = disabled, > 0 = pages per batch batch_page_size: int = 0 # 0 = disabled, > 0 = pages per batch
opensearch_url: str = "" # empty = disabled opensearch_url: str = "" # empty = disabled
embedding_url: str = "" # empty = disabled (e.g. http://localhost:8001) embedding_url: str = "" # empty = disabled (e.g. http://localhost:8001)
opensearch_default_limit: int = 1000 # max chunks returned by get_chunks
embedding_dimension: int = 384 # Granite Embedding 30M / all-MiniLM-L6-v2 embedding_dimension: int = 384 # Granite Embedding 30M / all-MiniLM-L6-v2
upload_dir: str = "./uploads" upload_dir: str = "./uploads"
db_path: str = "./data/docling_studio.db" db_path: str = "./data/docling_studio.db"
@ -54,6 +55,10 @@ class Settings:
errors.append(f"rate_limit_rpm must be >= 0 (got {self.rate_limit_rpm})") errors.append(f"rate_limit_rpm must be >= 0 (got {self.rate_limit_rpm})")
if self.batch_page_size < 0: if self.batch_page_size < 0:
errors.append(f"batch_page_size must be >= 0 (got {self.batch_page_size})") errors.append(f"batch_page_size must be >= 0 (got {self.batch_page_size})")
if self.opensearch_default_limit < 1:
errors.append(
f"opensearch_default_limit must be >= 1 (got {self.opensearch_default_limit})"
)
if self.embedding_dimension < 1: if self.embedding_dimension < 1:
errors.append(f"embedding_dimension must be >= 1 (got {self.embedding_dimension})") errors.append(f"embedding_dimension must be >= 1 (got {self.embedding_dimension})")
if self.default_table_mode not in ("accurate", "fast"): if self.default_table_mode not in ("accurate", "fast"):
@ -97,6 +102,7 @@ class Settings:
batch_page_size=int(os.environ.get("BATCH_PAGE_SIZE", "10")), batch_page_size=int(os.environ.get("BATCH_PAGE_SIZE", "10")),
opensearch_url=os.environ.get("OPENSEARCH_URL", ""), opensearch_url=os.environ.get("OPENSEARCH_URL", ""),
embedding_url=os.environ.get("EMBEDDING_URL", ""), embedding_url=os.environ.get("EMBEDDING_URL", ""),
opensearch_default_limit=int(os.environ.get("OPENSEARCH_DEFAULT_LIMIT", "1000")),
embedding_dimension=int(os.environ.get("EMBEDDING_DIMENSION", "384")), embedding_dimension=int(os.environ.get("EMBEDDING_DIMENSION", "384")),
upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"), upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"),
db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"), db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"),

View file

@ -47,6 +47,7 @@ def _build_converter():
return ServeConverter( return ServeConverter(
base_url=settings.docling_serve_url, base_url=settings.docling_serve_url,
api_key=settings.docling_serve_api_key, api_key=settings.docling_serve_api_key,
timeout=settings.conversion_timeout,
) )
else: else:
from infra.local_converter import LocalConverter from infra.local_converter import LocalConverter
@ -56,12 +57,15 @@ def _build_converter():
def _build_chunker(): def _build_chunker():
"""Build the chunker adapter — only available in local mode.""" """Build the chunker adapter.
if settings.conversion_engine == "local":
from infra.local_chunker import LocalChunker
return LocalChunker() Uses LocalChunker in all modes in remote mode it chunks the
return None DoclingDocument JSON returned by Docling Serve, so docling-core
(lightweight) is the only local dependency needed.
"""
from infra.local_chunker import LocalChunker
return LocalChunker()
def _build_repos() -> tuple[SqliteDocumentRepository, SqliteAnalysisRepository]: def _build_repos() -> tuple[SqliteDocumentRepository, SqliteAnalysisRepository]:
@ -99,7 +103,10 @@ def _build_ingestion_service() -> IngestionService | None:
from infra.opensearch_store import OpenSearchStore from infra.opensearch_store import OpenSearchStore
embedding = EmbeddingClient(settings.embedding_url) embedding = EmbeddingClient(settings.embedding_url)
vector_store = OpenSearchStore(settings.opensearch_url) vector_store = OpenSearchStore(
settings.opensearch_url,
default_limit=settings.opensearch_default_limit,
)
config = IngestionConfig( config = IngestionConfig(
embedding_dimension=settings.embedding_dimension, embedding_dimension=settings.embedding_dimension,
) )

View file

@ -9,11 +9,9 @@ from contextlib import asynccontextmanager
import aiosqlite import aiosqlite
from infra.settings import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
DB_PATH = settings.db_path DB_PATH = os.environ.get("DB_PATH", "./data/docling_studio.db")
_SCHEMA = """ _SCHEMA = """
CREATE TABLE IF NOT EXISTS documents ( CREATE TABLE IF NOT EXISTS documents (

View file

@ -0,0 +1,4 @@
-r requirements.txt
pytest>=8.0.0,<9.0.0
pytest-asyncio>=0.23.0,<1.0.0
pytestarch>=2.0.0,<3.0.0

View file

@ -1,4 +1,4 @@
docling-core>=2.0.0,<3.0.0 docling-core[chunking]>=2.0.0,<3.0.0
fastapi>=0.115.0,<1.0.0 fastapi>=0.115.0,<1.0.0
uvicorn[standard]>=0.32.0,<1.0.0 uvicorn[standard]>=0.32.0,<1.0.0
python-multipart>=0.0.12 python-multipart>=0.0.12

View file

@ -324,11 +324,18 @@ class AnalysisService:
file_path: str, file_path: str,
options: ConversionOptions, options: ConversionOptions,
) -> ConversionResult | None: ) -> ConversionResult | None:
"""Run batched or single conversion. Returns None if the job was deleted mid-batch.""" """Run batched or single conversion. Returns None if the job was deleted mid-batch.
Batching is only used for local mode it limits memory usage when
Docling runs in-process. In remote mode the Serve instance manages
its own resources, and batching would discard document_json (needed
for chunking).
"""
total_pages = _count_pdf_pages(file_path) total_pages = _count_pdf_pages(file_path)
batch_size = self._config.batch_page_size batch_size = self._config.batch_page_size
is_remote = self._is_remote_converter()
if batch_size > 0 and total_pages > batch_size: if batch_size > 0 and total_pages > batch_size and not is_remote:
return await self._run_batched_conversion( return await self._run_batched_conversion(
job_id, file_path, options, total_pages, batch_size job_id, file_path, options, total_pages, batch_size
) )
@ -337,6 +344,15 @@ class AnalysisService:
timeout=self._conversion_timeout, timeout=self._conversion_timeout,
) )
def _is_remote_converter(self) -> bool:
"""Check if the converter is a remote (Serve) adapter."""
try:
from infra.serve_converter import ServeConverter
return isinstance(self._converter, ServeConverter)
except ImportError:
return False
async def _finalize_analysis( async def _finalize_analysis(
self, self,
job_id: str, job_id: str,

View file

@ -0,0 +1,208 @@
"""Hexagonal architecture tests — enforce layer dependency rules.
Uses pytestarch for inter-layer dependency rules and ast-based import
scanning for external (third-party) dependency constraints.
Rules enforced:
- domain -> no import from api, services, infra, persistence
- services -> no import from api, infra, persistence
- api -> no import from infra, persistence
- infra -> no import from api, services
- persistence -> no import from api, services, infra
- domain -> no import of fastapi, sqlalchemy, httpx, opensearchpy
- services -> no import of fastapi
"""
from __future__ import annotations
import ast
from pathlib import Path
import pytest
from pytestarch import Rule, get_evaluable_architecture
# ---------------------------------------------------------------------------
# pytestarch evaluable (project root = document-parser/)
# ---------------------------------------------------------------------------
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
# pytestarch uses the directory name as module prefix when given absolute paths.
# We use the directory name to build qualified module references.
_PREFIX = _PROJECT_ROOT.name # "document-parser"
_evaluable = get_evaluable_architecture(str(_PROJECT_ROOT), str(_PROJECT_ROOT))
def _mod(layer: str) -> str:
"""Return the fully-qualified pytestarch module name for a layer."""
return f"{_PREFIX}.{layer}"
# ---------------------------------------------------------------------------
# Helper: collect top-level imports from all .py files in a package
# ---------------------------------------------------------------------------
def _collect_imports(package: str) -> set[str]:
"""Return the set of top-level module names imported by *package*."""
pkg_path = Path(_PROJECT_ROOT) / package
imports: set[str] = set()
for py_file in pkg_path.rglob("*.py"):
tree = ast.parse(py_file.read_text(), filename=str(py_file))
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
imports.add(alias.name.split(".")[0])
elif isinstance(node, ast.ImportFrom) and node.module:
imports.add(node.module.split(".")[0])
return imports
# ---------------------------------------------------------------------------
# Inter-layer dependency rules (pytestarch)
# ---------------------------------------------------------------------------
class TestDomainLayerIsolation:
"""domain must not depend on any other layer."""
@pytest.mark.parametrize("forbidden", ["api", "services", "infra", "persistence"])
def test_domain_does_not_import(self, forbidden: str):
rule = (
Rule()
.modules_that()
.are_sub_modules_of(_mod("domain"))
.should_not()
.import_modules_that()
.are_sub_modules_of(_mod(forbidden))
)
rule.assert_applies(_evaluable)
class TestServicesLayerIsolation:
"""services may import domain only."""
@pytest.mark.parametrize("forbidden", ["api", "infra", "persistence"])
def test_services_does_not_import(self, forbidden: str):
rule = (
Rule()
.modules_that()
.are_sub_modules_of(_mod("services"))
.should_not()
.import_modules_that()
.are_sub_modules_of(_mod(forbidden))
)
rule.assert_applies(_evaluable)
class TestApiLayerIsolation:
"""api may import services and domain, but not infra or persistence."""
@pytest.mark.parametrize("forbidden", ["infra", "persistence"])
def test_api_does_not_import(self, forbidden: str):
rule = (
Rule()
.modules_that()
.are_sub_modules_of(_mod("api"))
.should_not()
.import_modules_that()
.are_sub_modules_of(_mod(forbidden))
)
rule.assert_applies(_evaluable)
class TestInfraLayerIsolation:
"""infra may import domain (ports), but not api or services."""
@pytest.mark.parametrize("forbidden", ["api", "services"])
def test_infra_does_not_import(self, forbidden: str):
rule = (
Rule()
.modules_that()
.are_sub_modules_of(_mod("infra"))
.should_not()
.import_modules_that()
.are_sub_modules_of(_mod(forbidden))
)
rule.assert_applies(_evaluable)
class TestPersistenceLayerIsolation:
"""persistence may import domain, but not api, services, or infra."""
@pytest.mark.parametrize("forbidden", ["api", "services", "infra"])
def test_persistence_does_not_import(self, forbidden: str):
rule = (
Rule()
.modules_that()
.are_sub_modules_of(_mod("persistence"))
.should_not()
.import_modules_that()
.are_sub_modules_of(_mod(forbidden))
)
rule.assert_applies(_evaluable)
# ---------------------------------------------------------------------------
# External dependency rules (ast-based)
# ---------------------------------------------------------------------------
_DOMAIN_FORBIDDEN_EXTERNALS = {"fastapi", "sqlalchemy", "httpx", "opensearchpy"}
_SERVICES_FORBIDDEN_EXTERNALS = {"fastapi"}
class TestDomainExternalDependencies:
"""domain must not import infrastructure-specific third-party libraries."""
@pytest.mark.parametrize("lib", sorted(_DOMAIN_FORBIDDEN_EXTERNALS))
def test_domain_does_not_import_external(self, lib: str):
imports = _collect_imports("domain")
assert lib not in imports, f"domain imports forbidden external library '{lib}'"
class TestServicesExternalDependencies:
"""services must not import web-framework libraries."""
@pytest.mark.parametrize("lib", sorted(_SERVICES_FORBIDDEN_EXTERNALS))
def test_services_does_not_import_external(self, lib: str):
imports = _collect_imports("services")
assert lib not in imports, f"services imports forbidden external library '{lib}'"
# ---------------------------------------------------------------------------
# Convention: ports live exclusively in domain.ports
# ---------------------------------------------------------------------------
class TestPortConvention:
"""Protocol definitions (ports) must live in domain.ports only."""
def test_no_protocol_outside_domain_ports(self):
"""No Protocol subclass should be defined outside domain/ports.py."""
ports_file = Path(_PROJECT_ROOT) / "domain" / "ports.py"
for py_file in Path(_PROJECT_ROOT).rglob("*.py"):
if py_file == ports_file:
continue
# Skip test files and __pycache__
if "tests" in py_file.parts or "__pycache__" in py_file.parts:
continue
tree = ast.parse(py_file.read_text(), filename=str(py_file))
for node in ast.walk(tree):
if isinstance(node, ast.ClassDef):
for base in node.bases:
base_name = _get_name(base)
if base_name == "Protocol":
pytest.fail(
f"Protocol '{node.name}' defined in {py_file.relative_to(_PROJECT_ROOT)}"
f" — ports must live in domain/ports.py"
)
def _get_name(node: ast.expr) -> str:
"""Extract a simple name from an AST expression node."""
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
return node.attr
return ""

View file

@ -465,3 +465,68 @@ class TestRechunkEndpoint:
}, },
) )
assert resp.status_code == 422 assert resp.status_code == 422
# ---------------------------------------------------------------------------
# Remote chunking path — hybrid local chunking from Serve's document_json
# ---------------------------------------------------------------------------
class TestRemoteChunkingPath:
"""Verify that chunking works on document_json produced by Serve (remote mode)."""
@pytest.mark.asyncio
async def test_rechunk_with_serve_document_json(self):
"""AnalysisService.rechunk() works with a LocalChunker even in remote mode."""
from infra.local_chunker import LocalChunker
from services.analysis_service import AnalysisService
chunker = LocalChunker()
analysis_repo = AsyncMock()
document_repo = AsyncMock()
converter = AsyncMock() # ServeConverter mock — not used for rechunking
service = AnalysisService(
converter=converter,
analysis_repo=analysis_repo,
document_repo=document_repo,
chunker=chunker,
)
# Simulate a completed job with document_json from Serve
job = AnalysisJob(id="j-remote", document_id="d1")
job.mark_running()
job.mark_completed(
markdown="# Title\nParagraph text here.",
html="<h1>Title</h1><p>Paragraph text here.</p>",
pages_json="[]",
document_json=json.dumps({
"schema_name": "DoclingDocument",
"version": "1.0.0",
"name": "test",
"origin": {
"mimetype": "application/pdf",
"filename": "test.pdf",
"binary_hash": 0,
},
"furniture": {"self_ref": "#/furniture", "children": [], "content_layer": "furniture"},
"body": {"self_ref": "#/body", "children": [], "content_layer": "body"},
"groups": [],
"texts": [],
"pictures": [],
"tables": [],
"key_value_items": [],
"form_items": [],
"pages": {},
}),
)
analysis_repo.find_by_id = AsyncMock(return_value=job)
analysis_repo.update_chunks = AsyncMock(return_value=True)
chunks = await service.rechunk(
"j-remote",
{"chunker_type": "hybrid", "max_tokens": 512},
)
assert isinstance(chunks, list)
analysis_repo.update_chunks.assert_called_once()

View file

@ -39,7 +39,6 @@ class TestBuildFormData:
assert data["do_picture_classification"] == "false" assert data["do_picture_classification"] == "false"
assert data["do_picture_description"] == "false" assert data["do_picture_description"] == "false"
assert data["include_images"] == "false" assert data["include_images"] == "false"
assert data["generate_page_images"] == "false"
assert data["images_scale"] == "1.0" assert data["images_scale"] == "1.0"
assert set(data["to_formats"]) == {"md", "html", "json"} assert set(data["to_formats"]) == {"md", "html", "json"}
@ -56,9 +55,14 @@ class TestBuildFormData:
assert data["images_scale"] == "2.0" assert data["images_scale"] == "2.0"
assert data["include_images"] == "true" assert data["include_images"] == "true"
def test_page_range_included_when_set(self): def test_no_generate_page_images_field(self):
"""generate_page_images is a PdfPipelineOptions field, not a Serve field."""
data = _build_form_data(ConversionOptions())
assert "generate_page_images" not in data
def test_page_range_as_repeated_fields(self):
data = _build_form_data(ConversionOptions(), page_range=(11, 20)) data = _build_form_data(ConversionOptions(), page_range=(11, 20))
assert data["page_range"] == "11-20" assert data["page_range"] == ["11", "20"]
def test_page_range_absent_when_none(self): def test_page_range_absent_when_none(self):
data = _build_form_data(ConversionOptions()) data = _build_form_data(ConversionOptions())
@ -402,11 +406,12 @@ class TestServeConverterConvert:
assert len(result.pages[0].elements) == 1 assert len(result.pages[0].elements) == 1
assert result.pages[0].elements[0].type == "title" assert result.pages[0].elements[0].type == "title"
# Verify form fields sent as dict with list for repeated keys # Verify form fields sent correctly
call_kwargs = mock_client.post.call_args call_kwargs = mock_client.post.call_args
sent_data = call_kwargs.kwargs.get("data", {}) sent_data = call_kwargs.kwargs.get("data", {})
assert sent_data["do_ocr"] == "true" assert sent_data["do_ocr"] == "true"
assert set(sent_data["to_formats"]) == {"md", "html", "json"} assert set(sent_data["to_formats"]) == {"md", "html", "json"}
assert "generate_page_images" not in sent_data
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_http_error_raises(self, tmp_path): async def test_http_error_raises(self, tmp_path):
@ -414,6 +419,8 @@ class TestServeConverterConvert:
test_file.write_bytes(b"%PDF-1.4 fake content") test_file.write_bytes(b"%PDF-1.4 fake content")
mock_response = MagicMock() mock_response = MagicMock()
mock_response.status_code = 500
mock_response.text = "Internal Server Error"
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
"Server Error", "Server Error",
request=MagicMock(), request=MagicMock(),
@ -510,3 +517,17 @@ class TestConverterWiring:
converter = _build_converter() converter = _build_converter()
assert isinstance(converter, ServeConverter) assert isinstance(converter, ServeConverter)
assert converter._api_key == "my-key" assert converter._api_key == "my-key"
def test_remote_engine_builds_chunker(self):
"""Chunker must be available in remote mode (hybrid local chunking)."""
from infra.local_chunker import LocalChunker
from infra.settings import Settings
with patch(
"main.settings",
Settings(conversion_engine="remote", docling_serve_url="http://serve:5001"),
):
from main import _build_chunker
chunker = _build_chunker()
assert isinstance(chunker, LocalChunker)

View file

@ -19,6 +19,7 @@ class TestSettingsDefaults:
assert s.max_page_count == 0 assert s.max_page_count == 0
assert s.max_file_size_mb == 50 assert s.max_file_size_mb == 50
assert s.batch_page_size == 0 assert s.batch_page_size == 0
assert s.opensearch_default_limit == 1000
assert s.upload_dir == "./uploads" assert s.upload_dir == "./uploads"
assert s.db_path == "./data/docling_studio.db" assert s.db_path == "./data/docling_studio.db"
assert "http://localhost:3000" in s.cors_origins assert "http://localhost:3000" in s.cors_origins
@ -103,6 +104,12 @@ class TestSettingsValidation:
with pytest.raises(ValueError, match="lock_timeout must be > 0"): with pytest.raises(ValueError, match="lock_timeout must be > 0"):
Settings(lock_timeout=0) Settings(lock_timeout=0)
def test_zero_opensearch_default_limit_rejected(self):
import pytest
with pytest.raises(ValueError, match="opensearch_default_limit must be >= 1"):
Settings(opensearch_default_limit=0)
def test_invalid_table_mode_rejected(self): def test_invalid_table_mode_rejected(self):
import pytest import pytest
@ -146,6 +153,7 @@ class TestSettingsFromEnv:
monkeypatch.setenv("MAX_PAGE_COUNT", "20") monkeypatch.setenv("MAX_PAGE_COUNT", "20")
monkeypatch.setenv("MAX_FILE_SIZE_MB", "100") monkeypatch.setenv("MAX_FILE_SIZE_MB", "100")
monkeypatch.setenv("BATCH_PAGE_SIZE", "15") monkeypatch.setenv("BATCH_PAGE_SIZE", "15")
monkeypatch.setenv("OPENSEARCH_DEFAULT_LIMIT", "500")
monkeypatch.setenv("UPLOAD_DIR", "/data/uploads") monkeypatch.setenv("UPLOAD_DIR", "/data/uploads")
monkeypatch.setenv("DB_PATH", "/data/test.db") monkeypatch.setenv("DB_PATH", "/data/test.db")
monkeypatch.setenv("CORS_ORIGINS", "http://a.com, http://b.com") monkeypatch.setenv("CORS_ORIGINS", "http://a.com, http://b.com")
@ -163,6 +171,7 @@ class TestSettingsFromEnv:
assert s.max_page_count == 20 assert s.max_page_count == 20
assert s.max_file_size_mb == 100 assert s.max_file_size_mb == 100
assert s.batch_page_size == 15 assert s.batch_page_size == 15
assert s.opensearch_default_limit == 500
assert s.upload_dir == "/data/uploads" assert s.upload_dir == "/data/uploads"
assert s.db_path == "/data/test.db" assert s.db_path == "/data/test.db"
assert s.cors_origins == ["http://a.com", "http://b.com"] assert s.cors_origins == ["http://a.com", "http://b.com"]

View file

@ -29,12 +29,12 @@ describe('useFeatureFlagStore', () => {
expect(store.isEnabled('chunking')).toBe(true) expect(store.isEnabled('chunking')).toBe(true)
}) })
it('disables chunking when engine is remote', async () => { it('enables chunking when engine is remote', async () => {
mockApiFetch.mockResolvedValue({ status: 'ok', engine: 'remote' }) mockApiFetch.mockResolvedValue({ status: 'ok', engine: 'remote' })
const store = useFeatureFlagStore() const store = useFeatureFlagStore()
await store.load() await store.load()
expect(store.engine).toBe('remote') expect(store.engine).toBe('remote')
expect(store.isEnabled('chunking')).toBe(false) expect(store.isEnabled('chunking')).toBe(true)
}) })
it('enables disclaimer when deploymentMode is huggingface', async () => { it('enables disclaimer when deploymentMode is huggingface', async () => {

View file

@ -32,7 +32,7 @@ interface FeatureFlagContext {
const featureRegistry: Record<FeatureFlag, FeatureFlagDef> = { const featureRegistry: Record<FeatureFlag, FeatureFlagDef> = {
chunking: { chunking: {
description: 'Document chunking for RAG preparation', description: 'Document chunking for RAG preparation',
isEnabled: (ctx) => ctx.engine === 'local', isEnabled: (ctx) => ctx.engine !== null,
}, },
disclaimer: { disclaimer: {
description: 'Show shared-instance disclaimer banner', description: 'Show shared-instance disclaimer banner',

View file

@ -23,6 +23,6 @@ describe('useFeatureFlag', () => {
expect(flag.value).toBe(true) expect(flag.value).toBe(true)
store.$patch({ engine: 'remote' }) store.$patch({ engine: 'remote' })
expect(flag.value).toBe(false) expect(flag.value).toBe(true)
}) })
}) })