Add chunking tests and update existing test assertions
16 new chunking tests (domain, schemas, API endpoints). Update existing tests for chunkingOptions parameter and document_json mock.
This commit is contained in:
Pier-Jean Malandrino
parent
a9517d38eb
commit
05de0cc774
4 changed files with 562 additions and 85 deletions
|
|
@ -53,8 +53,12 @@ class TestDocumentEndpoints:
|
|||
@patch("services.document_service.find_by_id", new_callable=AsyncMock)
|
||||
def test_get_document(self, mock_find, client):
|
||||
mock_find.return_value = Document(
|
||||
id="d1", filename="test.pdf", content_type="application/pdf",
|
||||
file_size=2048, page_count=3, storage_path="/tmp/test",
|
||||
id="d1",
|
||||
filename="test.pdf",
|
||||
content_type="application/pdf",
|
||||
file_size=2048,
|
||||
page_count=3,
|
||||
storage_path="/tmp/test",
|
||||
)
|
||||
|
||||
resp = client.get("/api/documents/d1")
|
||||
|
|
@ -74,8 +78,10 @@ class TestDocumentEndpoints:
|
|||
@patch("services.document_service.upload", new_callable=AsyncMock)
|
||||
def test_upload_document(self, mock_upload, client):
|
||||
mock_upload.return_value = Document(
|
||||
id="new-1", filename="uploaded.pdf",
|
||||
content_type="application/pdf", file_size=512,
|
||||
id="new-1",
|
||||
filename="uploaded.pdf",
|
||||
content_type="application/pdf",
|
||||
file_size=512,
|
||||
storage_path="/tmp/uploaded",
|
||||
)
|
||||
|
||||
|
|
@ -115,9 +121,11 @@ class TestDocumentEndpoints:
|
|||
|
||||
class TestAnalysisEndpoints:
|
||||
def test_list_analyses(self, client, mock_analysis_service):
|
||||
mock_analysis_service.find_all = AsyncMock(return_value=[
|
||||
AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf"),
|
||||
])
|
||||
mock_analysis_service.find_all = AsyncMock(
|
||||
return_value=[
|
||||
AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf"),
|
||||
]
|
||||
)
|
||||
|
||||
resp = client.get("/api/analyses")
|
||||
assert resp.status_code == 200
|
||||
|
|
@ -146,37 +154,52 @@ class TestAnalysisEndpoints:
|
|||
assert resp.status_code == 404
|
||||
|
||||
def test_create_analysis(self, client, mock_analysis_service):
|
||||
mock_analysis_service.create = AsyncMock(return_value=AnalysisJob(
|
||||
id="j1", document_id="d1", document_filename="test.pdf",
|
||||
))
|
||||
mock_analysis_service.create = AsyncMock(
|
||||
return_value=AnalysisJob(
|
||||
id="j1",
|
||||
document_id="d1",
|
||||
document_filename="test.pdf",
|
||||
)
|
||||
)
|
||||
|
||||
resp = client.post("/api/analyses", json={"documentId": "d1"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["id"] == "j1"
|
||||
assert data["documentId"] == "d1"
|
||||
mock_analysis_service.create.assert_called_once_with("d1", pipeline_options=None)
|
||||
mock_analysis_service.create.assert_called_once_with(
|
||||
"d1",
|
||||
pipeline_options=None,
|
||||
chunking_options=None,
|
||||
)
|
||||
|
||||
def test_create_analysis_with_pipeline_options(self, client, mock_analysis_service):
|
||||
mock_analysis_service.create = AsyncMock(return_value=AnalysisJob(
|
||||
id="j2", document_id="d1", document_filename="test.pdf",
|
||||
))
|
||||
mock_analysis_service.create = AsyncMock(
|
||||
return_value=AnalysisJob(
|
||||
id="j2",
|
||||
document_id="d1",
|
||||
document_filename="test.pdf",
|
||||
)
|
||||
)
|
||||
|
||||
resp = client.post("/api/analyses", json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {
|
||||
"do_ocr": False,
|
||||
"do_table_structure": True,
|
||||
"table_mode": "fast",
|
||||
"do_code_enrichment": True,
|
||||
"do_formula_enrichment": False,
|
||||
"do_picture_classification": False,
|
||||
"do_picture_description": False,
|
||||
"generate_picture_images": True,
|
||||
"generate_page_images": False,
|
||||
"images_scale": 2.0,
|
||||
}
|
||||
})
|
||||
resp = client.post(
|
||||
"/api/analyses",
|
||||
json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {
|
||||
"do_ocr": False,
|
||||
"do_table_structure": True,
|
||||
"table_mode": "fast",
|
||||
"do_code_enrichment": True,
|
||||
"do_formula_enrichment": False,
|
||||
"do_picture_classification": False,
|
||||
"do_picture_description": False,
|
||||
"generate_picture_images": True,
|
||||
"generate_page_images": False,
|
||||
"images_scale": 2.0,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["id"] == "j2"
|
||||
|
|
@ -191,14 +214,17 @@ class TestAnalysisEndpoints:
|
|||
|
||||
def test_create_analysis_with_partial_pipeline_options(self, client, mock_analysis_service):
|
||||
"""Pipeline options should use defaults for unspecified fields."""
|
||||
mock_analysis_service.create = AsyncMock(return_value=AnalysisJob(
|
||||
id="j3", document_id="d1", document_filename="test.pdf",
|
||||
))
|
||||
mock_analysis_service.create = AsyncMock(
|
||||
return_value=AnalysisJob(
|
||||
id="j3",
|
||||
document_id="d1",
|
||||
document_filename="test.pdf",
|
||||
)
|
||||
)
|
||||
|
||||
resp = client.post("/api/analyses", json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": False}
|
||||
})
|
||||
resp = client.post(
|
||||
"/api/analyses", json={"documentId": "d1", "pipelineOptions": {"do_ocr": False}}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
opts = mock_analysis_service.create.call_args.kwargs["pipeline_options"]
|
||||
|
|
|
|||
292
document-parser/tests/test_chunking.py
Normal file
292
document-parser/tests/test_chunking.py
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
"""Tests for chunking feature — domain, schemas, service, and API endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api.schemas import ChunkingOptionsRequest, ChunkResponse, RechunkRequest
|
||||
from domain.models import AnalysisJob, AnalysisStatus
|
||||
from domain.value_objects import ChunkingOptions, ChunkResult
|
||||
from main import app
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain: value objects
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChunkingOptions:
|
||||
def test_defaults(self):
|
||||
opts = ChunkingOptions()
|
||||
assert opts.chunker_type == "hybrid"
|
||||
assert opts.max_tokens == 512
|
||||
assert opts.merge_peers is True
|
||||
assert opts.repeat_table_header is True
|
||||
|
||||
def test_custom_values(self):
|
||||
opts = ChunkingOptions(chunker_type="hierarchical", max_tokens=256, merge_peers=False)
|
||||
assert opts.chunker_type == "hierarchical"
|
||||
assert opts.max_tokens == 256
|
||||
assert opts.merge_peers is False
|
||||
|
||||
def test_is_default(self):
|
||||
assert ChunkingOptions().is_default()
|
||||
assert not ChunkingOptions(max_tokens=256).is_default()
|
||||
|
||||
|
||||
class TestChunkResult:
|
||||
def test_defaults(self):
|
||||
chunk = ChunkResult(text="hello")
|
||||
assert chunk.text == "hello"
|
||||
assert chunk.headings == []
|
||||
assert chunk.source_page is None
|
||||
assert chunk.token_count == 0
|
||||
|
||||
def test_full_values(self):
|
||||
chunk = ChunkResult(
|
||||
text="content",
|
||||
headings=["Title", "Section"],
|
||||
source_page=3,
|
||||
token_count=42,
|
||||
)
|
||||
assert chunk.headings == ["Title", "Section"]
|
||||
assert chunk.source_page == 3
|
||||
assert chunk.token_count == 42
|
||||
|
||||
def test_serializable(self):
|
||||
chunk = ChunkResult(text="x", headings=["h1"], source_page=1, token_count=10)
|
||||
data = asdict(chunk)
|
||||
assert data == {"text": "x", "headings": ["h1"], "source_page": 1, "token_count": 10}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain: AnalysisJob with chunking fields
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAnalysisJobChunking:
|
||||
def test_default_chunking_fields(self):
|
||||
job = AnalysisJob()
|
||||
assert job.document_json is None
|
||||
assert job.chunks_json is None
|
||||
|
||||
def test_mark_completed_with_chunks(self):
|
||||
job = AnalysisJob()
|
||||
job.mark_running()
|
||||
job.mark_completed(
|
||||
markdown="# Title",
|
||||
html="<h1>Title</h1>",
|
||||
pages_json="[]",
|
||||
document_json='{"name": "doc"}',
|
||||
chunks_json='[{"text": "chunk1"}]',
|
||||
)
|
||||
assert job.status == AnalysisStatus.COMPLETED
|
||||
assert job.document_json == '{"name": "doc"}'
|
||||
assert job.chunks_json == '[{"text": "chunk1"}]'
|
||||
|
||||
def test_mark_completed_without_chunks(self):
|
||||
job = AnalysisJob()
|
||||
job.mark_running()
|
||||
job.mark_completed(markdown="md", html="html", pages_json="[]")
|
||||
assert job.document_json is None
|
||||
assert job.chunks_json is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChunkingOptionsRequest:
|
||||
def test_defaults(self):
|
||||
opts = ChunkingOptionsRequest()
|
||||
assert opts.chunker_type == "hybrid"
|
||||
assert opts.max_tokens == 512
|
||||
assert opts.merge_peers is True
|
||||
assert opts.repeat_table_header is True
|
||||
|
||||
def test_custom_values(self):
|
||||
opts = ChunkingOptionsRequest(chunker_type="hierarchical", max_tokens=1024)
|
||||
assert opts.chunker_type == "hierarchical"
|
||||
assert opts.max_tokens == 1024
|
||||
|
||||
def test_invalid_chunker_type(self):
|
||||
with pytest.raises(ValueError, match="chunker_type"):
|
||||
ChunkingOptionsRequest(chunker_type="invalid")
|
||||
|
||||
def test_max_tokens_too_low(self):
|
||||
with pytest.raises(ValueError, match="max_tokens"):
|
||||
ChunkingOptionsRequest(max_tokens=10)
|
||||
|
||||
def test_max_tokens_too_high(self):
|
||||
with pytest.raises(ValueError, match="max_tokens"):
|
||||
ChunkingOptionsRequest(max_tokens=10000)
|
||||
|
||||
def test_boundary_max_tokens(self):
|
||||
opts_low = ChunkingOptionsRequest(max_tokens=64)
|
||||
assert opts_low.max_tokens == 64
|
||||
opts_high = ChunkingOptionsRequest(max_tokens=8192)
|
||||
assert opts_high.max_tokens == 8192
|
||||
|
||||
|
||||
class TestChunkResponse:
|
||||
def test_serializes_to_camel_case(self):
|
||||
resp = ChunkResponse(text="hello", headings=["H1"], source_page=1, token_count=5)
|
||||
data = resp.model_dump(by_alias=True)
|
||||
assert "sourcePage" in data
|
||||
assert "tokenCount" in data
|
||||
assert data["text"] == "hello"
|
||||
|
||||
|
||||
class TestRechunkRequest:
|
||||
def test_parses(self):
|
||||
req = RechunkRequest(chunkingOptions=ChunkingOptionsRequest(max_tokens=256))
|
||||
assert req.chunkingOptions.max_tokens == 256
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_analysis_service(client):
|
||||
mock_svc = MagicMock()
|
||||
original = getattr(app.state, "analysis_service", None)
|
||||
app.state.analysis_service = mock_svc
|
||||
yield mock_svc
|
||||
app.state.analysis_service = original
|
||||
|
||||
|
||||
class TestCreateAnalysisWithChunking:
|
||||
def test_create_with_chunking_options(self, client, mock_analysis_service):
|
||||
mock_analysis_service.create = AsyncMock(
|
||||
return_value=AnalysisJob(
|
||||
id="j1",
|
||||
document_id="d1",
|
||||
document_filename="test.pdf",
|
||||
)
|
||||
)
|
||||
|
||||
resp = client.post(
|
||||
"/api/analyses",
|
||||
json={
|
||||
"documentId": "d1",
|
||||
"chunkingOptions": {
|
||||
"chunker_type": "hybrid",
|
||||
"max_tokens": 256,
|
||||
"merge_peers": False,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
call_kwargs = mock_analysis_service.create.call_args
|
||||
chunking = call_kwargs.kwargs["chunking_options"]
|
||||
assert chunking["chunker_type"] == "hybrid"
|
||||
assert chunking["max_tokens"] == 256
|
||||
assert chunking["merge_peers"] is False
|
||||
|
||||
def test_create_without_chunking_options(self, client, mock_analysis_service):
|
||||
mock_analysis_service.create = AsyncMock(
|
||||
return_value=AnalysisJob(
|
||||
id="j1",
|
||||
document_id="d1",
|
||||
document_filename="test.pdf",
|
||||
)
|
||||
)
|
||||
|
||||
resp = client.post("/api/analyses", json={"documentId": "d1"})
|
||||
assert resp.status_code == 200
|
||||
|
||||
call_kwargs = mock_analysis_service.create.call_args
|
||||
assert call_kwargs.kwargs["chunking_options"] is None
|
||||
|
||||
def test_response_includes_chunking_fields(self, client, mock_analysis_service):
|
||||
job = AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf")
|
||||
job.mark_running()
|
||||
job.mark_completed(
|
||||
markdown="# Title",
|
||||
html="<h1>Title</h1>",
|
||||
pages_json="[]",
|
||||
document_json='{"name": "doc"}',
|
||||
chunks_json=json.dumps([asdict(ChunkResult(text="chunk1", token_count=5))]),
|
||||
)
|
||||
mock_analysis_service.find_by_id = AsyncMock(return_value=job)
|
||||
|
||||
resp = client.get("/api/analyses/j1")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["hasDocumentJson"] is True
|
||||
assert data["chunksJson"] is not None
|
||||
chunks = json.loads(data["chunksJson"])
|
||||
assert len(chunks) == 1
|
||||
assert chunks[0]["text"] == "chunk1"
|
||||
|
||||
|
||||
class TestRechunkEndpoint:
|
||||
def test_rechunk_success(self, client, mock_analysis_service):
|
||||
mock_analysis_service.rechunk = AsyncMock(
|
||||
return_value=[
|
||||
ChunkResult(text="chunk1", headings=["H1"], source_page=1, token_count=10),
|
||||
ChunkResult(text="chunk2", headings=["H1", "H2"], source_page=2, token_count=20),
|
||||
]
|
||||
)
|
||||
|
||||
resp = client.post(
|
||||
"/api/analyses/j1/rechunk",
|
||||
json={
|
||||
"chunkingOptions": {"chunker_type": "hybrid", "max_tokens": 128},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data) == 2
|
||||
assert data[0]["text"] == "chunk1"
|
||||
assert data[0]["sourcePage"] == 1
|
||||
assert data[0]["tokenCount"] == 10
|
||||
assert data[1]["headings"] == ["H1", "H2"]
|
||||
|
||||
def test_rechunk_not_completed(self, client, mock_analysis_service):
|
||||
mock_analysis_service.rechunk = AsyncMock(
|
||||
side_effect=ValueError("Analysis is not completed: j1"),
|
||||
)
|
||||
|
||||
resp = client.post(
|
||||
"/api/analyses/j1/rechunk",
|
||||
json={
|
||||
"chunkingOptions": {"chunker_type": "hybrid"},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_rechunk_no_document_json(self, client, mock_analysis_service):
|
||||
mock_analysis_service.rechunk = AsyncMock(
|
||||
side_effect=ValueError("No document data available for re-chunking: j1"),
|
||||
)
|
||||
|
||||
resp = client.post(
|
||||
"/api/analyses/j1/rechunk",
|
||||
json={
|
||||
"chunkingOptions": {"chunker_type": "hierarchical"},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_rechunk_invalid_chunker_type(self, client, mock_analysis_service):
|
||||
resp = client.post(
|
||||
"/api/analyses/j1/rechunk",
|
||||
json={
|
||||
"chunkingOptions": {"chunker_type": "invalid"},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
|
@ -23,6 +23,7 @@ from infra.local_converter import (
|
|||
# build_converter — verifies Docling pipeline options are wired correctly
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildConverter:
|
||||
"""Verify that build_converter produces a DocumentConverter with the right PdfPipelineOptions."""
|
||||
|
||||
|
|
@ -101,18 +102,20 @@ class TestBuildConverter:
|
|||
assert opts.images_scale == 2.0
|
||||
|
||||
def test_all_options_combined(self):
|
||||
conv = build_converter(ConversionOptions(
|
||||
do_ocr=False,
|
||||
do_table_structure=True,
|
||||
table_mode="fast",
|
||||
do_code_enrichment=True,
|
||||
do_formula_enrichment=True,
|
||||
do_picture_classification=True,
|
||||
do_picture_description=True,
|
||||
generate_picture_images=True,
|
||||
generate_page_images=True,
|
||||
images_scale=1.5,
|
||||
))
|
||||
conv = build_converter(
|
||||
ConversionOptions(
|
||||
do_ocr=False,
|
||||
do_table_structure=True,
|
||||
table_mode="fast",
|
||||
do_code_enrichment=True,
|
||||
do_formula_enrichment=True,
|
||||
do_picture_classification=True,
|
||||
do_picture_description=True,
|
||||
generate_picture_images=True,
|
||||
generate_page_images=True,
|
||||
images_scale=1.5,
|
||||
)
|
||||
)
|
||||
opts = self._get_pipeline_options(conv)
|
||||
assert opts.do_ocr is False
|
||||
assert opts.do_table_structure is True
|
||||
|
|
@ -130,6 +133,7 @@ class TestBuildConverter:
|
|||
# convert_document — default vs custom converter routing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConvertDocumentRouting:
|
||||
"""Verify convert_document uses default converter for default opts, custom otherwise."""
|
||||
|
||||
|
|
@ -142,6 +146,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_get_default.return_value = mock_conv
|
||||
|
||||
|
|
@ -159,6 +164,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -176,6 +182,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -193,6 +200,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -210,6 +218,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -226,6 +235,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -242,6 +252,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -258,6 +269,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -275,6 +287,7 @@ class TestConvertDocumentRouting:
|
|||
mock_result.document.iterate_items.return_value = []
|
||||
mock_result.document.export_to_markdown.return_value = ""
|
||||
mock_result.document.export_to_html.return_value = ""
|
||||
mock_result.document.export_to_dict.return_value = {}
|
||||
mock_conv.convert.return_value = mock_result
|
||||
mock_build.return_value = mock_conv
|
||||
|
||||
|
|
@ -299,30 +312,37 @@ class TestConvertDocumentRouting:
|
|||
# Service layer — pipeline options forwarding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestServiceForwardsPipelineOptions:
|
||||
"""Verify analysis_service.create and _run_analysis forward pipeline options."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_doc(self):
|
||||
from domain.models import Document
|
||||
|
||||
return Document(id="d1", filename="test.pdf", storage_path="/tmp/test.pdf")
|
||||
|
||||
@pytest.fixture
|
||||
def mock_job(self):
|
||||
from domain.models import AnalysisJob
|
||||
|
||||
return AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf")
|
||||
|
||||
@patch("services.analysis_service.document_repo")
|
||||
@patch("services.analysis_service.analysis_repo")
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_passes_pipeline_options_to_run(
|
||||
self, mock_analysis_repo, mock_doc_repo, mock_doc,
|
||||
self,
|
||||
mock_analysis_repo,
|
||||
mock_doc_repo,
|
||||
mock_doc,
|
||||
):
|
||||
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
|
||||
mock_analysis_repo.insert = AsyncMock()
|
||||
|
||||
mock_converter = AsyncMock()
|
||||
from services.analysis_service import AnalysisService
|
||||
|
||||
svc = AnalysisService(converter=mock_converter)
|
||||
|
||||
opts = {"do_ocr": False, "table_mode": "fast"}
|
||||
|
|
@ -335,13 +355,17 @@ class TestServiceForwardsPipelineOptions:
|
|||
@patch("services.analysis_service.analysis_repo")
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_passes_none_when_no_options(
|
||||
self, mock_analysis_repo, mock_doc_repo, mock_doc,
|
||||
self,
|
||||
mock_analysis_repo,
|
||||
mock_doc_repo,
|
||||
mock_doc,
|
||||
):
|
||||
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
|
||||
mock_analysis_repo.insert = AsyncMock()
|
||||
|
||||
mock_converter = AsyncMock()
|
||||
from services.analysis_service import AnalysisService
|
||||
|
||||
svc = AnalysisService(converter=mock_converter)
|
||||
|
||||
with patch("services.analysis_service.asyncio.create_task") as mock_task:
|
||||
|
|
@ -352,7 +376,10 @@ class TestServiceForwardsPipelineOptions:
|
|||
@patch("services.analysis_service.document_repo")
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_analysis_forwards_options_to_convert(
|
||||
self, mock_doc_repo, mock_analysis_repo, mock_job,
|
||||
self,
|
||||
mock_doc_repo,
|
||||
mock_analysis_repo,
|
||||
mock_job,
|
||||
):
|
||||
from domain.value_objects import ConversionResult, PageDetail
|
||||
|
||||
|
|
@ -369,6 +396,7 @@ class TestServiceForwardsPipelineOptions:
|
|||
)
|
||||
|
||||
from services.analysis_service import AnalysisService
|
||||
|
||||
svc = AnalysisService(converter=mock_converter)
|
||||
|
||||
opts = {
|
||||
|
|
@ -399,7 +427,10 @@ class TestServiceForwardsPipelineOptions:
|
|||
@patch("services.analysis_service.document_repo")
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_analysis_uses_defaults_when_no_options(
|
||||
self, mock_doc_repo, mock_analysis_repo, mock_job,
|
||||
self,
|
||||
mock_doc_repo,
|
||||
mock_analysis_repo,
|
||||
mock_job,
|
||||
):
|
||||
from domain.value_objects import ConversionResult, PageDetail
|
||||
|
||||
|
|
@ -416,6 +447,7 @@ class TestServiceForwardsPipelineOptions:
|
|||
)
|
||||
|
||||
from services.analysis_service import AnalysisService
|
||||
|
||||
svc = AnalysisService(converter=mock_converter)
|
||||
|
||||
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", None)
|
||||
|
|
@ -429,7 +461,10 @@ class TestServiceForwardsPipelineOptions:
|
|||
@patch("services.analysis_service.document_repo")
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_analysis_marks_failed_on_error(
|
||||
self, mock_doc_repo, mock_analysis_repo, mock_job,
|
||||
self,
|
||||
mock_doc_repo,
|
||||
mock_analysis_repo,
|
||||
mock_job,
|
||||
):
|
||||
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
||||
mock_analysis_repo.update_status = AsyncMock()
|
||||
|
|
@ -438,6 +473,7 @@ class TestServiceForwardsPipelineOptions:
|
|||
mock_converter.convert.side_effect = RuntimeError("Docling crashed")
|
||||
|
||||
from services.analysis_service import AnalysisService
|
||||
|
||||
svc = AnalysisService(converter=mock_converter)
|
||||
|
||||
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", {"do_ocr": False})
|
||||
|
|
@ -453,6 +489,7 @@ class TestServiceForwardsPipelineOptions:
|
|||
# API endpoint — full request/response with pipeline options
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAnalysisEndpointPipelineOptions:
|
||||
"""Integration-level tests for the analysis creation endpoint with pipeline options."""
|
||||
|
||||
|
|
@ -461,6 +498,7 @@ class TestAnalysisEndpointPipelineOptions:
|
|||
from fastapi.testclient import TestClient
|
||||
|
||||
from main import app
|
||||
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -468,6 +506,7 @@ class TestAnalysisEndpointPipelineOptions:
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
from main import app
|
||||
|
||||
mock = MagicMock()
|
||||
original = getattr(app.state, "analysis_service", None)
|
||||
app.state.analysis_service = mock
|
||||
|
|
@ -476,20 +515,25 @@ class TestAnalysisEndpointPipelineOptions:
|
|||
|
||||
def test_no_pipeline_options_sends_none(self, client, mock_svc):
|
||||
from domain.models import AnalysisJob
|
||||
|
||||
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||
|
||||
client.post("/api/analyses", json={"documentId": "d1"})
|
||||
|
||||
mock_svc.create.assert_called_once_with("d1", pipeline_options=None)
|
||||
mock_svc.create.assert_called_once_with("d1", pipeline_options=None, chunking_options=None)
|
||||
|
||||
def test_empty_pipeline_options_object_uses_defaults(self, client, mock_svc):
|
||||
from domain.models import AnalysisJob
|
||||
|
||||
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||
|
||||
client.post("/api/analyses", json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {},
|
||||
})
|
||||
client.post(
|
||||
"/api/analyses",
|
||||
json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {},
|
||||
},
|
||||
)
|
||||
|
||||
opts = mock_svc.create.call_args.kwargs["pipeline_options"]
|
||||
assert opts["do_ocr"] is True
|
||||
|
|
@ -501,12 +545,16 @@ class TestAnalysisEndpointPipelineOptions:
|
|||
|
||||
def test_partial_pipeline_options_merges_with_defaults(self, client, mock_svc):
|
||||
from domain.models import AnalysisJob
|
||||
|
||||
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||
|
||||
client.post("/api/analyses", json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": False, "images_scale": 1.5},
|
||||
})
|
||||
client.post(
|
||||
"/api/analyses",
|
||||
json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": False, "images_scale": 1.5},
|
||||
},
|
||||
)
|
||||
|
||||
opts = mock_svc.create.call_args.kwargs["pipeline_options"]
|
||||
assert opts["do_ocr"] is False
|
||||
|
|
@ -522,6 +570,7 @@ class TestAnalysisEndpointPipelineOptions:
|
|||
|
||||
def test_full_pipeline_options(self, client, mock_svc):
|
||||
from domain.models import AnalysisJob
|
||||
|
||||
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||
|
||||
payload = {
|
||||
|
|
@ -547,18 +596,25 @@ class TestAnalysisEndpointPipelineOptions:
|
|||
assert opts == payload["pipelineOptions"]
|
||||
|
||||
def test_invalid_pipeline_option_type_rejected(self, client, mock_svc):
|
||||
resp = client.post("/api/analyses", json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": "not-a-bool"},
|
||||
})
|
||||
resp = client.post(
|
||||
"/api/analyses",
|
||||
json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": "not-a-bool"},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
def test_unknown_pipeline_option_ignored(self, client, mock_svc):
|
||||
from domain.models import AnalysisJob
|
||||
|
||||
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||
|
||||
resp = client.post("/api/analyses", json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": True, "unknown_field": True},
|
||||
})
|
||||
resp = client.post(
|
||||
"/api/analyses",
|
||||
json={
|
||||
"documentId": "d1",
|
||||
"pipelineOptions": {"do_ocr": True, "unknown_field": True},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from infra.serve_converter import (
|
|||
# Unit tests — form data building
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildFormData:
|
||||
def test_default_options(self):
|
||||
data = _build_form_data(ConversionOptions())
|
||||
|
|
@ -36,7 +37,9 @@ class TestBuildFormData:
|
|||
|
||||
def test_custom_options(self):
|
||||
opts = ConversionOptions(
|
||||
do_ocr=False, table_mode="fast", images_scale=2.0,
|
||||
do_ocr=False,
|
||||
table_mode="fast",
|
||||
images_scale=2.0,
|
||||
generate_picture_images=True,
|
||||
)
|
||||
data = _build_form_data(opts)
|
||||
|
|
@ -50,6 +53,7 @@ class TestBuildFormData:
|
|||
# Unit tests — response parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseResponse:
|
||||
def test_minimal_response(self):
|
||||
data = {
|
||||
|
|
@ -82,12 +86,34 @@ class TestParseResponse:
|
|||
{
|
||||
"label": "title",
|
||||
"text": "Title",
|
||||
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40, "coord_origin": "TOPLEFT"}}],
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 10,
|
||||
"t": 20,
|
||||
"r": 200,
|
||||
"b": 40,
|
||||
"coord_origin": "TOPLEFT",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"label": "paragraph",
|
||||
"text": "Text",
|
||||
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 50, "r": 200, "b": 70, "coord_origin": "TOPLEFT"}}],
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 10,
|
||||
"t": 50,
|
||||
"r": 200,
|
||||
"b": 70,
|
||||
"coord_origin": "TOPLEFT",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"tables": [],
|
||||
|
|
@ -112,7 +138,9 @@ class TestParseResponse:
|
|||
"1": {"size": {"width": 612.0, "height": 792.0}},
|
||||
"2": {"size": {"width": 595.0, "height": 842.0}},
|
||||
},
|
||||
"texts": [], "tables": [], "pictures": [],
|
||||
"texts": [],
|
||||
"tables": [],
|
||||
"pictures": [],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -135,7 +163,9 @@ class TestParseResponse:
|
|||
def test_json_content_as_string(self):
|
||||
json_doc = {
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
"texts": [], "tables": [], "pictures": [],
|
||||
"texts": [],
|
||||
"tables": [],
|
||||
"pictures": [],
|
||||
}
|
||||
data = {
|
||||
"document": {
|
||||
|
|
@ -171,10 +201,40 @@ class TestParseResponse:
|
|||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
"texts": [],
|
||||
"tables": [
|
||||
{"label": "table", "text": "", "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 10, "r": 300, "b": 200, "coord_origin": "TOPLEFT"}}]},
|
||||
{
|
||||
"label": "table",
|
||||
"text": "",
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 10,
|
||||
"t": 10,
|
||||
"r": 300,
|
||||
"b": 200,
|
||||
"coord_origin": "TOPLEFT",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"pictures": [
|
||||
{"label": "picture", "text": "", "prov": [{"page_no": 1, "bbox": {"l": 50, "t": 300, "r": 250, "b": 500, "coord_origin": "TOPLEFT"}}]},
|
||||
{
|
||||
"label": "picture",
|
||||
"text": "",
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 50,
|
||||
"t": 300,
|
||||
"r": 250,
|
||||
"b": 500,
|
||||
"coord_origin": "TOPLEFT",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
|
@ -189,14 +249,19 @@ class TestParseResponse:
|
|||
# Unit tests — bbox extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractBbox:
|
||||
def test_topleft_passthrough(self):
|
||||
bbox = _extract_bbox({"l": 10, "t": 20, "r": 100, "b": 50, "coord_origin": "TOPLEFT"}, 792.0)
|
||||
bbox = _extract_bbox(
|
||||
{"l": 10, "t": 20, "r": 100, "b": 50, "coord_origin": "TOPLEFT"}, 792.0
|
||||
)
|
||||
assert bbox == [10, 20, 100, 50]
|
||||
|
||||
def test_bottomleft_conversion(self):
|
||||
# In BOTTOMLEFT: t (top of box) has higher y than b (bottom of box)
|
||||
bbox = _extract_bbox({"l": 10, "t": 772, "r": 100, "b": 742, "coord_origin": "BOTTOMLEFT"}, 792.0)
|
||||
bbox = _extract_bbox(
|
||||
{"l": 10, "t": 772, "r": 100, "b": 742, "coord_origin": "BOTTOMLEFT"}, 792.0
|
||||
)
|
||||
# new_top = 792 - 772 = 20, new_bottom = 792 - 742 = 50
|
||||
assert bbox == [10, 20, 100, 50]
|
||||
|
||||
|
|
@ -217,9 +282,11 @@ class TestExtractBbox:
|
|||
# Unit tests — label mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLabelMapping:
|
||||
def test_known_labels(self):
|
||||
from infra.serve_converter import _LABEL_MAP
|
||||
|
||||
assert _LABEL_MAP["table"] == "table"
|
||||
assert _LABEL_MAP["picture"] == "picture"
|
||||
assert _LABEL_MAP["figure"] == "picture"
|
||||
|
|
@ -232,6 +299,7 @@ class TestLabelMapping:
|
|||
|
||||
def test_unknown_label_defaults_to_text(self):
|
||||
from infra.serve_converter import _LABEL_MAP
|
||||
|
||||
assert _LABEL_MAP.get("unknown_thing", "text") == "text"
|
||||
|
||||
|
||||
|
|
@ -239,6 +307,7 @@ class TestLabelMapping:
|
|||
# Unit tests — ServeConverter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestServeConverter:
|
||||
def test_headers_with_api_key(self):
|
||||
conv = ServeConverter(base_url="http://localhost:5001", api_key="secret")
|
||||
|
|
@ -257,6 +326,7 @@ class TestServeConverter:
|
|||
# Integration tests — HTTP calls (mocked)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestServeConverterConvert:
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_conversion(self, tmp_path):
|
||||
|
|
@ -270,7 +340,22 @@ class TestServeConverterConvert:
|
|||
"json_content": {
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||
"texts": [
|
||||
{"label": "title", "text": "Converted", "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40, "coord_origin": "TOPLEFT"}}]},
|
||||
{
|
||||
"label": "title",
|
||||
"text": "Converted",
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 10,
|
||||
"t": 20,
|
||||
"r": 200,
|
||||
"b": 40,
|
||||
"coord_origin": "TOPLEFT",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"tables": [],
|
||||
"pictures": [],
|
||||
|
|
@ -312,7 +397,9 @@ class TestServeConverterConvert:
|
|||
|
||||
mock_response = MagicMock()
|
||||
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||
"Server Error", request=MagicMock(), response=MagicMock(status_code=500),
|
||||
"Server Error",
|
||||
request=MagicMock(),
|
||||
response=MagicMock(status_code=500),
|
||||
)
|
||||
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -322,8 +409,10 @@ class TestServeConverterConvert:
|
|||
|
||||
conv = ServeConverter(base_url="http://localhost:5001")
|
||||
|
||||
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client), \
|
||||
pytest.raises(httpx.HTTPStatusError):
|
||||
with (
|
||||
patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client),
|
||||
pytest.raises(httpx.HTTPStatusError),
|
||||
):
|
||||
await conv.convert(str(test_file), ConversionOptions())
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -358,6 +447,7 @@ class TestServeConverterConvert:
|
|||
# Integration — converter wiring in main.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConverterWiring:
|
||||
def test_local_engine_builds_local_converter(self):
|
||||
from infra.local_converter import LocalConverter
|
||||
|
|
@ -365,14 +455,19 @@ class TestConverterWiring:
|
|||
|
||||
with patch("main.settings", Settings(conversion_engine="local")):
|
||||
from main import _build_converter
|
||||
|
||||
converter = _build_converter()
|
||||
assert isinstance(converter, LocalConverter)
|
||||
|
||||
def test_remote_engine_builds_serve_converter(self):
|
||||
from infra.settings import Settings
|
||||
|
||||
with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001")):
|
||||
with patch(
|
||||
"main.settings",
|
||||
Settings(conversion_engine="remote", docling_serve_url="http://serve:5001"),
|
||||
):
|
||||
from main import _build_converter
|
||||
|
||||
converter = _build_converter()
|
||||
assert isinstance(converter, ServeConverter)
|
||||
assert converter._base_url == "http://serve:5001"
|
||||
|
|
@ -380,8 +475,16 @@ class TestConverterWiring:
|
|||
def test_remote_engine_passes_api_key(self):
|
||||
from infra.settings import Settings
|
||||
|
||||
with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001", docling_serve_api_key="my-key")):
|
||||
with patch(
|
||||
"main.settings",
|
||||
Settings(
|
||||
conversion_engine="remote",
|
||||
docling_serve_url="http://serve:5001",
|
||||
docling_serve_api_key="my-key",
|
||||
),
|
||||
):
|
||||
from main import _build_converter
|
||||
|
||||
converter = _build_converter()
|
||||
assert isinstance(converter, ServeConverter)
|
||||
assert converter._api_key == "my-key"
|
||||
|
|
|
|||
Loading…
Reference in a new issue