Merge pull request #28 from scub-france/fix/zombie-jobs-and-json-parse
Add Docling - serve
This commit is contained in:
commit
f190b39bec
21 changed files with 1237 additions and 356 deletions
|
|
@ -3,16 +3,24 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
|
|
||||||
from api.schemas import AnalysisResponse, CreateAnalysisRequest
|
from api.schemas import AnalysisResponse, CreateAnalysisRequest
|
||||||
from services import analysis_service
|
from services.analysis_service import AnalysisService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
router = APIRouter(prefix="/api/analyses", tags=["analyses"])
|
router = APIRouter(prefix="/api/analyses", tags=["analyses"])
|
||||||
|
|
||||||
|
|
||||||
|
def _get_service(request: Request) -> AnalysisService:
|
||||||
|
return request.app.state.analysis_service
|
||||||
|
|
||||||
|
|
||||||
|
ServiceDep = Annotated[AnalysisService, Depends(_get_service)]
|
||||||
|
|
||||||
|
|
||||||
def _to_response(job) -> AnalysisResponse:
|
def _to_response(job) -> AnalysisResponse:
|
||||||
return AnalysisResponse(
|
return AnalysisResponse(
|
||||||
id=job.id,
|
id=job.id,
|
||||||
|
|
@ -30,7 +38,7 @@ def _to_response(job) -> AnalysisResponse:
|
||||||
|
|
||||||
|
|
||||||
@router.post("", response_model=AnalysisResponse)
|
@router.post("", response_model=AnalysisResponse)
|
||||||
async def create_analysis(body: CreateAnalysisRequest):
|
async def create_analysis(body: CreateAnalysisRequest, service: ServiceDep):
|
||||||
"""Create a new analysis job for a document."""
|
"""Create a new analysis job for a document."""
|
||||||
if not body.documentId or not body.documentId.strip():
|
if not body.documentId or not body.documentId.strip():
|
||||||
raise HTTPException(status_code=400, detail="documentId is required")
|
raise HTTPException(status_code=400, detail="documentId is required")
|
||||||
|
|
@ -40,7 +48,7 @@ async def create_analysis(body: CreateAnalysisRequest):
|
||||||
pipeline_opts = body.pipelineOptions.model_dump()
|
pipeline_opts = body.pipelineOptions.model_dump()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
job = await analysis_service.create(body.documentId, pipeline_options=pipeline_opts)
|
job = await service.create(body.documentId, pipeline_options=pipeline_opts)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||||
|
|
||||||
|
|
@ -48,24 +56,24 @@ async def create_analysis(body: CreateAnalysisRequest):
|
||||||
|
|
||||||
|
|
||||||
@router.get("", response_model=list[AnalysisResponse])
|
@router.get("", response_model=list[AnalysisResponse])
|
||||||
async def list_analyses():
|
async def list_analyses(service: ServiceDep):
|
||||||
"""List all analysis jobs."""
|
"""List all analysis jobs."""
|
||||||
jobs = await analysis_service.find_all()
|
jobs = await service.find_all()
|
||||||
return [_to_response(j) for j in jobs]
|
return [_to_response(j) for j in jobs]
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{job_id}", response_model=AnalysisResponse)
|
@router.get("/{job_id}", response_model=AnalysisResponse)
|
||||||
async def get_analysis(job_id: str):
|
async def get_analysis(job_id: str, service: ServiceDep):
|
||||||
"""Get a single analysis job."""
|
"""Get a single analysis job."""
|
||||||
job = await analysis_service.find_by_id(job_id)
|
job = await service.find_by_id(job_id)
|
||||||
if not job:
|
if not job:
|
||||||
raise HTTPException(status_code=404, detail="Analysis not found")
|
raise HTTPException(status_code=404, detail="Analysis not found")
|
||||||
return _to_response(job)
|
return _to_response(job)
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{job_id}", status_code=204)
|
@router.delete("/{job_id}", status_code=204)
|
||||||
async def delete_analysis(job_id: str):
|
async def delete_analysis(job_id: str, service: ServiceDep):
|
||||||
"""Delete an analysis job."""
|
"""Delete an analysis job."""
|
||||||
deleted = await analysis_service.delete(job_id)
|
deleted = await service.delete(job_id)
|
||||||
if not deleted:
|
if not deleted:
|
||||||
raise HTTPException(status_code=404, detail="Analysis not found")
|
raise HTTPException(status_code=404, detail="Analysis not found")
|
||||||
|
|
|
||||||
24
document-parser/domain/ports.py
Normal file
24
document-parser/domain/ports.py
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
"""Domain ports — abstract interfaces that infrastructure must implement.
|
||||||
|
|
||||||
|
These protocols define what the domain NEEDS, not how it's done.
|
||||||
|
Infrastructure adapters (local Docling, Docling Serve, etc.) implement these.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING, Protocol
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from domain.value_objects import ConversionOptions, ConversionResult
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentConverter(Protocol):
|
||||||
|
"""Port for document conversion.
|
||||||
|
|
||||||
|
Any implementation (local Docling lib, remote Docling Serve, mock, etc.)
|
||||||
|
must satisfy this contract.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def convert(
|
||||||
|
self, file_path: str, options: ConversionOptions,
|
||||||
|
) -> ConversionResult: ...
|
||||||
51
document-parser/domain/value_objects.py
Normal file
51
document-parser/domain/value_objects.py
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
"""Domain value objects — pure data structures for document conversion.
|
||||||
|
|
||||||
|
These types define the contract between the domain and infrastructure layers.
|
||||||
|
They have ZERO external dependencies (no docling, no HTTP, no DB).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PageElement:
|
||||||
|
type: str
|
||||||
|
bbox: list[float]
|
||||||
|
content: str
|
||||||
|
level: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PageDetail:
|
||||||
|
page_number: int
|
||||||
|
width: float
|
||||||
|
height: float
|
||||||
|
elements: list[PageElement] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ConversionOptions:
|
||||||
|
do_ocr: bool = True
|
||||||
|
do_table_structure: bool = True
|
||||||
|
table_mode: str = "accurate"
|
||||||
|
do_code_enrichment: bool = False
|
||||||
|
do_formula_enrichment: bool = False
|
||||||
|
do_picture_classification: bool = False
|
||||||
|
do_picture_description: bool = False
|
||||||
|
generate_picture_images: bool = False
|
||||||
|
generate_page_images: bool = False
|
||||||
|
images_scale: float = 1.0
|
||||||
|
|
||||||
|
def is_default(self) -> bool:
|
||||||
|
return self == ConversionOptions()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ConversionResult:
|
||||||
|
page_count: int
|
||||||
|
content_markdown: str
|
||||||
|
content_html: str
|
||||||
|
pages: list[PageDetail]
|
||||||
|
skipped_items: int = 0
|
||||||
0
document-parser/infra/__init__.py
Normal file
0
document-parser/infra/__init__.py
Normal file
|
|
@ -1,15 +1,16 @@
|
||||||
"""Docling document extraction logic — pure domain, no HTTP concerns.
|
"""Local Docling converter — runs Docling as a Python library in-process.
|
||||||
|
|
||||||
Wraps the Docling library to convert documents and extract structured
|
This adapter implements the DocumentConverter port using the Docling library
|
||||||
per-page elements with bounding boxes and hierarchy levels.
|
directly. It wraps the blocking DocumentConverter in asyncio.to_thread for
|
||||||
|
non-blocking execution.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import contextlib
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
from dataclasses import dataclass, field
|
|
||||||
|
|
||||||
from docling.datamodel.base_models import InputFormat
|
from docling.datamodel.base_models import InputFormat
|
||||||
from docling.datamodel.pipeline_options import (
|
from docling.datamodel.pipeline_options import (
|
||||||
|
|
@ -17,7 +18,8 @@ from docling.datamodel.pipeline_options import (
|
||||||
TableFormerMode,
|
TableFormerMode,
|
||||||
TableStructureOptions,
|
TableStructureOptions,
|
||||||
)
|
)
|
||||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
from docling.document_converter import DocumentConverter as DoclingConverter
|
||||||
|
from docling.document_converter import PdfFormatOption
|
||||||
from docling_core.types.doc import (
|
from docling_core.types.doc import (
|
||||||
CodeItem,
|
CodeItem,
|
||||||
DocItem,
|
DocItem,
|
||||||
|
|
@ -33,10 +35,16 @@ from docling_core.types.doc import (
|
||||||
)
|
)
|
||||||
|
|
||||||
from domain.bbox import to_topleft_list
|
from domain.bbox import to_topleft_list
|
||||||
|
from domain.value_objects import (
|
||||||
|
ConversionOptions,
|
||||||
|
ConversionResult,
|
||||||
|
PageDetail,
|
||||||
|
PageElement,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Thread lock — DocumentConverter is not thread-safe
|
# Thread lock — DoclingConverter is not thread-safe
|
||||||
_converter_lock = threading.Lock()
|
_converter_lock = threading.Lock()
|
||||||
|
|
||||||
# US Letter page dimensions (points) — fallback when page size is unknown
|
# US Letter page dimensions (points) — fallback when page size is unknown
|
||||||
|
|
@ -44,61 +52,13 @@ _DEFAULT_PAGE_WIDTH = 612.0
|
||||||
_DEFAULT_PAGE_HEIGHT = 792.0
|
_DEFAULT_PAGE_HEIGHT = 792.0
|
||||||
|
|
||||||
# Default converter (lazy-init on first request)
|
# Default converter (lazy-init on first request)
|
||||||
_default_converter: DocumentConverter | None = None
|
_default_converter: DoclingConverter | None = None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Domain value objects
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class PageElement:
|
|
||||||
type: str
|
|
||||||
bbox: list[float]
|
|
||||||
content: str
|
|
||||||
level: int = 0
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class PageDetail:
|
|
||||||
page_number: int
|
|
||||||
width: float
|
|
||||||
height: float
|
|
||||||
elements: list[PageElement] = field(default_factory=list)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ConversionOptions:
|
|
||||||
do_ocr: bool = True
|
|
||||||
do_table_structure: bool = True
|
|
||||||
table_mode: str = "accurate"
|
|
||||||
do_code_enrichment: bool = False
|
|
||||||
do_formula_enrichment: bool = False
|
|
||||||
do_picture_classification: bool = False
|
|
||||||
do_picture_description: bool = False
|
|
||||||
generate_picture_images: bool = False
|
|
||||||
generate_page_images: bool = False
|
|
||||||
images_scale: float = 1.0
|
|
||||||
|
|
||||||
def is_default(self) -> bool:
|
|
||||||
return self == ConversionOptions()
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ConversionResult:
|
|
||||||
page_count: int
|
|
||||||
content_markdown: str
|
|
||||||
content_html: str
|
|
||||||
pages: list[PageDetail]
|
|
||||||
skipped_items: int = 0
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Element type detection
|
# Element type detection
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
# Mapping from Docling type to element type string.
|
|
||||||
# Order matters: most specific types before their parents.
|
|
||||||
_ELEMENT_TYPE_MAP: list[tuple[type, str]] = [
|
_ELEMENT_TYPE_MAP: list[tuple[type, str]] = [
|
||||||
(TableItem, "table"),
|
(TableItem, "table"),
|
||||||
(PictureItem, "picture"),
|
(PictureItem, "picture"),
|
||||||
|
|
@ -113,7 +73,6 @@ _ELEMENT_TYPE_MAP: list[tuple[type, str]] = [
|
||||||
|
|
||||||
|
|
||||||
def _get_element_type(item: DocItem) -> str:
|
def _get_element_type(item: DocItem) -> str:
|
||||||
"""Determine element type via isinstance on Docling's type hierarchy."""
|
|
||||||
for cls, type_name in _ELEMENT_TYPE_MAP:
|
for cls, type_name in _ELEMENT_TYPE_MAP:
|
||||||
if isinstance(item, cls):
|
if isinstance(item, cls):
|
||||||
return type_name
|
return type_name
|
||||||
|
|
@ -124,51 +83,50 @@ def _get_element_type(item: DocItem) -> str:
|
||||||
# Pipeline factory
|
# Pipeline factory
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def build_converter(options: ConversionOptions | None = None) -> DocumentConverter:
|
def _build_docling_converter(options: ConversionOptions) -> DoclingConverter:
|
||||||
"""Build a DocumentConverter with the given pipeline options."""
|
|
||||||
opts = options or ConversionOptions()
|
|
||||||
|
|
||||||
table_options = TableStructureOptions(
|
table_options = TableStructureOptions(
|
||||||
do_cell_matching=True,
|
do_cell_matching=True,
|
||||||
mode=TableFormerMode.ACCURATE if opts.table_mode == "accurate" else TableFormerMode.FAST,
|
mode=TableFormerMode.ACCURATE if options.table_mode == "accurate" else TableFormerMode.FAST,
|
||||||
)
|
)
|
||||||
|
|
||||||
pipeline_options = PdfPipelineOptions(
|
pipeline_options = PdfPipelineOptions(
|
||||||
do_ocr=opts.do_ocr,
|
do_ocr=options.do_ocr,
|
||||||
do_table_structure=opts.do_table_structure,
|
do_table_structure=options.do_table_structure,
|
||||||
table_structure_options=table_options,
|
table_structure_options=table_options,
|
||||||
do_code_enrichment=opts.do_code_enrichment,
|
do_code_enrichment=options.do_code_enrichment,
|
||||||
do_formula_enrichment=opts.do_formula_enrichment,
|
do_formula_enrichment=options.do_formula_enrichment,
|
||||||
do_picture_classification=opts.do_picture_classification,
|
do_picture_classification=options.do_picture_classification,
|
||||||
do_picture_description=opts.do_picture_description,
|
do_picture_description=options.do_picture_description,
|
||||||
generate_page_images=opts.generate_page_images,
|
generate_page_images=options.generate_page_images,
|
||||||
generate_picture_images=opts.generate_picture_images,
|
generate_picture_images=options.generate_picture_images,
|
||||||
images_scale=opts.images_scale,
|
images_scale=options.images_scale,
|
||||||
)
|
)
|
||||||
|
|
||||||
return DocumentConverter(
|
return DoclingConverter(
|
||||||
format_options={
|
format_options={
|
||||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
|
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_default_converter() -> DocumentConverter:
|
def _get_default_converter() -> DoclingConverter:
|
||||||
global _default_converter
|
global _default_converter
|
||||||
if _default_converter is None:
|
if _default_converter is None:
|
||||||
_default_converter = build_converter()
|
_default_converter = _build_docling_converter(ConversionOptions())
|
||||||
return _default_converter
|
return _default_converter
|
||||||
|
|
||||||
|
|
||||||
|
def _select_converter(options: ConversionOptions) -> DoclingConverter:
|
||||||
|
if options.is_default():
|
||||||
|
return _get_default_converter()
|
||||||
|
return _build_docling_converter(options)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Page extraction
|
# Page extraction
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def extract_pages_detail(doc_result) -> tuple[list[PageDetail], int]:
|
def _extract_pages_detail(doc_result) -> tuple[list[PageDetail], int]:
|
||||||
"""Extract per-page element details with bounding boxes from Docling result.
|
|
||||||
|
|
||||||
Returns (pages, skipped_count) for transparent error reporting.
|
|
||||||
"""
|
|
||||||
pages: dict[int, PageDetail] = {}
|
pages: dict[int, PageDetail] = {}
|
||||||
document = doc_result.document
|
document = doc_result.document
|
||||||
skipped = 0
|
skipped = 0
|
||||||
|
|
@ -193,7 +151,6 @@ def extract_pages_detail(doc_result) -> tuple[list[PageDetail], int]:
|
||||||
def _process_content_item(
|
def _process_content_item(
|
||||||
item: DocItem | GroupItem, level: int, pages: dict[int, PageDetail],
|
item: DocItem | GroupItem, level: int, pages: dict[int, PageDetail],
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Process a single content item and add it to the appropriate page."""
|
|
||||||
if isinstance(item, GroupItem):
|
if isinstance(item, GroupItem):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
@ -204,9 +161,6 @@ def _process_content_item(
|
||||||
try:
|
try:
|
||||||
page_no = prov.page_no
|
page_no = prov.page_no
|
||||||
if page_no not in pages:
|
if page_no not in pages:
|
||||||
# Fallback: page was not found in document.pages (corrupted PDF or
|
|
||||||
# Docling edge case). US Letter dimensions are used as a safe default.
|
|
||||||
# This may cause slight bbox misalignment on non-Letter pages (e.g. A4).
|
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Page %d not found in document metadata — using US Letter fallback (%sx%s pt)",
|
"Page %d not found in document metadata — using US Letter fallback (%sx%s pt)",
|
||||||
page_no, _DEFAULT_PAGE_WIDTH, _DEFAULT_PAGE_HEIGHT,
|
page_no, _DEFAULT_PAGE_WIDTH, _DEFAULT_PAGE_HEIGHT,
|
||||||
|
|
@ -242,49 +196,27 @@ def _process_content_item(
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Main conversion entry point
|
# Synchronous conversion (called via asyncio.to_thread)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def _select_converter(options: ConversionOptions) -> DocumentConverter:
|
def _convert_sync(file_path: str, options: ConversionOptions) -> ConversionResult:
|
||||||
"""Return the cached default converter or build a custom one."""
|
|
||||||
if options.is_default():
|
|
||||||
return get_default_converter()
|
|
||||||
return build_converter(options)
|
|
||||||
|
|
||||||
|
|
||||||
def _build_fallback_pages(doc, page_count: int) -> list[PageDetail]:
|
|
||||||
"""Create empty PageDetail entries when extraction yields nothing."""
|
|
||||||
return [
|
|
||||||
PageDetail(
|
|
||||||
page_number=i + 1,
|
|
||||||
width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else _DEFAULT_PAGE_WIDTH,
|
|
||||||
height=doc.pages[i + 1].size.height if (i + 1) in doc.pages else _DEFAULT_PAGE_HEIGHT,
|
|
||||||
)
|
|
||||||
for i in range(page_count)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def convert_document(
|
|
||||||
file_path: str,
|
|
||||||
options: ConversionOptions | None = None,
|
|
||||||
) -> ConversionResult:
|
|
||||||
"""Convert a document and return structured results.
|
|
||||||
|
|
||||||
This is the main entry point for document parsing. Runs synchronously
|
|
||||||
(caller should use asyncio.to_thread for non-blocking execution).
|
|
||||||
"""
|
|
||||||
opts = options or ConversionOptions()
|
|
||||||
|
|
||||||
with _converter_lock:
|
with _converter_lock:
|
||||||
conv = _select_converter(opts)
|
conv = _select_converter(options)
|
||||||
result = conv.convert(file_path)
|
result = conv.convert(file_path)
|
||||||
|
|
||||||
doc = result.document
|
doc = result.document
|
||||||
page_count = len(doc.pages)
|
page_count = len(doc.pages)
|
||||||
pages_detail, skipped = extract_pages_detail(result)
|
pages_detail, skipped = _extract_pages_detail(result)
|
||||||
|
|
||||||
if not pages_detail and page_count > 0:
|
if not pages_detail and page_count > 0:
|
||||||
pages_detail = _build_fallback_pages(doc, page_count)
|
pages_detail = [
|
||||||
|
PageDetail(
|
||||||
|
page_number=i + 1,
|
||||||
|
width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else _DEFAULT_PAGE_WIDTH,
|
||||||
|
height=doc.pages[i + 1].size.height if (i + 1) in doc.pages else _DEFAULT_PAGE_HEIGHT,
|
||||||
|
)
|
||||||
|
for i in range(page_count)
|
||||||
|
]
|
||||||
|
|
||||||
if skipped > 0:
|
if skipped > 0:
|
||||||
logger.info("Parsed: %d pages, %d items skipped", page_count, skipped)
|
logger.info("Parsed: %d pages, %d items skipped", page_count, skipped)
|
||||||
|
|
@ -296,3 +228,16 @@ def convert_document(
|
||||||
pages=pages_detail,
|
pages=pages_detail,
|
||||||
skipped_items=skipped,
|
skipped_items=skipped,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Public adapter class
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class LocalConverter:
|
||||||
|
"""Adapter that runs Docling locally as a Python library."""
|
||||||
|
|
||||||
|
async def convert(
|
||||||
|
self, file_path: str, options: ConversionOptions,
|
||||||
|
) -> ConversionResult:
|
||||||
|
return await asyncio.to_thread(_convert_sync, file_path, options)
|
||||||
234
document-parser/infra/serve_converter.py
Normal file
234
document-parser/infra/serve_converter.py
Normal file
|
|
@ -0,0 +1,234 @@
|
||||||
|
"""Remote Docling Serve converter — delegates conversion via HTTP.
|
||||||
|
|
||||||
|
This adapter implements the DocumentConverter port by calling a remote
|
||||||
|
Docling Serve instance's REST API (v1).
|
||||||
|
|
||||||
|
API contract based on docling-serve source code:
|
||||||
|
- Options are sent as individual multipart form fields (not a JSON blob)
|
||||||
|
- Response contains document.md_content, document.html_content, document.json_content
|
||||||
|
- json_content is a serialized DoclingDocument with texts[], tables[], pictures[]
|
||||||
|
- Bounding boxes use {l, t, r, b, coord_origin} format
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import mimetypes
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from domain.value_objects import (
|
||||||
|
ConversionOptions,
|
||||||
|
ConversionResult,
|
||||||
|
PageDetail,
|
||||||
|
PageElement,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_API_PREFIX = "/v1"
|
||||||
|
_DEFAULT_TIMEOUT = 600.0
|
||||||
|
|
||||||
|
# Docling Serve label → our element type
|
||||||
|
_LABEL_MAP = {
|
||||||
|
"table": "table",
|
||||||
|
"picture": "picture",
|
||||||
|
"figure": "picture",
|
||||||
|
"title": "title",
|
||||||
|
"section_header": "section_header",
|
||||||
|
"list_item": "list",
|
||||||
|
"formula": "formula",
|
||||||
|
"code": "code",
|
||||||
|
"caption": "text",
|
||||||
|
"footnote": "text",
|
||||||
|
"page_header": "text",
|
||||||
|
"page_footer": "text",
|
||||||
|
"paragraph": "text",
|
||||||
|
"text": "text",
|
||||||
|
"reference": "text",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ServeConverter:
|
||||||
|
"""Adapter that delegates document conversion to a remote Docling Serve instance."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
api_key: str | None = None,
|
||||||
|
timeout: float = _DEFAULT_TIMEOUT,
|
||||||
|
):
|
||||||
|
self._base_url = base_url.rstrip("/")
|
||||||
|
self._api_key = api_key
|
||||||
|
self._timeout = timeout
|
||||||
|
|
||||||
|
def _headers(self) -> dict[str, str]:
|
||||||
|
headers: dict[str, str] = {}
|
||||||
|
if self._api_key:
|
||||||
|
headers["X-Api-Key"] = self._api_key
|
||||||
|
return headers
|
||||||
|
|
||||||
|
async def convert(
|
||||||
|
self, file_path: str, options: ConversionOptions,
|
||||||
|
) -> ConversionResult:
|
||||||
|
"""Convert a document by uploading it to Docling Serve."""
|
||||||
|
path = Path(file_path)
|
||||||
|
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||||||
|
|
||||||
|
form_data = _build_form_data(options)
|
||||||
|
url = f"{self._base_url}{_API_PREFIX}/convert/file"
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
response = await client.post(
|
||||||
|
url,
|
||||||
|
files={"files": (path.name, f, content_type)},
|
||||||
|
data=form_data,
|
||||||
|
headers=self._headers(),
|
||||||
|
)
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
result_data = response.json()
|
||||||
|
|
||||||
|
return _parse_response(result_data)
|
||||||
|
|
||||||
|
async def health_check(self) -> bool:
|
||||||
|
"""Check if Docling Serve is reachable."""
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||||
|
resp = await client.get(
|
||||||
|
f"{self._base_url}/version",
|
||||||
|
headers=self._headers(),
|
||||||
|
)
|
||||||
|
return resp.status_code == 200
|
||||||
|
except httpx.HTTPError:
|
||||||
|
logger.warning("Docling Serve health check failed at %s", self._base_url, exc_info=True)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _build_form_data(options: ConversionOptions) -> dict[str, str | list[str]]:
|
||||||
|
"""Build form fields matching Docling Serve's multipart form contract.
|
||||||
|
|
||||||
|
Array fields (to_formats) are sent as lists — httpx encodes them as
|
||||||
|
repeated form keys (to_formats=md&to_formats=html&to_formats=json).
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"to_formats": ["md", "html", "json"],
|
||||||
|
"do_ocr": str(options.do_ocr).lower(),
|
||||||
|
"do_table_structure": str(options.do_table_structure).lower(),
|
||||||
|
"table_mode": options.table_mode,
|
||||||
|
"do_code_enrichment": str(options.do_code_enrichment).lower(),
|
||||||
|
"do_formula_enrichment": str(options.do_formula_enrichment).lower(),
|
||||||
|
"do_picture_classification": str(options.do_picture_classification).lower(),
|
||||||
|
"do_picture_description": str(options.do_picture_description).lower(),
|
||||||
|
"include_images": str(options.generate_picture_images).lower(),
|
||||||
|
"images_scale": str(options.images_scale),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_response(data: dict) -> ConversionResult:
|
||||||
|
"""Parse Docling Serve v1 ConvertDocumentResponse into our domain ConversionResult."""
|
||||||
|
document = data.get("document", {})
|
||||||
|
|
||||||
|
content_md = document.get("md_content") or ""
|
||||||
|
content_html = document.get("html_content") or ""
|
||||||
|
|
||||||
|
# json_content contains the full DoclingDocument with pages, elements, bboxes
|
||||||
|
json_content = document.get("json_content")
|
||||||
|
if isinstance(json_content, str):
|
||||||
|
try:
|
||||||
|
json_content = json.loads(json_content)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
logger.warning("Failed to parse json_content as JSON, ignoring structured data")
|
||||||
|
json_content = None
|
||||||
|
|
||||||
|
pages: list[PageDetail] = []
|
||||||
|
if json_content:
|
||||||
|
pages = _extract_pages_from_docling_document(json_content)
|
||||||
|
|
||||||
|
page_count = len(pages) if pages else 1
|
||||||
|
|
||||||
|
return ConversionResult(
|
||||||
|
page_count=page_count,
|
||||||
|
content_markdown=content_md,
|
||||||
|
content_html=content_html,
|
||||||
|
pages=pages,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_pages_from_docling_document(doc: dict) -> list[PageDetail]:
|
||||||
|
"""Extract pages with elements from a serialized DoclingDocument.
|
||||||
|
|
||||||
|
DoclingDocument structure:
|
||||||
|
- pages: {page_no: {size: {width, height}}}
|
||||||
|
- texts: [{label, text, prov: [{page_no, bbox: {l,t,r,b,coord_origin}}]}]
|
||||||
|
- tables: [{label, prov: [...], data: {...}}]
|
||||||
|
- pictures: [{label, prov: [...]}]
|
||||||
|
"""
|
||||||
|
pages_dict: dict[int, PageDetail] = {}
|
||||||
|
|
||||||
|
# Build page dimensions
|
||||||
|
for page_key, page_data in doc.get("pages", {}).items():
|
||||||
|
page_no = int(page_key)
|
||||||
|
size = page_data.get("size", {})
|
||||||
|
pages_dict[page_no] = PageDetail(
|
||||||
|
page_number=page_no,
|
||||||
|
width=size.get("width", 612.0),
|
||||||
|
height=size.get("height", 792.0),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Process all element arrays
|
||||||
|
for item in doc.get("texts", []):
|
||||||
|
_add_element(item, pages_dict)
|
||||||
|
|
||||||
|
for item in doc.get("tables", []):
|
||||||
|
_add_element(item, pages_dict)
|
||||||
|
|
||||||
|
for item in doc.get("pictures", []):
|
||||||
|
_add_element(item, pages_dict)
|
||||||
|
|
||||||
|
return sorted(pages_dict.values(), key=lambda p: p.page_number)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_element(item: dict, pages: dict[int, PageDetail]) -> None:
|
||||||
|
"""Add an element from a DoclingDocument array to the correct page."""
|
||||||
|
label = item.get("label", "text")
|
||||||
|
element_type = _LABEL_MAP.get(label, "text")
|
||||||
|
content = item.get("text", "") or ""
|
||||||
|
|
||||||
|
for prov in item.get("prov", []):
|
||||||
|
page_no = prov.get("page_no", 1)
|
||||||
|
if page_no not in pages:
|
||||||
|
pages[page_no] = PageDetail(
|
||||||
|
page_number=page_no, width=612.0, height=792.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
bbox_data = prov.get("bbox", {})
|
||||||
|
bbox = _extract_bbox(bbox_data, pages[page_no].height)
|
||||||
|
|
||||||
|
pages[page_no].elements.append(
|
||||||
|
PageElement(type=element_type, bbox=bbox, content=content, level=0)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_bbox(bbox_data: dict, page_height: float) -> list[float]:
|
||||||
|
"""Extract and normalize bbox to TOPLEFT [l, t, r, b] format."""
|
||||||
|
if not isinstance(bbox_data, dict):
|
||||||
|
return [0.0, 0.0, 0.0, 0.0]
|
||||||
|
|
||||||
|
left = bbox_data.get("l", 0.0)
|
||||||
|
top = bbox_data.get("t", 0.0)
|
||||||
|
right = bbox_data.get("r", 0.0)
|
||||||
|
bottom = bbox_data.get("b", 0.0)
|
||||||
|
coord_origin = bbox_data.get("coord_origin", "TOPLEFT")
|
||||||
|
|
||||||
|
if coord_origin == "BOTTOMLEFT":
|
||||||
|
# In BOTTOMLEFT: top has higher y, bottom has lower y
|
||||||
|
# In TOPLEFT: flip both — new_top = page_height - old_top
|
||||||
|
new_top = page_height - top
|
||||||
|
new_bottom = page_height - bottom
|
||||||
|
top, bottom = new_top, new_bottom
|
||||||
|
|
||||||
|
return [left, top, right, bottom]
|
||||||
30
document-parser/infra/settings.py
Normal file
30
document-parser/infra/settings.py
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
"""Centralized application settings — loaded from environment variables."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Settings:
|
||||||
|
conversion_engine: str = "local" # "local" or "remote"
|
||||||
|
docling_serve_url: str = "http://localhost:5001"
|
||||||
|
docling_serve_api_key: str | None = None
|
||||||
|
conversion_timeout: int = 600
|
||||||
|
upload_dir: str = "./uploads"
|
||||||
|
db_path: str = "./data/docling_studio.db"
|
||||||
|
cors_origins: list[str] = field(default_factory=lambda: ["http://localhost:3000", "http://localhost:5173"])
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls) -> Settings:
|
||||||
|
cors_raw = os.environ.get("CORS_ORIGINS", "http://localhost:3000,http://localhost:5173")
|
||||||
|
return cls(
|
||||||
|
conversion_engine=os.environ.get("CONVERSION_ENGINE", "local"),
|
||||||
|
docling_serve_url=os.environ.get("DOCLING_SERVE_URL", "http://localhost:5001"),
|
||||||
|
docling_serve_api_key=os.environ.get("DOCLING_SERVE_API_KEY"),
|
||||||
|
conversion_timeout=int(os.environ.get("CONVERSION_TIMEOUT", "600")),
|
||||||
|
upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"),
|
||||||
|
db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"),
|
||||||
|
cors_origins=[o.strip() for o in cors_raw.split(",")],
|
||||||
|
)
|
||||||
|
|
@ -1,14 +1,17 @@
|
||||||
"""Docling Studio — unified FastAPI backend.
|
"""Docling Studio — unified FastAPI backend.
|
||||||
|
|
||||||
Single service replacing both the Spring Boot backend and the document parser.
|
Single service providing document management (upload, CRUD), analysis
|
||||||
Provides document management (upload, CRUD), analysis orchestration (async Docling
|
orchestration (async Docling processing), and PDF preview — all backed
|
||||||
processing), and PDF preview — all backed by SQLite.
|
by SQLite.
|
||||||
|
|
||||||
|
Conversion engine is selected via CONVERSION_ENGINE env var:
|
||||||
|
- "local" → Docling runs in-process as a Python library (default)
|
||||||
|
- "remote" → delegates to a Docling Serve instance via HTTP
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
@ -16,7 +19,9 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
from api.analyses import router as analyses_router
|
from api.analyses import router as analyses_router
|
||||||
from api.documents import router as documents_router
|
from api.documents import router as documents_router
|
||||||
|
from infra.settings import Settings
|
||||||
from persistence.database import init_db
|
from persistence.database import init_db
|
||||||
|
from services.analysis_service import AnalysisService
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
|
|
@ -24,12 +29,45 @@ logging.basicConfig(
|
||||||
)
|
)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Settings & dependency wiring
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
settings = Settings.from_env()
|
||||||
|
|
||||||
|
|
||||||
|
def _build_converter():
|
||||||
|
"""Build the converter adapter based on configuration."""
|
||||||
|
if settings.conversion_engine == "remote":
|
||||||
|
from infra.serve_converter import ServeConverter
|
||||||
|
logger.info("Using remote Docling Serve at %s", settings.docling_serve_url)
|
||||||
|
return ServeConverter(
|
||||||
|
base_url=settings.docling_serve_url,
|
||||||
|
api_key=settings.docling_serve_api_key,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
from infra.local_converter import LocalConverter
|
||||||
|
logger.info("Using local Docling converter")
|
||||||
|
return LocalConverter()
|
||||||
|
|
||||||
|
|
||||||
|
def _build_analysis_service() -> AnalysisService:
|
||||||
|
converter = _build_converter()
|
||||||
|
return AnalysisService(
|
||||||
|
converter=converter,
|
||||||
|
conversion_timeout=settings.conversion_timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# FastAPI app
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
"""Startup: initialize database. Shutdown: nothing special needed."""
|
|
||||||
await init_db()
|
await init_db()
|
||||||
logger.info("Docling Studio backend ready")
|
app.state.analysis_service = _build_analysis_service()
|
||||||
|
logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine)
|
||||||
yield
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -39,20 +77,14 @@ app = FastAPI(
|
||||||
lifespan=lifespan,
|
lifespan=lifespan,
|
||||||
)
|
)
|
||||||
|
|
||||||
# CORS — configurable via env, defaults for local dev
|
|
||||||
allowed_origins = os.environ.get(
|
|
||||||
"CORS_ORIGINS", "http://localhost:3000,http://localhost:5173"
|
|
||||||
).split(",")
|
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
allow_origins=[o.strip() for o in allowed_origins],
|
allow_origins=settings.cors_origins,
|
||||||
allow_credentials=True,
|
allow_credentials=True,
|
||||||
allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
|
allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
|
||||||
allow_headers=["Content-Type", "Authorization"],
|
allow_headers=["Content-Type", "Authorization"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Mount routers
|
|
||||||
app.include_router(documents_router)
|
app.include_router(documents_router)
|
||||||
app.include_router(analyses_router)
|
app.include_router(analyses_router)
|
||||||
|
|
||||||
|
|
@ -60,4 +92,4 @@ app.include_router(analyses_router)
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
def health():
|
def health():
|
||||||
"""Health check endpoint."""
|
"""Health check endpoint."""
|
||||||
return {"status": "ok"}
|
return {"status": "ok", "engine": settings.conversion_engine}
|
||||||
|
|
|
||||||
|
|
@ -2,10 +2,19 @@
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from domain.models import AnalysisJob, AnalysisStatus
|
from domain.models import AnalysisJob, AnalysisStatus
|
||||||
from persistence.database import get_connection
|
from persistence.database import get_connection
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_dt(value: str | None) -> datetime | None:
|
||||||
|
"""Parse an ISO-format datetime string back into a datetime object."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
return datetime.fromisoformat(value)
|
||||||
|
|
||||||
|
|
||||||
def _row_to_job(row) -> AnalysisJob:
|
def _row_to_job(row) -> AnalysisJob:
|
||||||
return AnalysisJob(
|
return AnalysisJob(
|
||||||
id=row["id"],
|
id=row["id"],
|
||||||
|
|
@ -15,9 +24,9 @@ def _row_to_job(row) -> AnalysisJob:
|
||||||
content_html=row["content_html"],
|
content_html=row["content_html"],
|
||||||
pages_json=row["pages_json"],
|
pages_json=row["pages_json"],
|
||||||
error_message=row["error_message"],
|
error_message=row["error_message"],
|
||||||
started_at=row["started_at"],
|
started_at=_parse_dt(row["started_at"]),
|
||||||
completed_at=row["completed_at"],
|
completed_at=_parse_dt(row["completed_at"]),
|
||||||
created_at=row["created_at"],
|
created_at=_parse_dt(row["created_at"]) or datetime.now(),
|
||||||
document_filename=row["filename"] if "filename" in row.keys() else None, # noqa: SIM118
|
document_filename=row["filename"] if "filename" in row.keys() else None, # noqa: SIM118
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,16 @@
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from domain.models import Document
|
from domain.models import Document
|
||||||
from persistence.database import get_connection
|
from persistence.database import get_connection
|
||||||
|
|
||||||
|
|
||||||
def _row_to_document(row) -> Document:
|
def _row_to_document(row) -> Document:
|
||||||
|
created = row["created_at"]
|
||||||
|
if isinstance(created, str):
|
||||||
|
created = datetime.fromisoformat(created)
|
||||||
return Document(
|
return Document(
|
||||||
id=row["id"],
|
id=row["id"],
|
||||||
filename=row["filename"],
|
filename=row["filename"],
|
||||||
|
|
@ -14,7 +19,7 @@ def _row_to_document(row) -> Document:
|
||||||
file_size=row["file_size"],
|
file_size=row["file_size"],
|
||||||
page_count=row["page_count"],
|
page_count=row["page_count"],
|
||||||
storage_path=row["storage_path"],
|
storage_path=row["storage_path"],
|
||||||
created_at=row["created_at"],
|
created_at=created,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,3 +6,4 @@ python-multipart>=0.0.12
|
||||||
pdf2image>=1.17.0,<2.0.0
|
pdf2image>=1.17.0,<2.0.0
|
||||||
pillow>=10.0.0,<11.0.0
|
pillow>=10.0.0,<11.0.0
|
||||||
aiosqlite>=0.20.0,<1.0.0
|
aiosqlite>=0.20.0,<1.0.0
|
||||||
|
httpx>=0.27.0,<1.0.0
|
||||||
|
|
|
||||||
|
|
@ -1,108 +1,125 @@
|
||||||
"""Analysis service — async document parsing orchestration."""
|
"""Analysis service — async document parsing orchestration.
|
||||||
|
|
||||||
|
Uses an injected DocumentConverter (port) so the service is decoupled
|
||||||
|
from the conversion implementation (local Docling lib vs remote Docling Serve).
|
||||||
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import functools
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import asdict
|
from dataclasses import asdict
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from domain.models import AnalysisJob
|
from domain.models import AnalysisJob
|
||||||
from domain.parsing import ConversionOptions, ConversionResult, convert_document
|
from domain.value_objects import ConversionOptions, ConversionResult
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from domain.ports import DocumentConverter
|
||||||
from persistence import analysis_repo, document_repo
|
from persistence import analysis_repo, document_repo
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Maximum time (seconds) allowed for a single document conversion.
|
|
||||||
CONVERSION_TIMEOUT = int(__import__("os").environ.get("CONVERSION_TIMEOUT", "600"))
|
class AnalysisService:
|
||||||
|
"""Orchestrates document analysis using an injected converter."""
|
||||||
|
|
||||||
|
def __init__(self, converter: DocumentConverter, conversion_timeout: int = 600):
|
||||||
|
self._converter = converter
|
||||||
|
self._conversion_timeout = conversion_timeout
|
||||||
|
|
||||||
|
async def create(self, document_id: str, *, pipeline_options: dict | None = None) -> AnalysisJob:
|
||||||
|
"""Create a new analysis job and launch background processing."""
|
||||||
|
doc = await document_repo.find_by_id(document_id)
|
||||||
|
if not doc:
|
||||||
|
raise ValueError(f"Document not found: {document_id}")
|
||||||
|
|
||||||
|
job = AnalysisJob(document_id=document_id)
|
||||||
|
job.document_filename = doc.filename
|
||||||
|
await analysis_repo.insert(job)
|
||||||
|
|
||||||
|
task = asyncio.create_task(
|
||||||
|
self._run_analysis(job.id, doc.storage_path, doc.filename, pipeline_options)
|
||||||
|
)
|
||||||
|
task.add_done_callback(functools.partial(_on_task_done, job_id=job.id))
|
||||||
|
|
||||||
|
return job
|
||||||
|
|
||||||
|
async def find_all(self) -> list[AnalysisJob]:
|
||||||
|
return await analysis_repo.find_all()
|
||||||
|
|
||||||
|
async def find_by_id(self, job_id: str) -> AnalysisJob | None:
|
||||||
|
return await analysis_repo.find_by_id(job_id)
|
||||||
|
|
||||||
|
async def delete(self, job_id: str) -> bool:
|
||||||
|
return await analysis_repo.delete(job_id)
|
||||||
|
|
||||||
|
async def _run_analysis(
|
||||||
|
self, job_id: str, file_path: str, filename: str, pipeline_options: dict | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Background task: run conversion and update job status."""
|
||||||
|
try:
|
||||||
|
job = await analysis_repo.find_by_id(job_id)
|
||||||
|
if not job:
|
||||||
|
logger.error("Analysis job %s not found", job_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
job.mark_running()
|
||||||
|
await analysis_repo.update_status(job)
|
||||||
|
logger.info("Analysis started: %s (file: %s)", job_id, filename)
|
||||||
|
|
||||||
|
options = ConversionOptions(**(pipeline_options or {}))
|
||||||
|
|
||||||
|
result: ConversionResult = await asyncio.wait_for(
|
||||||
|
self._converter.convert(file_path, options),
|
||||||
|
timeout=self._conversion_timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
pages_json = json.dumps([asdict(p) for p in result.pages])
|
||||||
|
|
||||||
|
job.mark_completed(
|
||||||
|
markdown=result.content_markdown,
|
||||||
|
html=result.content_html,
|
||||||
|
pages_json=pages_json,
|
||||||
|
)
|
||||||
|
await analysis_repo.update_status(job)
|
||||||
|
|
||||||
|
if result.page_count:
|
||||||
|
await document_repo.update_page_count(job.document_id, result.page_count)
|
||||||
|
|
||||||
|
logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count)
|
||||||
|
|
||||||
|
except TimeoutError:
|
||||||
|
logger.error("Analysis timed out after %ds: %s", self._conversion_timeout, job_id)
|
||||||
|
await _mark_failed(job_id, f"Conversion timed out after {self._conversion_timeout}s")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Analysis failed: %s", job_id)
|
||||||
|
await _mark_failed(job_id, str(e))
|
||||||
|
|
||||||
|
|
||||||
async def create(document_id: str, *, pipeline_options: dict | None = None) -> AnalysisJob:
|
_background_tasks: set[asyncio.Task] = set()
|
||||||
"""Create a new analysis job and launch background processing."""
|
|
||||||
doc = await document_repo.find_by_id(document_id)
|
|
||||||
if not doc:
|
|
||||||
raise ValueError(f"Document not found: {document_id}")
|
|
||||||
|
|
||||||
job = AnalysisJob(document_id=document_id)
|
|
||||||
job.document_filename = doc.filename
|
|
||||||
await analysis_repo.insert(job)
|
|
||||||
|
|
||||||
# Fire background task with error logging callback
|
|
||||||
task = asyncio.create_task(
|
|
||||||
_run_analysis(job.id, doc.storage_path, doc.filename, pipeline_options)
|
|
||||||
)
|
|
||||||
task.add_done_callback(_on_task_done)
|
|
||||||
|
|
||||||
return job
|
|
||||||
|
|
||||||
|
|
||||||
def _on_task_done(task: asyncio.Task) -> None:
|
def _on_task_done(task: asyncio.Task, *, job_id: str) -> None:
|
||||||
"""Log unhandled exceptions from background analysis tasks."""
|
"""Log unhandled exceptions from background analysis tasks and mark job as FAILED."""
|
||||||
if task.cancelled():
|
if task.cancelled():
|
||||||
logger.warning("Analysis task was cancelled")
|
logger.warning("Analysis task was cancelled: %s", job_id)
|
||||||
|
_schedule_mark_failed(job_id, "Task was cancelled")
|
||||||
return
|
return
|
||||||
exc = task.exception()
|
exc = task.exception()
|
||||||
if exc:
|
if exc:
|
||||||
logger.error("Unhandled exception in analysis task: %s", exc, exc_info=True)
|
logger.error("Unhandled exception in analysis task %s: %s", job_id, exc, exc_info=True)
|
||||||
|
_schedule_mark_failed(job_id, str(exc))
|
||||||
|
|
||||||
|
|
||||||
async def find_all() -> list[AnalysisJob]:
|
def _schedule_mark_failed(job_id: str, error: str) -> None:
|
||||||
return await analysis_repo.find_all()
|
"""Schedule _mark_failed as a tracked background task."""
|
||||||
|
t = asyncio.ensure_future(_mark_failed(job_id, error))
|
||||||
|
_background_tasks.add(t)
|
||||||
async def find_by_id(job_id: str) -> AnalysisJob | None:
|
t.add_done_callback(_background_tasks.discard)
|
||||||
return await analysis_repo.find_by_id(job_id)
|
|
||||||
|
|
||||||
|
|
||||||
async def delete(job_id: str) -> bool:
|
|
||||||
return await analysis_repo.delete(job_id)
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_analysis(
|
|
||||||
job_id: str, file_path: str, filename: str, pipeline_options: dict | None = None,
|
|
||||||
) -> None:
|
|
||||||
"""Background task: run Docling conversion and update job status."""
|
|
||||||
try:
|
|
||||||
job = await analysis_repo.find_by_id(job_id)
|
|
||||||
if not job:
|
|
||||||
logger.error("Analysis job %s not found", job_id)
|
|
||||||
return
|
|
||||||
|
|
||||||
job.mark_running()
|
|
||||||
await analysis_repo.update_status(job)
|
|
||||||
logger.info("Analysis started: %s (file: %s)", job_id, filename)
|
|
||||||
|
|
||||||
# Build conversion options from pipeline dict
|
|
||||||
options = ConversionOptions(**(pipeline_options or {}))
|
|
||||||
|
|
||||||
# Run blocking Docling conversion in a thread with timeout
|
|
||||||
result: ConversionResult = await asyncio.wait_for(
|
|
||||||
asyncio.to_thread(convert_document, file_path, options),
|
|
||||||
timeout=CONVERSION_TIMEOUT,
|
|
||||||
)
|
|
||||||
|
|
||||||
pages_json = json.dumps([asdict(p) for p in result.pages])
|
|
||||||
|
|
||||||
job.mark_completed(
|
|
||||||
markdown=result.content_markdown,
|
|
||||||
html=result.content_html,
|
|
||||||
pages_json=pages_json,
|
|
||||||
)
|
|
||||||
await analysis_repo.update_status(job)
|
|
||||||
|
|
||||||
# Update document page count if available
|
|
||||||
if result.page_count:
|
|
||||||
await document_repo.update_page_count(job.document_id, result.page_count)
|
|
||||||
|
|
||||||
logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count)
|
|
||||||
|
|
||||||
except TimeoutError:
|
|
||||||
logger.error("Analysis timed out after %ds: %s", CONVERSION_TIMEOUT, job_id)
|
|
||||||
await _mark_failed(job_id, f"Conversion timed out after {CONVERSION_TIMEOUT}s")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception("Analysis failed: %s", job_id)
|
|
||||||
await _mark_failed(job_id, str(e))
|
|
||||||
|
|
||||||
|
|
||||||
async def _mark_failed(job_id: str, error: str) -> None:
|
async def _mark_failed(job_id: str, error: str) -> None:
|
||||||
|
|
|
||||||
|
|
@ -70,10 +70,14 @@ async def delete(doc_id: str) -> bool:
|
||||||
# Delete associated analyses first (cascade)
|
# Delete associated analyses first (cascade)
|
||||||
await analysis_repo.delete_by_document(doc_id)
|
await analysis_repo.delete_by_document(doc_id)
|
||||||
|
|
||||||
# Delete file from disk
|
# Delete file from disk (only if inside UPLOAD_DIR)
|
||||||
try:
|
try:
|
||||||
if os.path.exists(doc.storage_path):
|
real_path = os.path.realpath(doc.storage_path)
|
||||||
os.unlink(doc.storage_path)
|
real_upload_dir = os.path.realpath(UPLOAD_DIR)
|
||||||
|
if real_path.startswith(real_upload_dir + os.sep) and os.path.exists(real_path):
|
||||||
|
os.unlink(real_path)
|
||||||
|
elif os.path.exists(doc.storage_path):
|
||||||
|
logger.warning("Refused to delete file outside upload dir: %s", doc.storage_path)
|
||||||
except OSError:
|
except OSError:
|
||||||
logger.warning("Could not delete file: %s", doc.storage_path)
|
logger.warning("Could not delete file: %s", doc.storage_path)
|
||||||
|
|
||||||
|
|
|
||||||
71
document-parser/tests/test_analysis_service.py
Normal file
71
document-parser/tests/test_analysis_service.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
"""Tests for AnalysisService — focus on _on_task_done callback (bug #1 fix)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from services.analysis_service import _on_task_done
|
||||||
|
|
||||||
|
|
||||||
|
class TestOnTaskDone:
|
||||||
|
"""Bug #1: _on_task_done must call _mark_failed when the task raises."""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_exception_marks_job_failed(self):
|
||||||
|
"""When a background task raises, the job should be marked FAILED."""
|
||||||
|
job_id = "job-123"
|
||||||
|
|
||||||
|
# Create a task that raises
|
||||||
|
async def failing_task():
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
|
||||||
|
task = asyncio.create_task(failing_task())
|
||||||
|
await asyncio.sleep(0) # let the task fail
|
||||||
|
|
||||||
|
with patch("services.analysis_service._mark_failed", new_callable=AsyncMock) as mock_mark:
|
||||||
|
_on_task_done(task, job_id=job_id)
|
||||||
|
# ensure_future schedules it; give the event loop a tick
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
mock_mark.assert_called_once_with(job_id, "boom")
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cancelled_task_marks_job_failed(self):
|
||||||
|
"""When a background task is cancelled, the job should be marked FAILED."""
|
||||||
|
job_id = "job-456"
|
||||||
|
|
||||||
|
async def slow_task():
|
||||||
|
await asyncio.sleep(999)
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
|
||||||
|
task = asyncio.create_task(slow_task())
|
||||||
|
task.cancel()
|
||||||
|
with contextlib.suppress(asyncio.CancelledError):
|
||||||
|
await task
|
||||||
|
|
||||||
|
with patch("services.analysis_service._mark_failed", new_callable=AsyncMock) as mock_mark:
|
||||||
|
_on_task_done(task, job_id=job_id)
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
mock_mark.assert_called_once_with(job_id, "Task was cancelled")
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_successful_task_does_not_mark_failed(self):
|
||||||
|
"""When a background task succeeds, _mark_failed should not be called."""
|
||||||
|
job_id = "job-789"
|
||||||
|
|
||||||
|
async def ok_task():
|
||||||
|
return "done"
|
||||||
|
|
||||||
|
task = asyncio.create_task(ok_task())
|
||||||
|
await task
|
||||||
|
|
||||||
|
with patch("services.analysis_service._mark_failed", new_callable=AsyncMock) as mock_mark:
|
||||||
|
_on_task_done(task, job_id=job_id)
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
mock_mark.assert_not_called()
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
"""Tests for FastAPI API endpoints using TestClient."""
|
"""Tests for FastAPI API endpoints using TestClient."""
|
||||||
|
|
||||||
from unittest.mock import AsyncMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
@ -14,11 +14,23 @@ def client():
|
||||||
return TestClient(app, raise_server_exceptions=False)
|
return TestClient(app, raise_server_exceptions=False)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_analysis_service(client):
|
||||||
|
"""Inject a mock AnalysisService into app.state for the duration of the test."""
|
||||||
|
mock_svc = MagicMock()
|
||||||
|
original = getattr(app.state, "analysis_service", None)
|
||||||
|
app.state.analysis_service = mock_svc
|
||||||
|
yield mock_svc
|
||||||
|
app.state.analysis_service = original
|
||||||
|
|
||||||
|
|
||||||
class TestHealthEndpoint:
|
class TestHealthEndpoint:
|
||||||
def test_health(self, client):
|
def test_health(self, client):
|
||||||
resp = client.get("/health")
|
resp = client.get("/health")
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
assert resp.json() == {"status": "ok"}
|
data = resp.json()
|
||||||
|
assert data["status"] == "ok"
|
||||||
|
assert "engine" in data
|
||||||
|
|
||||||
|
|
||||||
class TestDocumentEndpoints:
|
class TestDocumentEndpoints:
|
||||||
|
|
@ -102,11 +114,10 @@ class TestDocumentEndpoints:
|
||||||
|
|
||||||
|
|
||||||
class TestAnalysisEndpoints:
|
class TestAnalysisEndpoints:
|
||||||
@patch("services.analysis_service.find_all", new_callable=AsyncMock)
|
def test_list_analyses(self, client, mock_analysis_service):
|
||||||
def test_list_analyses(self, mock_find_all, client):
|
mock_analysis_service.find_all = AsyncMock(return_value=[
|
||||||
mock_find_all.return_value = [
|
|
||||||
AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf"),
|
AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf"),
|
||||||
]
|
])
|
||||||
|
|
||||||
resp = client.get("/api/analyses")
|
resp = client.get("/api/analyses")
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
|
|
@ -117,11 +128,10 @@ class TestAnalysisEndpoints:
|
||||||
assert data[0]["documentFilename"] == "test.pdf"
|
assert data[0]["documentFilename"] == "test.pdf"
|
||||||
assert data[0]["status"] == "PENDING"
|
assert data[0]["status"] == "PENDING"
|
||||||
|
|
||||||
@patch("services.analysis_service.find_by_id", new_callable=AsyncMock)
|
def test_get_analysis(self, client, mock_analysis_service):
|
||||||
def test_get_analysis(self, mock_find, client):
|
|
||||||
job = AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf")
|
job = AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf")
|
||||||
job.mark_running()
|
job.mark_running()
|
||||||
mock_find.return_value = job
|
mock_analysis_service.find_by_id = AsyncMock(return_value=job)
|
||||||
|
|
||||||
resp = client.get("/api/analyses/j1")
|
resp = client.get("/api/analyses/j1")
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
|
|
@ -129,31 +139,28 @@ class TestAnalysisEndpoints:
|
||||||
assert data["status"] == "RUNNING"
|
assert data["status"] == "RUNNING"
|
||||||
assert data["startedAt"] is not None
|
assert data["startedAt"] is not None
|
||||||
|
|
||||||
@patch("services.analysis_service.find_by_id", new_callable=AsyncMock)
|
def test_get_analysis_not_found(self, client, mock_analysis_service):
|
||||||
def test_get_analysis_not_found(self, mock_find, client):
|
mock_analysis_service.find_by_id = AsyncMock(return_value=None)
|
||||||
mock_find.return_value = None
|
|
||||||
|
|
||||||
resp = client.get("/api/analyses/missing")
|
resp = client.get("/api/analyses/missing")
|
||||||
assert resp.status_code == 404
|
assert resp.status_code == 404
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_create_analysis(self, client, mock_analysis_service):
|
||||||
def test_create_analysis(self, mock_create, client):
|
mock_analysis_service.create = AsyncMock(return_value=AnalysisJob(
|
||||||
mock_create.return_value = AnalysisJob(
|
|
||||||
id="j1", document_id="d1", document_filename="test.pdf",
|
id="j1", document_id="d1", document_filename="test.pdf",
|
||||||
)
|
))
|
||||||
|
|
||||||
resp = client.post("/api/analyses", json={"documentId": "d1"})
|
resp = client.post("/api/analyses", json={"documentId": "d1"})
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
assert data["id"] == "j1"
|
assert data["id"] == "j1"
|
||||||
assert data["documentId"] == "d1"
|
assert data["documentId"] == "d1"
|
||||||
mock_create.assert_called_once_with("d1", pipeline_options=None)
|
mock_analysis_service.create.assert_called_once_with("d1", pipeline_options=None)
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_create_analysis_with_pipeline_options(self, client, mock_analysis_service):
|
||||||
def test_create_analysis_with_pipeline_options(self, mock_create, client):
|
mock_analysis_service.create = AsyncMock(return_value=AnalysisJob(
|
||||||
mock_create.return_value = AnalysisJob(
|
|
||||||
id="j2", document_id="d1", document_filename="test.pdf",
|
id="j2", document_id="d1", document_filename="test.pdf",
|
||||||
)
|
))
|
||||||
|
|
||||||
resp = client.post("/api/analyses", json={
|
resp = client.post("/api/analyses", json={
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
|
|
@ -174,7 +181,7 @@ class TestAnalysisEndpoints:
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
assert data["id"] == "j2"
|
assert data["id"] == "j2"
|
||||||
|
|
||||||
call_kwargs = mock_create.call_args
|
call_kwargs = mock_analysis_service.create.call_args
|
||||||
opts = call_kwargs.kwargs["pipeline_options"]
|
opts = call_kwargs.kwargs["pipeline_options"]
|
||||||
assert opts["do_ocr"] is False
|
assert opts["do_ocr"] is False
|
||||||
assert opts["table_mode"] == "fast"
|
assert opts["table_mode"] == "fast"
|
||||||
|
|
@ -182,12 +189,11 @@ class TestAnalysisEndpoints:
|
||||||
assert opts["generate_picture_images"] is True
|
assert opts["generate_picture_images"] is True
|
||||||
assert opts["images_scale"] == 2.0
|
assert opts["images_scale"] == 2.0
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_create_analysis_with_partial_pipeline_options(self, client, mock_analysis_service):
|
||||||
def test_create_analysis_with_partial_pipeline_options(self, mock_create, client):
|
|
||||||
"""Pipeline options should use defaults for unspecified fields."""
|
"""Pipeline options should use defaults for unspecified fields."""
|
||||||
mock_create.return_value = AnalysisJob(
|
mock_analysis_service.create = AsyncMock(return_value=AnalysisJob(
|
||||||
id="j3", document_id="d1", document_filename="test.pdf",
|
id="j3", document_id="d1", document_filename="test.pdf",
|
||||||
)
|
))
|
||||||
|
|
||||||
resp = client.post("/api/analyses", json={
|
resp = client.post("/api/analyses", json={
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
|
|
@ -195,38 +201,35 @@ class TestAnalysisEndpoints:
|
||||||
})
|
})
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
|
|
||||||
opts = mock_create.call_args.kwargs["pipeline_options"]
|
opts = mock_analysis_service.create.call_args.kwargs["pipeline_options"]
|
||||||
assert opts["do_ocr"] is False
|
assert opts["do_ocr"] is False
|
||||||
# Defaults
|
# Defaults
|
||||||
assert opts["do_table_structure"] is True
|
assert opts["do_table_structure"] is True
|
||||||
assert opts["table_mode"] == "accurate"
|
assert opts["table_mode"] == "accurate"
|
||||||
assert opts["do_code_enrichment"] is False
|
assert opts["do_code_enrichment"] is False
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_create_analysis_document_not_found(self, client, mock_analysis_service):
|
||||||
def test_create_analysis_document_not_found(self, mock_create, client):
|
mock_analysis_service.create = AsyncMock(side_effect=ValueError("Document not found: d99"))
|
||||||
mock_create.side_effect = ValueError("Document not found: d99")
|
|
||||||
|
|
||||||
resp = client.post("/api/analyses", json={"documentId": "d99"})
|
resp = client.post("/api/analyses", json={"documentId": "d99"})
|
||||||
assert resp.status_code == 404
|
assert resp.status_code == 404
|
||||||
|
|
||||||
def test_create_analysis_empty_document_id(self, client):
|
def test_create_analysis_empty_document_id(self, client, mock_analysis_service):
|
||||||
resp = client.post("/api/analyses", json={"documentId": ""})
|
resp = client.post("/api/analyses", json={"documentId": ""})
|
||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
def test_create_analysis_whitespace_document_id(self, client):
|
def test_create_analysis_whitespace_document_id(self, client, mock_analysis_service):
|
||||||
resp = client.post("/api/analyses", json={"documentId": " "})
|
resp = client.post("/api/analyses", json={"documentId": " "})
|
||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
@patch("services.analysis_service.delete", new_callable=AsyncMock)
|
def test_delete_analysis(self, client, mock_analysis_service):
|
||||||
def test_delete_analysis(self, mock_delete, client):
|
mock_analysis_service.delete = AsyncMock(return_value=True)
|
||||||
mock_delete.return_value = True
|
|
||||||
|
|
||||||
resp = client.delete("/api/analyses/j1")
|
resp = client.delete("/api/analyses/j1")
|
||||||
assert resp.status_code == 204
|
assert resp.status_code == 204
|
||||||
|
|
||||||
@patch("services.analysis_service.delete", new_callable=AsyncMock)
|
def test_delete_analysis_not_found(self, client, mock_analysis_service):
|
||||||
def test_delete_analysis_not_found(self, mock_delete, client):
|
mock_analysis_service.delete = AsyncMock(return_value=False)
|
||||||
mock_delete.return_value = False
|
|
||||||
|
|
||||||
resp = client.delete("/api/analyses/missing")
|
resp = client.delete("/api/analyses/missing")
|
||||||
assert resp.status_code == 404
|
assert resp.status_code == 404
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,12 @@ from docling.datamodel.pipeline_options import (
|
||||||
TableFormerMode,
|
TableFormerMode,
|
||||||
)
|
)
|
||||||
|
|
||||||
from domain.parsing import (
|
from domain.value_objects import ConversionOptions
|
||||||
ConversionOptions,
|
from infra.local_converter import (
|
||||||
build_converter,
|
_build_docling_converter as build_converter,
|
||||||
convert_document,
|
)
|
||||||
|
from infra.local_converter import (
|
||||||
|
_convert_sync as convert_document,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -30,7 +32,7 @@ class TestBuildConverter:
|
||||||
return fmt_opt.pipeline_options
|
return fmt_opt.pipeline_options
|
||||||
|
|
||||||
def test_defaults(self):
|
def test_defaults(self):
|
||||||
conv = build_converter()
|
conv = build_converter(ConversionOptions())
|
||||||
opts = self._get_pipeline_options(conv)
|
opts = self._get_pipeline_options(conv)
|
||||||
assert opts.do_ocr is True
|
assert opts.do_ocr is True
|
||||||
assert opts.do_table_structure is True
|
assert opts.do_table_structure is True
|
||||||
|
|
@ -131,8 +133,8 @@ class TestBuildConverter:
|
||||||
class TestConvertDocumentRouting:
|
class TestConvertDocumentRouting:
|
||||||
"""Verify convert_document uses default converter for default opts, custom otherwise."""
|
"""Verify convert_document uses default converter for default opts, custom otherwise."""
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_default_converter_with_all_defaults(self, mock_build, mock_get_default):
|
def test_uses_default_converter_with_all_defaults(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -143,13 +145,13 @@ class TestConvertDocumentRouting:
|
||||||
mock_conv.convert.return_value = mock_result
|
mock_conv.convert.return_value = mock_result
|
||||||
mock_get_default.return_value = mock_conv
|
mock_get_default.return_value = mock_conv
|
||||||
|
|
||||||
convert_document("/tmp/test.pdf")
|
convert_document("/tmp/test.pdf", ConversionOptions())
|
||||||
|
|
||||||
mock_get_default.assert_called_once()
|
mock_get_default.assert_called_once()
|
||||||
mock_build.assert_not_called()
|
mock_build.assert_not_called()
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_ocr_disabled(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_ocr_disabled(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -165,8 +167,8 @@ class TestConvertDocumentRouting:
|
||||||
mock_build.assert_called_once()
|
mock_build.assert_called_once()
|
||||||
mock_get_default.assert_not_called()
|
mock_get_default.assert_not_called()
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_table_mode_fast(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_table_mode_fast(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -182,8 +184,8 @@ class TestConvertDocumentRouting:
|
||||||
|
|
||||||
mock_build.assert_called_once_with(opts)
|
mock_build.assert_called_once_with(opts)
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_code_enrichment_on(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_code_enrichment_on(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -199,8 +201,8 @@ class TestConvertDocumentRouting:
|
||||||
|
|
||||||
mock_build.assert_called_once_with(opts)
|
mock_build.assert_called_once_with(opts)
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_formula_enrichment_on(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_formula_enrichment_on(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -215,8 +217,8 @@ class TestConvertDocumentRouting:
|
||||||
|
|
||||||
mock_build.assert_called_once()
|
mock_build.assert_called_once()
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_picture_options_on(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_picture_options_on(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -231,8 +233,8 @@ class TestConvertDocumentRouting:
|
||||||
|
|
||||||
mock_build.assert_called_once()
|
mock_build.assert_called_once()
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_generate_images_on(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_generate_images_on(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -247,8 +249,8 @@ class TestConvertDocumentRouting:
|
||||||
|
|
||||||
mock_build.assert_called_once()
|
mock_build.assert_called_once()
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_uses_custom_converter_when_images_scale_changed(self, mock_build, mock_get_default):
|
def test_uses_custom_converter_when_images_scale_changed(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -264,8 +266,8 @@ class TestConvertDocumentRouting:
|
||||||
|
|
||||||
mock_build.assert_called_once_with(opts)
|
mock_build.assert_called_once_with(opts)
|
||||||
|
|
||||||
@patch("domain.parsing.get_default_converter")
|
@patch("infra.local_converter._get_default_converter")
|
||||||
@patch("domain.parsing.build_converter")
|
@patch("infra.local_converter._build_docling_converter")
|
||||||
def test_forwards_all_options_to_build_converter(self, mock_build, mock_get_default):
|
def test_forwards_all_options_to_build_converter(self, mock_build, mock_get_default):
|
||||||
mock_conv = MagicMock()
|
mock_conv = MagicMock()
|
||||||
mock_result = MagicMock()
|
mock_result = MagicMock()
|
||||||
|
|
@ -312,25 +314,21 @@ class TestServiceForwardsPipelineOptions:
|
||||||
|
|
||||||
@patch("services.analysis_service.document_repo")
|
@patch("services.analysis_service.document_repo")
|
||||||
@patch("services.analysis_service.analysis_repo")
|
@patch("services.analysis_service.analysis_repo")
|
||||||
@patch("services.analysis_service._run_analysis")
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_create_passes_pipeline_options_to_run(
|
async def test_create_passes_pipeline_options_to_run(
|
||||||
self, mock_run, mock_analysis_repo, mock_doc_repo, mock_doc,
|
self, mock_analysis_repo, mock_doc_repo, mock_doc,
|
||||||
):
|
):
|
||||||
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
|
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
|
||||||
mock_analysis_repo.insert = AsyncMock()
|
mock_analysis_repo.insert = AsyncMock()
|
||||||
# Patch _run_analysis as a coroutine that we can inspect
|
|
||||||
mock_run.return_value = None
|
|
||||||
|
|
||||||
from services import analysis_service
|
mock_converter = AsyncMock()
|
||||||
|
from services.analysis_service import AnalysisService
|
||||||
|
svc = AnalysisService(converter=mock_converter)
|
||||||
|
|
||||||
opts = {"do_ocr": False, "table_mode": "fast"}
|
opts = {"do_ocr": False, "table_mode": "fast"}
|
||||||
|
|
||||||
# We need to patch asyncio.create_task to capture the coroutine args
|
|
||||||
with patch("services.analysis_service.asyncio.create_task") as mock_task:
|
with patch("services.analysis_service.asyncio.create_task") as mock_task:
|
||||||
await analysis_service.create("d1", pipeline_options=opts)
|
await svc.create("d1", pipeline_options=opts)
|
||||||
|
|
||||||
# create_task should have been called with _run_analysis(...)
|
|
||||||
mock_task.assert_called_once()
|
mock_task.assert_called_once()
|
||||||
|
|
||||||
@patch("services.analysis_service.document_repo")
|
@patch("services.analysis_service.document_repo")
|
||||||
|
|
@ -342,32 +340,36 @@ class TestServiceForwardsPipelineOptions:
|
||||||
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
|
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
|
||||||
mock_analysis_repo.insert = AsyncMock()
|
mock_analysis_repo.insert = AsyncMock()
|
||||||
|
|
||||||
from services import analysis_service
|
mock_converter = AsyncMock()
|
||||||
|
from services.analysis_service import AnalysisService
|
||||||
|
svc = AnalysisService(converter=mock_converter)
|
||||||
|
|
||||||
with patch("services.analysis_service.asyncio.create_task") as mock_task:
|
with patch("services.analysis_service.asyncio.create_task") as mock_task:
|
||||||
await analysis_service.create("d1")
|
await svc.create("d1")
|
||||||
mock_task.assert_called_once()
|
mock_task.assert_called_once()
|
||||||
|
|
||||||
@patch("services.analysis_service.analysis_repo")
|
@patch("services.analysis_service.analysis_repo")
|
||||||
@patch("services.analysis_service.document_repo")
|
@patch("services.analysis_service.document_repo")
|
||||||
@patch("services.analysis_service.convert_document")
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_run_analysis_forwards_options_to_convert(
|
async def test_run_analysis_forwards_options_to_convert(
|
||||||
self, mock_convert, mock_doc_repo, mock_analysis_repo, mock_job,
|
self, mock_doc_repo, mock_analysis_repo, mock_job,
|
||||||
):
|
):
|
||||||
from domain.parsing import ConversionResult, PageDetail
|
from domain.value_objects import ConversionResult, PageDetail
|
||||||
|
|
||||||
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
||||||
mock_analysis_repo.update_status = AsyncMock()
|
mock_analysis_repo.update_status = AsyncMock()
|
||||||
mock_doc_repo.update_page_count = AsyncMock()
|
mock_doc_repo.update_page_count = AsyncMock()
|
||||||
mock_convert.return_value = ConversionResult(
|
|
||||||
|
mock_converter = AsyncMock()
|
||||||
|
mock_converter.convert.return_value = ConversionResult(
|
||||||
page_count=1,
|
page_count=1,
|
||||||
content_markdown="# Test",
|
content_markdown="# Test",
|
||||||
content_html="<h1>Test</h1>",
|
content_html="<h1>Test</h1>",
|
||||||
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
|
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
|
||||||
)
|
)
|
||||||
|
|
||||||
from services.analysis_service import _run_analysis
|
from services.analysis_service import AnalysisService
|
||||||
|
svc = AnalysisService(converter=mock_converter)
|
||||||
|
|
||||||
opts = {
|
opts = {
|
||||||
"do_ocr": False,
|
"do_ocr": False,
|
||||||
|
|
@ -381,10 +383,10 @@ class TestServiceForwardsPipelineOptions:
|
||||||
"images_scale": 2.0,
|
"images_scale": 2.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
await _run_analysis("j1", "/tmp/test.pdf", "test.pdf", opts)
|
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", opts)
|
||||||
|
|
||||||
mock_convert.assert_called_once()
|
mock_converter.convert.assert_called_once()
|
||||||
call_args = mock_convert.call_args
|
call_args = mock_converter.convert.call_args
|
||||||
assert call_args[0][0] == "/tmp/test.pdf"
|
assert call_args[0][0] == "/tmp/test.pdf"
|
||||||
conv_opts = call_args[0][1]
|
conv_opts = call_args[0][1]
|
||||||
assert conv_opts.do_ocr is False
|
assert conv_opts.do_ocr is False
|
||||||
|
|
@ -395,47 +397,50 @@ class TestServiceForwardsPipelineOptions:
|
||||||
|
|
||||||
@patch("services.analysis_service.analysis_repo")
|
@patch("services.analysis_service.analysis_repo")
|
||||||
@patch("services.analysis_service.document_repo")
|
@patch("services.analysis_service.document_repo")
|
||||||
@patch("services.analysis_service.convert_document")
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_run_analysis_uses_defaults_when_no_options(
|
async def test_run_analysis_uses_defaults_when_no_options(
|
||||||
self, mock_convert, mock_doc_repo, mock_analysis_repo, mock_job,
|
self, mock_doc_repo, mock_analysis_repo, mock_job,
|
||||||
):
|
):
|
||||||
from domain.parsing import ConversionResult, PageDetail
|
from domain.value_objects import ConversionResult, PageDetail
|
||||||
|
|
||||||
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
||||||
mock_analysis_repo.update_status = AsyncMock()
|
mock_analysis_repo.update_status = AsyncMock()
|
||||||
mock_doc_repo.update_page_count = AsyncMock()
|
mock_doc_repo.update_page_count = AsyncMock()
|
||||||
mock_convert.return_value = ConversionResult(
|
|
||||||
|
mock_converter = AsyncMock()
|
||||||
|
mock_converter.convert.return_value = ConversionResult(
|
||||||
page_count=1,
|
page_count=1,
|
||||||
content_markdown="",
|
content_markdown="",
|
||||||
content_html="",
|
content_html="",
|
||||||
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
|
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
|
||||||
)
|
)
|
||||||
|
|
||||||
from services.analysis_service import _run_analysis
|
from services.analysis_service import AnalysisService
|
||||||
|
svc = AnalysisService(converter=mock_converter)
|
||||||
|
|
||||||
await _run_analysis("j1", "/tmp/test.pdf", "test.pdf", None)
|
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", None)
|
||||||
|
|
||||||
# Called with file_path and default ConversionOptions
|
mock_converter.convert.assert_called_once()
|
||||||
mock_convert.assert_called_once()
|
call_args = mock_converter.convert.call_args
|
||||||
call_args = mock_convert.call_args
|
|
||||||
assert call_args[0][0] == "/tmp/test.pdf"
|
assert call_args[0][0] == "/tmp/test.pdf"
|
||||||
assert call_args[0][1] == ConversionOptions()
|
assert call_args[0][1] == ConversionOptions()
|
||||||
|
|
||||||
@patch("services.analysis_service.analysis_repo")
|
@patch("services.analysis_service.analysis_repo")
|
||||||
@patch("services.analysis_service.document_repo")
|
@patch("services.analysis_service.document_repo")
|
||||||
@patch("services.analysis_service.convert_document")
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_run_analysis_marks_failed_on_error(
|
async def test_run_analysis_marks_failed_on_error(
|
||||||
self, mock_convert, mock_doc_repo, mock_analysis_repo, mock_job,
|
self, mock_doc_repo, mock_analysis_repo, mock_job,
|
||||||
):
|
):
|
||||||
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
|
||||||
mock_analysis_repo.update_status = AsyncMock()
|
mock_analysis_repo.update_status = AsyncMock()
|
||||||
mock_convert.side_effect = RuntimeError("Docling crashed")
|
|
||||||
|
|
||||||
from services.analysis_service import _run_analysis
|
mock_converter = AsyncMock()
|
||||||
|
mock_converter.convert.side_effect = RuntimeError("Docling crashed")
|
||||||
|
|
||||||
await _run_analysis("j1", "/tmp/test.pdf", "test.pdf", {"do_ocr": False})
|
from services.analysis_service import AnalysisService
|
||||||
|
svc = AnalysisService(converter=mock_converter)
|
||||||
|
|
||||||
|
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", {"do_ocr": False})
|
||||||
|
|
||||||
# Should have called update_status twice: RUNNING then FAILED
|
# Should have called update_status twice: RUNNING then FAILED
|
||||||
assert mock_analysis_repo.update_status.call_count == 2
|
assert mock_analysis_repo.update_status.call_count == 2
|
||||||
|
|
@ -458,26 +463,35 @@ class TestAnalysisEndpointPipelineOptions:
|
||||||
from main import app
|
from main import app
|
||||||
return TestClient(app, raise_server_exceptions=False)
|
return TestClient(app, raise_server_exceptions=False)
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
@pytest.fixture
|
||||||
def test_no_pipeline_options_sends_none(self, mock_create, client):
|
def mock_svc(self, client):
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from main import app
|
||||||
|
mock = MagicMock()
|
||||||
|
original = getattr(app.state, "analysis_service", None)
|
||||||
|
app.state.analysis_service = mock
|
||||||
|
yield mock
|
||||||
|
app.state.analysis_service = original
|
||||||
|
|
||||||
|
def test_no_pipeline_options_sends_none(self, client, mock_svc):
|
||||||
from domain.models import AnalysisJob
|
from domain.models import AnalysisJob
|
||||||
mock_create.return_value = AnalysisJob(id="j1", document_id="d1")
|
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||||
|
|
||||||
client.post("/api/analyses", json={"documentId": "d1"})
|
client.post("/api/analyses", json={"documentId": "d1"})
|
||||||
|
|
||||||
mock_create.assert_called_once_with("d1", pipeline_options=None)
|
mock_svc.create.assert_called_once_with("d1", pipeline_options=None)
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_empty_pipeline_options_object_uses_defaults(self, client, mock_svc):
|
||||||
def test_empty_pipeline_options_object_uses_defaults(self, mock_create, client):
|
|
||||||
from domain.models import AnalysisJob
|
from domain.models import AnalysisJob
|
||||||
mock_create.return_value = AnalysisJob(id="j1", document_id="d1")
|
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||||
|
|
||||||
client.post("/api/analyses", json={
|
client.post("/api/analyses", json={
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
"pipelineOptions": {},
|
"pipelineOptions": {},
|
||||||
})
|
})
|
||||||
|
|
||||||
opts = mock_create.call_args.kwargs["pipeline_options"]
|
opts = mock_svc.create.call_args.kwargs["pipeline_options"]
|
||||||
assert opts["do_ocr"] is True
|
assert opts["do_ocr"] is True
|
||||||
assert opts["do_table_structure"] is True
|
assert opts["do_table_structure"] is True
|
||||||
assert opts["table_mode"] == "accurate"
|
assert opts["table_mode"] == "accurate"
|
||||||
|
|
@ -485,20 +499,18 @@ class TestAnalysisEndpointPipelineOptions:
|
||||||
assert opts["do_formula_enrichment"] is False
|
assert opts["do_formula_enrichment"] is False
|
||||||
assert opts["images_scale"] == 1.0
|
assert opts["images_scale"] == 1.0
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_partial_pipeline_options_merges_with_defaults(self, client, mock_svc):
|
||||||
def test_partial_pipeline_options_merges_with_defaults(self, mock_create, client):
|
|
||||||
from domain.models import AnalysisJob
|
from domain.models import AnalysisJob
|
||||||
mock_create.return_value = AnalysisJob(id="j1", document_id="d1")
|
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||||
|
|
||||||
client.post("/api/analyses", json={
|
client.post("/api/analyses", json={
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
"pipelineOptions": {"do_ocr": False, "images_scale": 1.5},
|
"pipelineOptions": {"do_ocr": False, "images_scale": 1.5},
|
||||||
})
|
})
|
||||||
|
|
||||||
opts = mock_create.call_args.kwargs["pipeline_options"]
|
opts = mock_svc.create.call_args.kwargs["pipeline_options"]
|
||||||
assert opts["do_ocr"] is False
|
assert opts["do_ocr"] is False
|
||||||
assert opts["images_scale"] == 1.5
|
assert opts["images_scale"] == 1.5
|
||||||
# All other fields should have defaults
|
|
||||||
assert opts["do_table_structure"] is True
|
assert opts["do_table_structure"] is True
|
||||||
assert opts["table_mode"] == "accurate"
|
assert opts["table_mode"] == "accurate"
|
||||||
assert opts["do_code_enrichment"] is False
|
assert opts["do_code_enrichment"] is False
|
||||||
|
|
@ -508,10 +520,9 @@ class TestAnalysisEndpointPipelineOptions:
|
||||||
assert opts["generate_picture_images"] is False
|
assert opts["generate_picture_images"] is False
|
||||||
assert opts["generate_page_images"] is False
|
assert opts["generate_page_images"] is False
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_full_pipeline_options(self, client, mock_svc):
|
||||||
def test_full_pipeline_options(self, mock_create, client):
|
|
||||||
from domain.models import AnalysisJob
|
from domain.models import AnalysisJob
|
||||||
mock_create.return_value = AnalysisJob(id="j1", document_id="d1")
|
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
|
|
@ -532,25 +543,22 @@ class TestAnalysisEndpointPipelineOptions:
|
||||||
resp = client.post("/api/analyses", json=payload)
|
resp = client.post("/api/analyses", json=payload)
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
|
|
||||||
opts = mock_create.call_args.kwargs["pipeline_options"]
|
opts = mock_svc.create.call_args.kwargs["pipeline_options"]
|
||||||
assert opts == payload["pipelineOptions"]
|
assert opts == payload["pipelineOptions"]
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_invalid_pipeline_option_type_rejected(self, client, mock_svc):
|
||||||
def test_invalid_pipeline_option_type_rejected(self, mock_create, client):
|
|
||||||
resp = client.post("/api/analyses", json={
|
resp = client.post("/api/analyses", json={
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
"pipelineOptions": {"do_ocr": "not-a-bool"},
|
"pipelineOptions": {"do_ocr": "not-a-bool"},
|
||||||
})
|
})
|
||||||
assert resp.status_code == 422
|
assert resp.status_code == 422
|
||||||
|
|
||||||
@patch("services.analysis_service.create", new_callable=AsyncMock)
|
def test_unknown_pipeline_option_ignored(self, client, mock_svc):
|
||||||
def test_unknown_pipeline_option_ignored(self, mock_create, client):
|
|
||||||
from domain.models import AnalysisJob
|
from domain.models import AnalysisJob
|
||||||
mock_create.return_value = AnalysisJob(id="j1", document_id="d1")
|
mock_svc.create = AsyncMock(return_value=AnalysisJob(id="j1", document_id="d1"))
|
||||||
|
|
||||||
resp = client.post("/api/analyses", json={
|
resp = client.post("/api/analyses", json={
|
||||||
"documentId": "d1",
|
"documentId": "d1",
|
||||||
"pipelineOptions": {"do_ocr": True, "unknown_field": True},
|
"pipelineOptions": {"do_ocr": True, "unknown_field": True},
|
||||||
})
|
})
|
||||||
# Pydantic ignores extra fields by default
|
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
|
|
|
||||||
387
document-parser/tests/test_serve_converter.py
Normal file
387
document-parser/tests/test_serve_converter.py
Normal file
|
|
@ -0,0 +1,387 @@
|
||||||
|
"""Tests for the ServeConverter adapter (Docling Serve HTTP client)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from domain.value_objects import ConversionOptions, ConversionResult
|
||||||
|
from infra.serve_converter import (
|
||||||
|
ServeConverter,
|
||||||
|
_build_form_data,
|
||||||
|
_extract_bbox,
|
||||||
|
_parse_response,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unit tests — form data building
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestBuildFormData:
|
||||||
|
def test_default_options(self):
|
||||||
|
data = _build_form_data(ConversionOptions())
|
||||||
|
assert data["do_ocr"] == "true"
|
||||||
|
assert data["do_table_structure"] == "true"
|
||||||
|
assert data["table_mode"] == "accurate"
|
||||||
|
assert data["do_code_enrichment"] == "false"
|
||||||
|
assert data["do_formula_enrichment"] == "false"
|
||||||
|
assert data["do_picture_classification"] == "false"
|
||||||
|
assert data["do_picture_description"] == "false"
|
||||||
|
assert data["include_images"] == "false"
|
||||||
|
assert data["images_scale"] == "1.0"
|
||||||
|
assert set(data["to_formats"]) == {"md", "html", "json"}
|
||||||
|
|
||||||
|
def test_custom_options(self):
|
||||||
|
opts = ConversionOptions(
|
||||||
|
do_ocr=False, table_mode="fast", images_scale=2.0,
|
||||||
|
generate_picture_images=True,
|
||||||
|
)
|
||||||
|
data = _build_form_data(opts)
|
||||||
|
assert data["do_ocr"] == "false"
|
||||||
|
assert data["table_mode"] == "fast"
|
||||||
|
assert data["images_scale"] == "2.0"
|
||||||
|
assert data["include_images"] == "true"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unit tests — response parsing
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestParseResponse:
|
||||||
|
def test_minimal_response(self):
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "# Hello",
|
||||||
|
"html_content": "<h1>Hello</h1>",
|
||||||
|
"json_content": {
|
||||||
|
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||||
|
"texts": [],
|
||||||
|
"tables": [],
|
||||||
|
"pictures": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
assert isinstance(result, ConversionResult)
|
||||||
|
assert result.content_markdown == "# Hello"
|
||||||
|
assert result.content_html == "<h1>Hello</h1>"
|
||||||
|
assert result.page_count == 1
|
||||||
|
assert result.pages[0].width == 612.0
|
||||||
|
|
||||||
|
def test_response_with_elements(self):
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "# Title\nText",
|
||||||
|
"html_content": "<h1>Title</h1><p>Text</p>",
|
||||||
|
"json_content": {
|
||||||
|
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||||
|
"texts": [
|
||||||
|
{
|
||||||
|
"label": "title",
|
||||||
|
"text": "Title",
|
||||||
|
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40, "coord_origin": "TOPLEFT"}}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "paragraph",
|
||||||
|
"text": "Text",
|
||||||
|
"prov": [{"page_no": 1, "bbox": {"l": 10, "t": 50, "r": 200, "b": 70, "coord_origin": "TOPLEFT"}}],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"tables": [],
|
||||||
|
"pictures": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
assert len(result.pages[0].elements) == 2
|
||||||
|
assert result.pages[0].elements[0].type == "title"
|
||||||
|
assert result.pages[0].elements[0].content == "Title"
|
||||||
|
assert result.pages[0].elements[0].bbox == [10, 20, 200, 40]
|
||||||
|
assert result.pages[0].elements[1].type == "text"
|
||||||
|
|
||||||
|
def test_multi_page(self):
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "",
|
||||||
|
"html_content": "",
|
||||||
|
"json_content": {
|
||||||
|
"pages": {
|
||||||
|
"1": {"size": {"width": 612.0, "height": 792.0}},
|
||||||
|
"2": {"size": {"width": 595.0, "height": 842.0}},
|
||||||
|
},
|
||||||
|
"texts": [], "tables": [], "pictures": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
assert result.page_count == 2
|
||||||
|
assert result.pages[1].width == 595.0
|
||||||
|
|
||||||
|
def test_no_json_content(self):
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "text",
|
||||||
|
"html_content": "<p>text</p>",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
assert result.content_markdown == "text"
|
||||||
|
assert result.pages == []
|
||||||
|
assert result.page_count == 1
|
||||||
|
|
||||||
|
def test_json_content_as_string(self):
|
||||||
|
json_doc = {
|
||||||
|
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||||
|
"texts": [], "tables": [], "pictures": [],
|
||||||
|
}
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "",
|
||||||
|
"html_content": "",
|
||||||
|
"json_content": json.dumps(json_doc),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
assert result.page_count == 1
|
||||||
|
|
||||||
|
def test_json_content_malformed_string_falls_back(self):
|
||||||
|
"""Bug #5: malformed JSON string in json_content must not crash."""
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "# Hello",
|
||||||
|
"html_content": "<h1>Hello</h1>",
|
||||||
|
"json_content": "NOT VALID JSON {{{",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
assert isinstance(result, ConversionResult)
|
||||||
|
assert result.content_markdown == "# Hello"
|
||||||
|
assert result.pages == []
|
||||||
|
assert result.page_count == 1
|
||||||
|
|
||||||
|
def test_tables_and_pictures(self):
|
||||||
|
data = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "",
|
||||||
|
"html_content": "",
|
||||||
|
"json_content": {
|
||||||
|
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||||
|
"texts": [],
|
||||||
|
"tables": [
|
||||||
|
{"label": "table", "text": "", "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 10, "r": 300, "b": 200, "coord_origin": "TOPLEFT"}}]},
|
||||||
|
],
|
||||||
|
"pictures": [
|
||||||
|
{"label": "picture", "text": "", "prov": [{"page_no": 1, "bbox": {"l": 50, "t": 300, "r": 250, "b": 500, "coord_origin": "TOPLEFT"}}]},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = _parse_response(data)
|
||||||
|
types = [e.type for e in result.pages[0].elements]
|
||||||
|
assert "table" in types
|
||||||
|
assert "picture" in types
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unit tests — bbox extraction
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestExtractBbox:
|
||||||
|
def test_topleft_passthrough(self):
|
||||||
|
bbox = _extract_bbox({"l": 10, "t": 20, "r": 100, "b": 50, "coord_origin": "TOPLEFT"}, 792.0)
|
||||||
|
assert bbox == [10, 20, 100, 50]
|
||||||
|
|
||||||
|
def test_bottomleft_conversion(self):
|
||||||
|
# In BOTTOMLEFT: t (top of box) has higher y than b (bottom of box)
|
||||||
|
bbox = _extract_bbox({"l": 10, "t": 772, "r": 100, "b": 742, "coord_origin": "BOTTOMLEFT"}, 792.0)
|
||||||
|
# new_top = 792 - 772 = 20, new_bottom = 792 - 742 = 50
|
||||||
|
assert bbox == [10, 20, 100, 50]
|
||||||
|
|
||||||
|
def test_missing_coord_origin_defaults_topleft(self):
|
||||||
|
bbox = _extract_bbox({"l": 10, "t": 20, "r": 100, "b": 50}, 792.0)
|
||||||
|
assert bbox == [10, 20, 100, 50]
|
||||||
|
|
||||||
|
def test_empty_dict(self):
|
||||||
|
bbox = _extract_bbox({}, 792.0)
|
||||||
|
assert bbox == [0.0, 0.0, 0.0, 0.0]
|
||||||
|
|
||||||
|
def test_non_dict_returns_zeros(self):
|
||||||
|
bbox = _extract_bbox("invalid", 792.0)
|
||||||
|
assert bbox == [0.0, 0.0, 0.0, 0.0]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unit tests — label mapping
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestLabelMapping:
|
||||||
|
def test_known_labels(self):
|
||||||
|
from infra.serve_converter import _LABEL_MAP
|
||||||
|
assert _LABEL_MAP["table"] == "table"
|
||||||
|
assert _LABEL_MAP["picture"] == "picture"
|
||||||
|
assert _LABEL_MAP["figure"] == "picture"
|
||||||
|
assert _LABEL_MAP["title"] == "title"
|
||||||
|
assert _LABEL_MAP["section_header"] == "section_header"
|
||||||
|
assert _LABEL_MAP["list_item"] == "list"
|
||||||
|
assert _LABEL_MAP["formula"] == "formula"
|
||||||
|
assert _LABEL_MAP["code"] == "code"
|
||||||
|
assert _LABEL_MAP["paragraph"] == "text"
|
||||||
|
|
||||||
|
def test_unknown_label_defaults_to_text(self):
|
||||||
|
from infra.serve_converter import _LABEL_MAP
|
||||||
|
assert _LABEL_MAP.get("unknown_thing", "text") == "text"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unit tests — ServeConverter
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestServeConverter:
|
||||||
|
def test_headers_with_api_key(self):
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001", api_key="secret")
|
||||||
|
assert conv._headers() == {"X-Api-Key": "secret"}
|
||||||
|
|
||||||
|
def test_headers_without_api_key(self):
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001")
|
||||||
|
assert conv._headers() == {}
|
||||||
|
|
||||||
|
def test_base_url_trailing_slash_stripped(self):
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001/")
|
||||||
|
assert conv._base_url == "http://localhost:5001"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Integration tests — HTTP calls (mocked)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestServeConverterConvert:
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_successful_conversion(self, tmp_path):
|
||||||
|
test_file = tmp_path / "test.pdf"
|
||||||
|
test_file.write_bytes(b"%PDF-1.4 fake content")
|
||||||
|
|
||||||
|
serve_response = {
|
||||||
|
"document": {
|
||||||
|
"md_content": "# Converted",
|
||||||
|
"html_content": "<h1>Converted</h1>",
|
||||||
|
"json_content": {
|
||||||
|
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}}},
|
||||||
|
"texts": [
|
||||||
|
{"label": "title", "text": "Converted", "prov": [{"page_no": 1, "bbox": {"l": 10, "t": 20, "r": 200, "b": 40, "coord_origin": "TOPLEFT"}}]},
|
||||||
|
],
|
||||||
|
"tables": [],
|
||||||
|
"pictures": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
mock_response.json.return_value = serve_response
|
||||||
|
mock_response.raise_for_status = MagicMock()
|
||||||
|
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.post.return_value = mock_response
|
||||||
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||||
|
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||||
|
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001", api_key="test-key")
|
||||||
|
|
||||||
|
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||||
|
result = await conv.convert(str(test_file), ConversionOptions())
|
||||||
|
|
||||||
|
assert isinstance(result, ConversionResult)
|
||||||
|
assert result.content_markdown == "# Converted"
|
||||||
|
assert result.page_count == 1
|
||||||
|
assert len(result.pages[0].elements) == 1
|
||||||
|
assert result.pages[0].elements[0].type == "title"
|
||||||
|
|
||||||
|
# Verify form fields sent as dict with list for repeated keys
|
||||||
|
call_kwargs = mock_client.post.call_args
|
||||||
|
sent_data = call_kwargs.kwargs.get("data", {})
|
||||||
|
assert sent_data["do_ocr"] == "true"
|
||||||
|
assert set(sent_data["to_formats"]) == {"md", "html", "json"}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_http_error_raises(self, tmp_path):
|
||||||
|
test_file = tmp_path / "test.pdf"
|
||||||
|
test_file.write_bytes(b"%PDF-1.4 fake content")
|
||||||
|
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||||
|
"Server Error", request=MagicMock(), response=MagicMock(status_code=500),
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.post.return_value = mock_response
|
||||||
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||||
|
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||||
|
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001")
|
||||||
|
|
||||||
|
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client), \
|
||||||
|
pytest.raises(httpx.HTTPStatusError):
|
||||||
|
await conv.convert(str(test_file), ConversionOptions())
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_health_check_success(self):
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.get.return_value = mock_response
|
||||||
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||||
|
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||||
|
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001")
|
||||||
|
|
||||||
|
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||||
|
assert await conv.health_check() is True
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_health_check_failure(self):
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.get.side_effect = httpx.ConnectError("Connection refused")
|
||||||
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||||
|
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||||
|
|
||||||
|
conv = ServeConverter(base_url="http://localhost:5001")
|
||||||
|
|
||||||
|
with patch("infra.serve_converter.httpx.AsyncClient", return_value=mock_client):
|
||||||
|
assert await conv.health_check() is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Integration — converter wiring in main.py
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestConverterWiring:
|
||||||
|
def test_local_engine_builds_local_converter(self):
|
||||||
|
from infra.local_converter import LocalConverter
|
||||||
|
from infra.settings import Settings
|
||||||
|
|
||||||
|
with patch("main.settings", Settings(conversion_engine="local")):
|
||||||
|
from main import _build_converter
|
||||||
|
converter = _build_converter()
|
||||||
|
assert isinstance(converter, LocalConverter)
|
||||||
|
|
||||||
|
def test_remote_engine_builds_serve_converter(self):
|
||||||
|
from infra.settings import Settings
|
||||||
|
|
||||||
|
with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001")):
|
||||||
|
from main import _build_converter
|
||||||
|
converter = _build_converter()
|
||||||
|
assert isinstance(converter, ServeConverter)
|
||||||
|
assert converter._base_url == "http://serve:5001"
|
||||||
|
|
||||||
|
def test_remote_engine_passes_api_key(self):
|
||||||
|
from infra.settings import Settings
|
||||||
|
|
||||||
|
with patch("main.settings", Settings(conversion_engine="remote", docling_serve_url="http://serve:5001", docling_serve_api_key="my-key")):
|
||||||
|
from main import _build_converter
|
||||||
|
converter = _build_converter()
|
||||||
|
assert isinstance(converter, ServeConverter)
|
||||||
|
assert converter._api_key == "my-key"
|
||||||
|
|
@ -9,6 +9,8 @@ export const useAnalysisStore = defineStore('analysis', () => {
|
||||||
const running = ref(false)
|
const running = ref(false)
|
||||||
const error = ref<string | null>(null)
|
const error = ref<string | null>(null)
|
||||||
const pollingInterval = ref<ReturnType<typeof setInterval> | null>(null)
|
const pollingInterval = ref<ReturnType<typeof setInterval> | null>(null)
|
||||||
|
const pollingTimeout = ref<ReturnType<typeof setTimeout> | null>(null)
|
||||||
|
const MAX_POLLING_DURATION = 10 * 60 * 1000 // 10 minutes
|
||||||
|
|
||||||
const currentPages = computed<Page[]>(() => {
|
const currentPages = computed<Page[]>(() => {
|
||||||
if (!currentAnalysis.value?.pagesJson) return []
|
if (!currentAnalysis.value?.pagesJson) return []
|
||||||
|
|
@ -72,6 +74,13 @@ export const useAnalysisStore = defineStore('analysis', () => {
|
||||||
running.value = false
|
running.value = false
|
||||||
}
|
}
|
||||||
}, 2000)
|
}, 2000)
|
||||||
|
pollingTimeout.value = setTimeout(() => {
|
||||||
|
if (pollingInterval.value) {
|
||||||
|
error.value = 'Analysis timed out'
|
||||||
|
stopPolling()
|
||||||
|
running.value = false
|
||||||
|
}
|
||||||
|
}, MAX_POLLING_DURATION)
|
||||||
}
|
}
|
||||||
|
|
||||||
function stopPolling(): void {
|
function stopPolling(): void {
|
||||||
|
|
@ -79,6 +88,10 @@ export const useAnalysisStore = defineStore('analysis', () => {
|
||||||
clearInterval(pollingInterval.value)
|
clearInterval(pollingInterval.value)
|
||||||
pollingInterval.value = null
|
pollingInterval.value = null
|
||||||
}
|
}
|
||||||
|
if (pollingTimeout.value) {
|
||||||
|
clearTimeout(pollingTimeout.value)
|
||||||
|
pollingTimeout.value = null
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function select(id: string): Promise<void> {
|
async function select(id: string): Promise<void> {
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,8 @@ import { ref } from 'vue'
|
||||||
import type { Document } from '../../shared/types'
|
import type { Document } from '../../shared/types'
|
||||||
import * as api from './api'
|
import * as api from './api'
|
||||||
|
|
||||||
|
const MAX_FILE_SIZE = 50 * 1024 * 1024 // 50 MB
|
||||||
|
|
||||||
export const useDocumentStore = defineStore('document', () => {
|
export const useDocumentStore = defineStore('document', () => {
|
||||||
const documents = ref<Document[]>([])
|
const documents = ref<Document[]>([])
|
||||||
const selectedId = ref<string | null>(null)
|
const selectedId = ref<string | null>(null)
|
||||||
|
|
@ -24,6 +26,10 @@ export const useDocumentStore = defineStore('document', () => {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function upload(file: File): Promise<Document> {
|
async function upload(file: File): Promise<Document> {
|
||||||
|
if (file.size > MAX_FILE_SIZE) {
|
||||||
|
error.value = 'File too large (max 50 MB)'
|
||||||
|
throw new Error(error.value)
|
||||||
|
}
|
||||||
uploading.value = true
|
uploading.value = true
|
||||||
error.value = null
|
error.value = null
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@
|
||||||
</svg>
|
</svg>
|
||||||
<span class="upload-text">{{ t('upload.drop') }}</span>
|
<span class="upload-text">{{ t('upload.drop') }}</span>
|
||||||
<span class="upload-hint">{{ t('upload.maxSize') }}</span>
|
<span class="upload-hint">{{ t('upload.maxSize') }}</span>
|
||||||
|
<span v-if="store.error" class="upload-error">{{ store.error }}</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
@ -42,8 +43,13 @@ async function onFileSelect(e: Event) {
|
||||||
const target = e.target as HTMLInputElement
|
const target = e.target as HTMLInputElement
|
||||||
const file = target.files?.[0]
|
const file = target.files?.[0]
|
||||||
if (file) {
|
if (file) {
|
||||||
const doc = await store.upload(file)
|
try {
|
||||||
if (doc) emit('uploaded', doc.id)
|
store.clearError()
|
||||||
|
const doc = await store.upload(file)
|
||||||
|
if (doc) emit('uploaded', doc.id)
|
||||||
|
} catch {
|
||||||
|
// error is already set in store.upload
|
||||||
|
}
|
||||||
}
|
}
|
||||||
target.value = ''
|
target.value = ''
|
||||||
}
|
}
|
||||||
|
|
@ -51,9 +57,14 @@ async function onFileSelect(e: Event) {
|
||||||
async function onDrop(e: DragEvent) {
|
async function onDrop(e: DragEvent) {
|
||||||
dragging.value = false
|
dragging.value = false
|
||||||
const file = e.dataTransfer?.files?.[0]
|
const file = e.dataTransfer?.files?.[0]
|
||||||
if (file && file.type === 'application/pdf') {
|
if (file && (file.type === 'application/pdf' || file.name.toLowerCase().endsWith('.pdf'))) {
|
||||||
const doc = await store.upload(file)
|
try {
|
||||||
if (doc) emit('uploaded', doc.id)
|
store.clearError()
|
||||||
|
const doc = await store.upload(file)
|
||||||
|
if (doc) emit('uploaded', doc.id)
|
||||||
|
} catch {
|
||||||
|
// error is already set in store.upload
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
@ -103,6 +114,12 @@ async function onDrop(e: DragEvent) {
|
||||||
color: var(--text-muted);
|
color: var(--text-muted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.upload-error {
|
||||||
|
font-size: 13px;
|
||||||
|
color: var(--error, #e53e3e);
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
.spinner {
|
.spinner {
|
||||||
width: 24px;
|
width: 24px;
|
||||||
height: 24px;
|
height: 24px;
|
||||||
|
|
|
||||||
|
|
@ -2,13 +2,29 @@ import { defineStore } from 'pinia'
|
||||||
import { ref, watch, watchEffect } from 'vue'
|
import { ref, watch, watchEffect } from 'vue'
|
||||||
import type { Locale, Theme } from '../../shared/types'
|
import type { Locale, Theme } from '../../shared/types'
|
||||||
|
|
||||||
|
function safeGetItem(key: string): string | null {
|
||||||
|
try {
|
||||||
|
return localStorage.getItem(key)
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeSetItem(key: string, value: string): void {
|
||||||
|
try {
|
||||||
|
localStorage.setItem(key, value)
|
||||||
|
} catch {
|
||||||
|
// localStorage unavailable (private browsing, quota exceeded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export const useSettingsStore = defineStore('settings', () => {
|
export const useSettingsStore = defineStore('settings', () => {
|
||||||
const apiUrl = ref('http://localhost:8000')
|
const apiUrl = ref('http://localhost:8000')
|
||||||
const theme = ref<Theme>((localStorage.getItem('docling-theme') as Theme) || 'dark')
|
const theme = ref<Theme>((safeGetItem('docling-theme') as Theme) || 'dark')
|
||||||
const locale = ref<Locale>((localStorage.getItem('docling-locale') as Locale) || 'fr')
|
const locale = ref<Locale>((safeGetItem('docling-locale') as Locale) || 'fr')
|
||||||
|
|
||||||
watch(theme, (v) => localStorage.setItem('docling-theme', v))
|
watch(theme, (v) => safeSetItem('docling-theme', v))
|
||||||
watch(locale, (v) => localStorage.setItem('docling-locale', v))
|
watch(locale, (v) => safeSetItem('docling-locale', v))
|
||||||
|
|
||||||
watchEffect(() => {
|
watchEffect(() => {
|
||||||
document.documentElement.classList.toggle('light', theme.value === 'light')
|
document.documentElement.classList.toggle('light', theme.value === 'light')
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue