Add chunking domain objects and DocumentChunker port

ChunkingOptions and ChunkResult value objects, document_json/chunks_json
fields on AnalysisJob, and DocumentChunker protocol for adapter injection.
This commit is contained in:
Pier-Jean Malandrino 2026-04-02 12:05:05 +02:00
parent d42c97160d
commit 78c27b087d
3 changed files with 52 additions and 3 deletions

View file

@ -42,6 +42,8 @@ class AnalysisJob:
content_markdown: str | None = None
content_html: str | None = None
pages_json: str | None = None
document_json: str | None = None
chunks_json: str | None = None
error_message: str | None = None
started_at: datetime | None = None
completed_at: datetime | None = None
@ -55,12 +57,19 @@ class AnalysisJob:
self.started_at = _utcnow()
def mark_completed(
self, markdown: str, html: str, pages_json: str,
self,
markdown: str,
html: str,
pages_json: str,
document_json: str | None = None,
chunks_json: str | None = None,
) -> None:
self.status = AnalysisStatus.COMPLETED
self.content_markdown = markdown
self.content_html = html
self.pages_json = pages_json
self.document_json = document_json
self.chunks_json = chunks_json
self.completed_at = _utcnow()
def mark_failed(self, error: str) -> None:

View file

@ -9,7 +9,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Protocol
if TYPE_CHECKING:
from domain.value_objects import ConversionOptions, ConversionResult
from domain.value_objects import (
ChunkingOptions,
ChunkResult,
ConversionOptions,
ConversionResult,
)
class DocumentConverter(Protocol):
@ -20,5 +25,20 @@ class DocumentConverter(Protocol):
"""
async def convert(
self, file_path: str, options: ConversionOptions,
self,
file_path: str,
options: ConversionOptions,
) -> ConversionResult: ...
class DocumentChunker(Protocol):
"""Port for document chunking.
Takes a serialized DoclingDocument (JSON) and returns chunks.
"""
async def chunk(
self,
document_json: str,
options: ChunkingOptions,
) -> list[ChunkResult]: ...

View file

@ -49,3 +49,23 @@ class ConversionResult:
content_html: str
pages: list[PageDetail]
skipped_items: int = 0
document_json: str | None = None
@dataclass
class ChunkingOptions:
chunker_type: str = "hybrid" # "hybrid", "hierarchical", "page"
max_tokens: int = 512
merge_peers: bool = True
repeat_table_header: bool = True
def is_default(self) -> bool:
return self == ChunkingOptions()
@dataclass
class ChunkResult:
text: str
headings: list[str] = field(default_factory=list)
source_page: int | None = None
token_count: int = 0