diff --git a/document-parser/domain/models.py b/document-parser/domain/models.py index 3df9eb3..66fc3fe 100644 --- a/document-parser/domain/models.py +++ b/document-parser/domain/models.py @@ -42,6 +42,8 @@ class AnalysisJob: content_markdown: str | None = None content_html: str | None = None pages_json: str | None = None + document_json: str | None = None + chunks_json: str | None = None error_message: str | None = None started_at: datetime | None = None completed_at: datetime | None = None @@ -55,12 +57,19 @@ class AnalysisJob: self.started_at = _utcnow() def mark_completed( - self, markdown: str, html: str, pages_json: str, + self, + markdown: str, + html: str, + pages_json: str, + document_json: str | None = None, + chunks_json: str | None = None, ) -> None: self.status = AnalysisStatus.COMPLETED self.content_markdown = markdown self.content_html = html self.pages_json = pages_json + self.document_json = document_json + self.chunks_json = chunks_json self.completed_at = _utcnow() def mark_failed(self, error: str) -> None: diff --git a/document-parser/domain/ports.py b/document-parser/domain/ports.py index bc36c9b..f4ea64d 100644 --- a/document-parser/domain/ports.py +++ b/document-parser/domain/ports.py @@ -9,7 +9,12 @@ from __future__ import annotations from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: - from domain.value_objects import ConversionOptions, ConversionResult + from domain.value_objects import ( + ChunkingOptions, + ChunkResult, + ConversionOptions, + ConversionResult, + ) class DocumentConverter(Protocol): @@ -20,5 +25,20 @@ class DocumentConverter(Protocol): """ async def convert( - self, file_path: str, options: ConversionOptions, + self, + file_path: str, + options: ConversionOptions, ) -> ConversionResult: ... + + +class DocumentChunker(Protocol): + """Port for document chunking. + + Takes a serialized DoclingDocument (JSON) and returns chunks. + """ + + async def chunk( + self, + document_json: str, + options: ChunkingOptions, + ) -> list[ChunkResult]: ... diff --git a/document-parser/domain/value_objects.py b/document-parser/domain/value_objects.py index b4c5959..19af834 100644 --- a/document-parser/domain/value_objects.py +++ b/document-parser/domain/value_objects.py @@ -49,3 +49,23 @@ class ConversionResult: content_html: str pages: list[PageDetail] skipped_items: int = 0 + document_json: str | None = None + + +@dataclass +class ChunkingOptions: + chunker_type: str = "hybrid" # "hybrid", "hierarchical", "page" + max_tokens: int = 512 + merge_peers: bool = True + repeat_table_header: bool = True + + def is_default(self) -> bool: + return self == ChunkingOptions() + + +@dataclass +class ChunkResult: + text: str + headings: list[str] = field(default_factory=list) + source_page: int | None = None + token_count: int = 0