Add chunking domain objects and DocumentChunker port
ChunkingOptions and ChunkResult value objects, document_json/chunks_json fields on AnalysisJob, and DocumentChunker protocol for adapter injection.
This commit is contained in:
parent
d42c97160d
commit
78c27b087d
3 changed files with 52 additions and 3 deletions
|
|
@ -42,6 +42,8 @@ class AnalysisJob:
|
|||
content_markdown: str | None = None
|
||||
content_html: str | None = None
|
||||
pages_json: str | None = None
|
||||
document_json: str | None = None
|
||||
chunks_json: str | None = None
|
||||
error_message: str | None = None
|
||||
started_at: datetime | None = None
|
||||
completed_at: datetime | None = None
|
||||
|
|
@ -55,12 +57,19 @@ class AnalysisJob:
|
|||
self.started_at = _utcnow()
|
||||
|
||||
def mark_completed(
|
||||
self, markdown: str, html: str, pages_json: str,
|
||||
self,
|
||||
markdown: str,
|
||||
html: str,
|
||||
pages_json: str,
|
||||
document_json: str | None = None,
|
||||
chunks_json: str | None = None,
|
||||
) -> None:
|
||||
self.status = AnalysisStatus.COMPLETED
|
||||
self.content_markdown = markdown
|
||||
self.content_html = html
|
||||
self.pages_json = pages_json
|
||||
self.document_json = document_json
|
||||
self.chunks_json = chunks_json
|
||||
self.completed_at = _utcnow()
|
||||
|
||||
def mark_failed(self, error: str) -> None:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,12 @@ from __future__ import annotations
|
|||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from domain.value_objects import ConversionOptions, ConversionResult
|
||||
from domain.value_objects import (
|
||||
ChunkingOptions,
|
||||
ChunkResult,
|
||||
ConversionOptions,
|
||||
ConversionResult,
|
||||
)
|
||||
|
||||
|
||||
class DocumentConverter(Protocol):
|
||||
|
|
@ -20,5 +25,20 @@ class DocumentConverter(Protocol):
|
|||
"""
|
||||
|
||||
async def convert(
|
||||
self, file_path: str, options: ConversionOptions,
|
||||
self,
|
||||
file_path: str,
|
||||
options: ConversionOptions,
|
||||
) -> ConversionResult: ...
|
||||
|
||||
|
||||
class DocumentChunker(Protocol):
|
||||
"""Port for document chunking.
|
||||
|
||||
Takes a serialized DoclingDocument (JSON) and returns chunks.
|
||||
"""
|
||||
|
||||
async def chunk(
|
||||
self,
|
||||
document_json: str,
|
||||
options: ChunkingOptions,
|
||||
) -> list[ChunkResult]: ...
|
||||
|
|
|
|||
|
|
@ -49,3 +49,23 @@ class ConversionResult:
|
|||
content_html: str
|
||||
pages: list[PageDetail]
|
||||
skipped_items: int = 0
|
||||
document_json: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkingOptions:
|
||||
chunker_type: str = "hybrid" # "hybrid", "hierarchical", "page"
|
||||
max_tokens: int = 512
|
||||
merge_peers: bool = True
|
||||
repeat_table_header: bool = True
|
||||
|
||||
def is_default(self) -> bool:
|
||||
return self == ChunkingOptions()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkResult:
|
||||
text: str
|
||||
headings: list[str] = field(default_factory=list)
|
||||
source_page: int | None = None
|
||||
token_count: int = 0
|
||||
|
|
|
|||
Loading…
Reference in a new issue