Add chunking domain objects and DocumentChunker port
ChunkingOptions and ChunkResult value objects, document_json/chunks_json fields on AnalysisJob, and DocumentChunker protocol for adapter injection.
This commit is contained in:
parent
d42c97160d
commit
78c27b087d
3 changed files with 52 additions and 3 deletions
|
|
@ -42,6 +42,8 @@ class AnalysisJob:
|
||||||
content_markdown: str | None = None
|
content_markdown: str | None = None
|
||||||
content_html: str | None = None
|
content_html: str | None = None
|
||||||
pages_json: str | None = None
|
pages_json: str | None = None
|
||||||
|
document_json: str | None = None
|
||||||
|
chunks_json: str | None = None
|
||||||
error_message: str | None = None
|
error_message: str | None = None
|
||||||
started_at: datetime | None = None
|
started_at: datetime | None = None
|
||||||
completed_at: datetime | None = None
|
completed_at: datetime | None = None
|
||||||
|
|
@ -55,12 +57,19 @@ class AnalysisJob:
|
||||||
self.started_at = _utcnow()
|
self.started_at = _utcnow()
|
||||||
|
|
||||||
def mark_completed(
|
def mark_completed(
|
||||||
self, markdown: str, html: str, pages_json: str,
|
self,
|
||||||
|
markdown: str,
|
||||||
|
html: str,
|
||||||
|
pages_json: str,
|
||||||
|
document_json: str | None = None,
|
||||||
|
chunks_json: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.status = AnalysisStatus.COMPLETED
|
self.status = AnalysisStatus.COMPLETED
|
||||||
self.content_markdown = markdown
|
self.content_markdown = markdown
|
||||||
self.content_html = html
|
self.content_html = html
|
||||||
self.pages_json = pages_json
|
self.pages_json = pages_json
|
||||||
|
self.document_json = document_json
|
||||||
|
self.chunks_json = chunks_json
|
||||||
self.completed_at = _utcnow()
|
self.completed_at = _utcnow()
|
||||||
|
|
||||||
def mark_failed(self, error: str) -> None:
|
def mark_failed(self, error: str) -> None:
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,12 @@ from __future__ import annotations
|
||||||
from typing import TYPE_CHECKING, Protocol
|
from typing import TYPE_CHECKING, Protocol
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from domain.value_objects import ConversionOptions, ConversionResult
|
from domain.value_objects import (
|
||||||
|
ChunkingOptions,
|
||||||
|
ChunkResult,
|
||||||
|
ConversionOptions,
|
||||||
|
ConversionResult,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DocumentConverter(Protocol):
|
class DocumentConverter(Protocol):
|
||||||
|
|
@ -20,5 +25,20 @@ class DocumentConverter(Protocol):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def convert(
|
async def convert(
|
||||||
self, file_path: str, options: ConversionOptions,
|
self,
|
||||||
|
file_path: str,
|
||||||
|
options: ConversionOptions,
|
||||||
) -> ConversionResult: ...
|
) -> ConversionResult: ...
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentChunker(Protocol):
|
||||||
|
"""Port for document chunking.
|
||||||
|
|
||||||
|
Takes a serialized DoclingDocument (JSON) and returns chunks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def chunk(
|
||||||
|
self,
|
||||||
|
document_json: str,
|
||||||
|
options: ChunkingOptions,
|
||||||
|
) -> list[ChunkResult]: ...
|
||||||
|
|
|
||||||
|
|
@ -49,3 +49,23 @@ class ConversionResult:
|
||||||
content_html: str
|
content_html: str
|
||||||
pages: list[PageDetail]
|
pages: list[PageDetail]
|
||||||
skipped_items: int = 0
|
skipped_items: int = 0
|
||||||
|
document_json: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ChunkingOptions:
|
||||||
|
chunker_type: str = "hybrid" # "hybrid", "hierarchical", "page"
|
||||||
|
max_tokens: int = 512
|
||||||
|
merge_peers: bool = True
|
||||||
|
repeat_table_header: bool = True
|
||||||
|
|
||||||
|
def is_default(self) -> bool:
|
||||||
|
return self == ChunkingOptions()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ChunkResult:
|
||||||
|
text: str
|
||||||
|
headings: list[str] = field(default_factory=list)
|
||||||
|
source_page: int | None = None
|
||||||
|
token_count: int = 0
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue