docling-serve chunker implementation. Lacks tests
This commit is contained in:
parent
75cbdb47a8
commit
91de82ec14
1 changed files with 86 additions and 6 deletions
|
|
@ -1,7 +1,10 @@
|
||||||
|
from io import BytesIO
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
from haiku.rag.chunkers.base import DocumentChunker
|
from haiku.rag.chunkers.base import DocumentChunker
|
||||||
from haiku.rag.config import AppConfig
|
from haiku.rag.config import AppConfig, Config
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from docling_core.types.doc.document import DoclingDocument
|
from docling_core.types.doc.document import DoclingDocument
|
||||||
|
|
@ -10,15 +13,26 @@ if TYPE_CHECKING:
|
||||||
class DoclingServeChunker(DocumentChunker):
|
class DoclingServeChunker(DocumentChunker):
|
||||||
"""Remote document chunker using docling-serve API.
|
"""Remote document chunker using docling-serve API.
|
||||||
|
|
||||||
Placeholder - will be implemented in a future commit.
|
Sends DoclingDocument JSON to docling-serve for chunking. Supports both hybrid
|
||||||
|
and hierarchical chunking strategies via remote API.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
config: Application configuration containing docling-serve settings.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, config: AppConfig):
|
def __init__(self, config: AppConfig = Config):
|
||||||
raise NotImplementedError("DoclingServeChunker not yet implemented")
|
self.config = config
|
||||||
|
self.base_url = config.providers.docling_serve.base_url.rstrip("/")
|
||||||
|
self.api_key = config.providers.docling_serve.api_key
|
||||||
|
self.timeout = config.providers.docling_serve.timeout
|
||||||
|
self.chunker_type = config.processing.chunker_type
|
||||||
|
|
||||||
async def chunk(self, document: "DoclingDocument") -> list[str]:
|
async def chunk(self, document: "DoclingDocument") -> list[str]:
|
||||||
"""Split the document into chunks via docling-serve.
|
"""Split the document into chunks via docling-serve.
|
||||||
|
|
||||||
|
Exports the DoclingDocument to JSON and sends it to docling-serve's chunking
|
||||||
|
endpoint. The API will chunk the document and return the text chunks.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
document: The DoclingDocument to be split into chunks.
|
document: The DoclingDocument to be split into chunks.
|
||||||
|
|
||||||
|
|
@ -26,6 +40,72 @@ class DoclingServeChunker(DocumentChunker):
|
||||||
A list of text chunks with semantic boundaries.
|
A list of text chunks with semantic boundaries.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
NotImplementedError: This chunker is not yet implemented.
|
ValueError: If chunking fails or service is unavailable.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError("DoclingServeChunker not yet implemented")
|
if document is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Determine endpoint based on chunker_type
|
||||||
|
if self.chunker_type == "hierarchical":
|
||||||
|
url = f"{self.base_url}/v1/chunk/hierarchical/file"
|
||||||
|
else:
|
||||||
|
url = f"{self.base_url}/v1/chunk/hybrid/file"
|
||||||
|
|
||||||
|
# Export document to JSON
|
||||||
|
doc_json = document.model_dump_json()
|
||||||
|
doc_bytes = doc_json.encode("utf-8")
|
||||||
|
|
||||||
|
# Prepare multipart request with DoclingDocument JSON
|
||||||
|
files = {"files": ("document.json", BytesIO(doc_bytes), "application/json")}
|
||||||
|
|
||||||
|
# Build form data with chunking parameters
|
||||||
|
data = {
|
||||||
|
"chunking_max_tokens": str(self.config.processing.chunk_size),
|
||||||
|
"chunking_tokenizer": self.config.processing.chunking_tokenizer,
|
||||||
|
"chunking_merge_peers": str(
|
||||||
|
self.config.processing.chunking_merge_peers
|
||||||
|
).lower(),
|
||||||
|
"chunking_use_markdown_tables": str(
|
||||||
|
self.config.processing.chunking_use_markdown_tables
|
||||||
|
).lower(),
|
||||||
|
}
|
||||||
|
|
||||||
|
headers = {}
|
||||||
|
if self.api_key:
|
||||||
|
headers["X-Api-Key"] = self.api_key
|
||||||
|
|
||||||
|
response = requests.post(
|
||||||
|
url,
|
||||||
|
files=files,
|
||||||
|
data=data,
|
||||||
|
headers=headers,
|
||||||
|
timeout=self.timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
result = response.json()
|
||||||
|
|
||||||
|
# Extract text from chunks
|
||||||
|
chunks = result.get("chunks", [])
|
||||||
|
return [chunk["text"] for chunk in chunks]
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError as e:
|
||||||
|
raise ValueError(
|
||||||
|
f"Could not connect to docling-serve at {self.base_url}. "
|
||||||
|
f"Ensure the service is running and accessible. Error: {e}"
|
||||||
|
)
|
||||||
|
except requests.exceptions.Timeout as e:
|
||||||
|
raise ValueError(
|
||||||
|
f"Request to docling-serve timed out after {self.timeout}s. "
|
||||||
|
f"Consider increasing the timeout in configuration. Error: {e}"
|
||||||
|
)
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
if e.response.status_code == 401:
|
||||||
|
raise ValueError(
|
||||||
|
"Authentication failed. Check your API key configuration."
|
||||||
|
)
|
||||||
|
raise ValueError(f"HTTP error from docling-serve: {e}")
|
||||||
|
except Exception as e:
|
||||||
|
raise ValueError(f"Failed to chunk via docling-serve: {e}")
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue