Store docling-serve chunk bodies via raw_text
This commit is contained in:
parent
68715c9657
commit
3cc3c98986
6 changed files with 291370 additions and 92062 deletions
|
|
@ -15,6 +15,7 @@
|
|||
### Fixed
|
||||
|
||||
- `docling-serve` converter requests `include_page_images`, so `generate_page_images` produces page rasters on docling-serve versions that gate them behind that flag.
|
||||
- `docling-serve` chunker stores chunk bodies via `raw_text`, so section headings aren't duplicated into chunk content on docling-serve versions where the chunk `text` field is heading-contextualized.
|
||||
|
||||
## [0.60.0] - 2026-06-22
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ docling-serve is a REST API service that provides:
|
|||
|
||||
## Setup
|
||||
|
||||
haiku.rag is tested against docling-serve 1.25.0.
|
||||
|
||||
### Docker Compose (Recommended)
|
||||
|
||||
The slim Docker image with docker-compose is the recommended setup. See `examples/docker/docker-compose.yml` for a complete configuration that includes both services.
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ class DoclingServeChunker(DocumentChunker):
|
|||
"chunking_use_markdown_tables": str(
|
||||
self.config.processing.chunking_use_markdown_tables
|
||||
).lower(),
|
||||
"chunking_include_raw_text": "true",
|
||||
}
|
||||
if opts.ocr_lang:
|
||||
data["convert_ocr_lang"] = opts.ocr_lang
|
||||
|
|
@ -154,7 +155,10 @@ class DoclingServeChunker(DocumentChunker):
|
|||
result: list[Chunk] = []
|
||||
|
||||
for chunk in raw_chunks:
|
||||
text = chunk.get("text", "")
|
||||
# ``text`` is the embedding-targeted serialization (headings and
|
||||
# captions prepended); ``raw_text`` is the bare body. Store the
|
||||
# body and let the embedding/FTS layer contextualize from headings.
|
||||
text = chunk.get("raw_text") or chunk.get("text", "")
|
||||
|
||||
# doc_items from docling-serve is a list of ref strings like ["#/texts/1", "#/tables/0"]
|
||||
doc_items = chunk.get("doc_items", [])
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in a new issue