diff --git a/CHANGELOG.md b/CHANGELOG.md index 0863dd9c..40e4966a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,12 @@ - **Configurable judge and reflect models**: `evaluations run` and `evaluations optimize` accept `--judge-model provider:name`; `optimize` also accepts `--reflect-model provider:name`. Both fall back to `config.qa.model` when not specified. - **`parse_model_option`**: Utility in `haiku.rag.utils` for parsing `provider:name` strings into `ModelConfig` +- **New format extensions**: `.tex`, `.latex`, `.qmd` (Quarto), `.rmd` (R Markdown) supported in both local and serve converters ### Changed - **LLMJudge**: Custom evaluator now accepts `ModelConfig` instead of a model name string +- **Docling upgrade**: docling-core ≥2.70.2 (schema 1.10.0), docling ≥2.81.0. Adds field data model support for structured form/KV content, wide table chunking fixes, and rich table cell hang fix ## [0.34.1] - 2026-03-16 diff --git a/docs/server.md b/docs/server.md index 40bd46b4..8917f4e6 100644 --- a/docs/server.md +++ b/docs/server.md @@ -92,6 +92,9 @@ The file monitor processes documents using [Docling](https://github.com/DS4SD/do - Microsoft PowerPoint (`.pptx`) - HTML (`.html`, `.htm`) - Markdown (`.md`) +- Quarto Markdown (`.qmd`) +- R Markdown (`.rmd`) +- LaTeX (`.tex`, `.latex`) - AsciiDoc (`.adoc`, `.asciidoc`) **Data formats:** diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index 15db857e..ba932fe6 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -1174,7 +1174,14 @@ class HaikuRAG: return merged # Label groups for type-aware expansion - _STRUCTURAL_LABELS = {"table", "code", "list_item", "form", "key_value_region"} + _STRUCTURAL_LABELS = { + "table", + "code", + "list_item", + "form", + "key_value_region", + "field_region", + } def _extract_item_text(self, item, docling_doc) -> str | None: """Extract text content from a DocItem. diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index bc2284d1..f57c2e91 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -33,10 +33,14 @@ class DoclingLocalConverter(DocumentConverter): ".xhtml", ".jpeg", ".jpg", + ".latex", ".md", ".pdf", ".png", ".pptx", + ".qmd", + ".rmd", + ".tex", ".tiff", ".xlsx", ".xml", diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py index 2f61bf97..3648514b 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py @@ -38,10 +38,14 @@ class DoclingServeConverter(DocumentConverter): ".xhtml", ".jpeg", ".jpg", + ".latex", ".md", ".pdf", ".png", ".pptx", + ".qmd", + ".rmd", + ".tex", ".tiff", ".xlsx", ".xml", diff --git a/haiku_rag_slim/haiku/rag/store/models/chunk.py b/haiku_rag_slim/haiku/rag/store/models/chunk.py index 07d1a06c..d662970b 100644 --- a/haiku_rag_slim/haiku/rag/store/models/chunk.py +++ b/haiku_rag_slim/haiku/rag/store/models/chunk.py @@ -196,15 +196,22 @@ class SearchResult(BaseModel): "table": 1, "code": 2, "form": 3, + "field_region": 3, "key_value_region": 4, - "list_item": 5, - "formula": 6, - "chart": 7, - "picture": 8, - "caption": 9, - "footnote": 10, - "section_header": 11, - "title": 12, + "field_item": 4, + "field_key": 5, + "field_value": 6, + "field_heading": 7, + "list_item": 8, + "formula": 9, + "chart": 10, + "picture": 11, + "caption": 12, + "footnote": 13, + "field_hint": 14, + "marker": 15, + "section_header": 16, + "title": 17, } # Find highest priority label diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index d2d128bc..ac38c8f2 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ dependencies = [ "cachetools>=7.0.2", - "docling-core>=2.67.1", + "docling-core>=2.70.2", "haiku.skills>=0.8.0", "httpx>=0.28.1", "jsonpatch>=1.33", @@ -41,7 +41,7 @@ dependencies = [ [project.optional-dependencies] # Document processing -docling = ["docling>=2.76.0", "opencv-python-headless>=4.13.0.92"] +docling = ["docling>=2.81.0", "opencv-python-headless>=4.13.0.92"] # Embedding providers voyageai = ["pydantic-ai-slim[voyageai]"] # Rerankers diff --git a/tests/test_converters.py b/tests/test_converters.py index c5a8435c..6ab991a1 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -24,7 +24,7 @@ def create_mock_docling_document(name: str = "test") -> dict: """Create a minimal valid DoclingDocument JSON structure for mocking.""" return { "schema_name": "DoclingDocument", - "version": "1.9.0", + "version": "1.10.0", "name": name, "origin": { "mimetype": "text/markdown", @@ -624,7 +624,7 @@ class TestDoclingServeConverter: doc = await converter.convert_text("# Test", name="test.md") assert isinstance(doc, DoclingDocument) - assert doc.version == "1.9.0" + assert doc.version == "1.10.0" mock_client.post.assert_called_once() @pytest.mark.asyncio diff --git a/uv.lock b/uv.lock index cfa549a7..20047bc7 100644 --- a/uv.lock +++ b/uv.lock @@ -802,7 +802,7 @@ wheels = [ [[package]] name = "docling" -version = "2.76.0" +version = "2.81.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "accelerate" }, @@ -832,17 +832,19 @@ dependencies = [ { name = "requests" }, { name = "rtree" }, { name = "scipy" }, + { name = "torch" }, + { name = "torchvision" }, { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/d0/c63c9e39ff832cc51636cc085e6c069a07b1cd0c8d3621d4591d38e1c0d9/docling-2.76.0.tar.gz", hash = "sha256:c942ecbf3254c63d6853edc6cc439ce3ca4f71e469983272b579cf9a197e9356", size = 368671, upload-time = "2026-03-02T14:44:25.656Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/4d/23340afd09522e0428bca3cc721f7c1cb856ba646357570e1baf2a060b3a/docling-2.81.0.tar.gz", hash = "sha256:7a1d45295b9fcf8b506c67747ae311c0b7e8f80b8e70001e1849e21c0864ac08", size = 390757, upload-time = "2026-03-20T21:34:15.588Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/b5/ca589bcfc8c1bff2efb19144bc9a1693531ad3267a079187142393965597/docling-2.76.0-py3-none-any.whl", hash = "sha256:b3c4f15ce80b43744c214fa9d6415c728b060a99077803c00567fda7f84f3f20", size = 397384, upload-time = "2026-03-02T14:44:24.028Z" }, + { url = "https://files.pythonhosted.org/packages/b0/43/88390014f7b7b03a5fa2446851d0ec69e92a3f2414da19bf2793ab642eb2/docling-2.81.0-py3-none-any.whl", hash = "sha256:4f264d06b1415d9d27034145148d115ab3d4b72eb4daa3fdbea7356d6f70f92f", size = 423326, upload-time = "2026-03-20T21:34:13.824Z" }, ] [[package]] name = "docling-core" -version = "2.67.1" +version = "2.70.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "defusedxml" }, @@ -857,9 +859,9 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/fd/ab17be001cd51ce15adcae0ba873263ad144603acb279e0b631affa810a4/docling_core-2.67.1.tar.gz", hash = "sha256:9ab34fbd03e5afedd20c21007d7ac0dc9cf97fdb9eaf87b29fd40da6272958f0", size = 264637, upload-time = "2026-03-05T09:12:02.162Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/fb/2d88db67502418bb3e04e7ed2ac1893510d0c93c33defef08a6047d705e8/docling_core-2.70.2.tar.gz", hash = "sha256:f4cf53c86afc0f14bd526f9ee94125f68f77a284a079a99d1b83c9368ab5508c", size = 300733, upload-time = "2026-03-20T15:38:12.816Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/06/bb8d1f236a1560003b19ba4b005313a95366d9de852fa4062f1f45753f09/docling_core-2.67.1-py3-none-any.whl", hash = "sha256:8e47dd13eef1ca2ce6585e042aabc4b13a51f5c25f6137fc4a7075300ec4c24f", size = 245613, upload-time = "2026-03-05T09:12:00.599Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d1/cc3ff3e9afd17447510270622429eab6e0edc94f0c842f463075f20c3cab/docling_core-2.70.2-py3-none-any.whl", hash = "sha256:80d456b3500654f5ded421bd08d10bf255d76167c2d7f5ca7b0db1c8eed53a0a", size = 266702, upload-time = "2026-03-20T15:38:11.35Z" }, ] [package.optional-dependencies] @@ -875,7 +877,7 @@ chunking = [ [[package]] name = "docling-ibm-models" -version = "3.11.0" +version = "3.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "accelerate" }, @@ -892,9 +894,9 @@ dependencies = [ { name = "tqdm" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b6/91/f883e0a2b3466e1126dfd4463f386c70f5b90d271c27b6f5a97d2f8312e6/docling_ibm_models-3.11.0.tar.gz", hash = "sha256:454401563a8e79cb33b718bc559d9bacca8a0183583e48f8e616c9184c1f5eb1", size = 87721, upload-time = "2026-01-23T12:29:35.384Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/57/b4cee1ea5a7d34a8a96787aa2dc371e4dd17a1a3bd6131cf3ced9024f1be/docling_ibm_models-3.12.0.tar.gz", hash = "sha256:85c2b6c9dbb7fbb8eaf0f2a462b5984626457a6dc33148643491270c27767b46", size = 98458, upload-time = "2026-03-09T12:27:35.744Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/5d/97e9c2e10fbd3ee1723ac82c335f8211a9633c0397cc11ed057c3ba4006e/docling_ibm_models-3.11.0-py3-none-any.whl", hash = "sha256:68f7961069d643bfdab21b1c9ef24a979db293496f4c2283d95b1025a9ac5347", size = 87352, upload-time = "2026-01-23T12:29:34.045Z" }, + { url = "https://files.pythonhosted.org/packages/98/ed/820fdcea9aa329119855a658366fb13098b375a2b024358b1d7836f47419/docling_ibm_models-3.12.0-py3-none-any.whl", hash = "sha256:008fe1f5571db413782efe510c1d6327ea9df20b5255d416d0f4b56cfd090238", size = 93800, upload-time = "2026-03-09T12:27:34.477Z" }, ] [[package]] @@ -1511,8 +1513,8 @@ zeroentropy = [ requires-dist = [ { name = "cachetools", specifier = ">=7.0.2" }, { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.20.7" }, - { name = "docling", marker = "extra == 'docling'", specifier = ">=2.76.0" }, - { name = "docling-core", specifier = ">=2.67.1" }, + { name = "docling", marker = "extra == 'docling'", specifier = ">=2.81.0" }, + { name = "docling-core", specifier = ">=2.70.2" }, { name = "haiku-skills", specifier = ">=0.8.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "jsonpatch", specifier = ">=1.33" },