From 8e7e494fd188ff2b265223cb8089e9ecece6c232 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 30 Jul 2026 14:07:00 +0300 Subject: [PATCH] Strip leading dots from names handed to docling --- CHANGELOG.md | 4 ++ .../haiku/rag/converters/docling_local.py | 6 +- .../haiku/rag/converters/docling_serve.py | 16 +++-- .../haiku/rag/converters/text_utils.py | 10 +++ tests/test_converters.py | 66 ++++++++++++++++++- 5 files changed, 94 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee052a5d..bac87711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Fixed + +- Dotfiles (`.gitignore`, `.env`) are parsed as their actual format instead of a single unstructured text block. Docling ignores the extension of a name starting with a dot, so converters strip leading dots from the name they hand it. + ## [0.71.0] - 2026-07-29 ### Added diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index 459274e5..0b358e5f 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, ClassVar from haiku.rag.config import AppConfig from haiku.rag.converters.base import DocumentConverter, vlm_api_url -from haiku.rag.converters.text_utils import TextFileHandler +from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name if TYPE_CHECKING: from docling.datamodel.base_models import InputFormat @@ -287,7 +287,9 @@ class DoclingLocalConverter(DocumentConverter): f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}" ) - doc_name = f"content.{format}" if name == "content.md" else name + doc_name = docling_safe_name( + f"content.{format}" if name == "content.md" else name + ) if format == "plain": return TextFileHandler._create_simple_docling_document(text, doc_name) diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py index c9a9ba2a..aea9a155 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar from haiku.rag.config import AppConfig from haiku.rag.converters.base import DocumentConverter, vlm_api_url -from haiku.rag.converters.text_utils import TextFileHandler +from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name from haiku.rag.providers.docling_serve import DoclingServeClient if TYPE_CHECKING: @@ -245,7 +245,13 @@ class DoclingServeConverter(DocumentConverter): return f.read() file_content = await asyncio.to_thread(read_file) - files = {"files": (path.name, file_content, "application/octet-stream")} + files = { + "files": ( + docling_safe_name(path.name), + file_content, + "application/octet-stream", + ) + } return await self._make_request(files, path.name) SUPPORTED_FORMATS = ("md", "html", "plain") @@ -276,8 +282,6 @@ class DoclingServeConverter(DocumentConverter): Raises: ValueError: If the text cannot be converted or format is unsupported. """ - from haiku.rag.converters.text_utils import TextFileHandler - if format not in self.SUPPORTED_FORMATS: raise ValueError( f"Unsupported format: {format}. " @@ -285,7 +289,9 @@ class DoclingServeConverter(DocumentConverter): ) # Derive document name from format to tell docling which parser to use - doc_name = f"content.{format}" if name == "content.md" else name + doc_name = docling_safe_name( + f"content.{format}" if name == "content.md" else name + ) # Plain text doesn't need remote parsing - create document directly if format == "plain": diff --git a/haiku_rag_slim/haiku/rag/converters/text_utils.py b/haiku_rag_slim/haiku/rag/converters/text_utils.py index c51a945d..b0e593b1 100644 --- a/haiku_rag_slim/haiku/rag/converters/text_utils.py +++ b/haiku_rag_slim/haiku/rag/converters/text_utils.py @@ -6,6 +6,16 @@ if TYPE_CHECKING: from docling_core.types.doc.document import DoclingDocument +def docling_safe_name(name: str) -> str: + """Strip leading dots from a filename handed to docling. + + Docling ignores the extension of any name starting with a dot, so a name + derived from a dotfile (".gitignore" -> ".gitignore.md") is classified as + an unknown format and parsed as one unstructured text block. + """ + return name.lstrip(".") or name + + class TextFileHandler: """Handles conversion of text files to DoclingDocument format. diff --git a/tests/test_converters.py b/tests/test_converters.py index e6579281..53cb058e 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -14,7 +14,7 @@ from haiku.rag.converters import get_converter from haiku.rag.converters.base import vlm_api_url from haiku.rag.converters.docling_local import DoclingLocalConverter from haiku.rag.converters.docling_serve import DoclingServeConverter -from haiku.rag.converters.text_utils import TextFileHandler +from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name class TestVlmApiUrl: @@ -165,6 +165,13 @@ async def test_parse_zip_runs_off_event_loop_thread(): class TestTextFileHandler: """Tests for TextFileHandler utility class.""" + def test_docling_safe_name(self): + """Leading dots are stripped, other names are untouched, and a name of + nothing but dots is kept rather than reduced to an empty filename.""" + assert docling_safe_name(".gitignore.md") == "gitignore.md" + assert docling_safe_name("notes.md") == "notes.md" + assert docling_safe_name("...") == "..." + def test_text_extensions_defined(self): """Test that text extensions list is defined.""" assert len(TextFileHandler.text_extensions) > 0 @@ -338,6 +345,21 @@ class TestTextToDoclingWithFormat: assert "section_header" in labels assert "list_item" in labels + @pytest.mark.asyncio + async def test_dotfile_derived_name_parses_as_markdown(self): + """Docling ignores the extension of a stream name starting with a dot, + so a name derived from a dotfile (".gitignore" -> ".gitignore.md") must + be normalized or the whole document collapses to one plain-text block. + """ + config = AppConfig() + converter = DoclingLocalConverter(config) + + text = "Overview.\n\n## History\n\n- First item\n- Second item" + doc = await converter.convert_text(text, name=".gitignore.md") + labels = [str(getattr(item, "label", "")) for item, _ in doc.iterate_items()] + assert "section_header" in labels + assert "list_item" in labels + @pytest.mark.asyncio async def test_plain_text_without_markdown_syntax_fallback(self): """Test that plain text without markdown syntax falls back gracefully. @@ -1305,6 +1327,48 @@ class TestDoclingServeConverter: call_kwargs = mock_client.post.call_args.kwargs assert "files" in call_kwargs + @pytest.mark.asyncio + async def test_dotfile_uploads_with_detectable_name(self, converter): + """docling-serve runs the same format detection on the uploaded + filename, which ignores the extension of any name starting with a dot. + """ + doc_json = create_mock_docling_document("test") + submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json) + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=submit_resp) + mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp]) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client_class.return_value = mock_client + + await converter.convert_text("# Test", name=".gitignore.md") + uploaded_name = mock_client.post.call_args.kwargs["files"]["files"][0] + assert uploaded_name == "gitignore.md" + + @pytest.mark.asyncio + async def test_dotfile_path_uploads_with_detectable_name(self, converter, tmp_path): + """A dotfile with no extension is uploaded under a name docling can + still probe by content rather than one it refuses to classify. + """ + doc_json = create_mock_docling_document("test") + submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json) + dotfile = tmp_path / ".customrc" + dotfile.write_bytes(b"\x00binary") + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=submit_resp) + mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp]) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client_class.return_value = mock_client + + await converter.convert_file(dotfile) + uploaded_name = mock_client.post.call_args.kwargs["files"]["files"][0] + assert uploaded_name == "customrc" + class TestDoclingServeConverterPictureDescription: """Tests for DoclingServeConverter picture description support."""