Strip leading dots from names handed to docling
This commit is contained in:
parent
ebd1fc7f3e
commit
8e7e494fd1
5 changed files with 94 additions and 8 deletions
|
|
@ -1,6 +1,10 @@
|
|||
# Changelog
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Dotfiles (`.gitignore`, `.env`) are parsed as their actual format instead of a single unstructured text block. Docling ignores the extension of a name starting with a dot, so converters strip leading dots from the name they hand it.
|
||||
|
||||
## [0.71.0] - 2026-07-29
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, ClassVar
|
|||
|
||||
from haiku.rag.config import AppConfig
|
||||
from haiku.rag.converters.base import DocumentConverter, vlm_api_url
|
||||
from haiku.rag.converters.text_utils import TextFileHandler
|
||||
from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
|
|
@ -287,7 +287,9 @@ class DoclingLocalConverter(DocumentConverter):
|
|||
f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}"
|
||||
)
|
||||
|
||||
doc_name = f"content.{format}" if name == "content.md" else name
|
||||
doc_name = docling_safe_name(
|
||||
f"content.{format}" if name == "content.md" else name
|
||||
)
|
||||
|
||||
if format == "plain":
|
||||
return TextFileHandler._create_simple_docling_document(text, doc_name)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar
|
|||
|
||||
from haiku.rag.config import AppConfig
|
||||
from haiku.rag.converters.base import DocumentConverter, vlm_api_url
|
||||
from haiku.rag.converters.text_utils import TextFileHandler
|
||||
from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
|
||||
from haiku.rag.providers.docling_serve import DoclingServeClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -245,7 +245,13 @@ class DoclingServeConverter(DocumentConverter):
|
|||
return f.read()
|
||||
|
||||
file_content = await asyncio.to_thread(read_file)
|
||||
files = {"files": (path.name, file_content, "application/octet-stream")}
|
||||
files = {
|
||||
"files": (
|
||||
docling_safe_name(path.name),
|
||||
file_content,
|
||||
"application/octet-stream",
|
||||
)
|
||||
}
|
||||
return await self._make_request(files, path.name)
|
||||
|
||||
SUPPORTED_FORMATS = ("md", "html", "plain")
|
||||
|
|
@ -276,8 +282,6 @@ class DoclingServeConverter(DocumentConverter):
|
|||
Raises:
|
||||
ValueError: If the text cannot be converted or format is unsupported.
|
||||
"""
|
||||
from haiku.rag.converters.text_utils import TextFileHandler
|
||||
|
||||
if format not in self.SUPPORTED_FORMATS:
|
||||
raise ValueError(
|
||||
f"Unsupported format: {format}. "
|
||||
|
|
@ -285,7 +289,9 @@ class DoclingServeConverter(DocumentConverter):
|
|||
)
|
||||
|
||||
# Derive document name from format to tell docling which parser to use
|
||||
doc_name = f"content.{format}" if name == "content.md" else name
|
||||
doc_name = docling_safe_name(
|
||||
f"content.{format}" if name == "content.md" else name
|
||||
)
|
||||
|
||||
# Plain text doesn't need remote parsing - create document directly
|
||||
if format == "plain":
|
||||
|
|
|
|||
|
|
@ -6,6 +6,16 @@ if TYPE_CHECKING:
|
|||
from docling_core.types.doc.document import DoclingDocument
|
||||
|
||||
|
||||
def docling_safe_name(name: str) -> str:
|
||||
"""Strip leading dots from a filename handed to docling.
|
||||
|
||||
Docling ignores the extension of any name starting with a dot, so a name
|
||||
derived from a dotfile (".gitignore" -> ".gitignore.md") is classified as
|
||||
an unknown format and parsed as one unstructured text block.
|
||||
"""
|
||||
return name.lstrip(".") or name
|
||||
|
||||
|
||||
class TextFileHandler:
|
||||
"""Handles conversion of text files to DoclingDocument format.
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from haiku.rag.converters import get_converter
|
|||
from haiku.rag.converters.base import vlm_api_url
|
||||
from haiku.rag.converters.docling_local import DoclingLocalConverter
|
||||
from haiku.rag.converters.docling_serve import DoclingServeConverter
|
||||
from haiku.rag.converters.text_utils import TextFileHandler
|
||||
from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
|
||||
|
||||
|
||||
class TestVlmApiUrl:
|
||||
|
|
@ -165,6 +165,13 @@ async def test_parse_zip_runs_off_event_loop_thread():
|
|||
class TestTextFileHandler:
|
||||
"""Tests for TextFileHandler utility class."""
|
||||
|
||||
def test_docling_safe_name(self):
|
||||
"""Leading dots are stripped, other names are untouched, and a name of
|
||||
nothing but dots is kept rather than reduced to an empty filename."""
|
||||
assert docling_safe_name(".gitignore.md") == "gitignore.md"
|
||||
assert docling_safe_name("notes.md") == "notes.md"
|
||||
assert docling_safe_name("...") == "..."
|
||||
|
||||
def test_text_extensions_defined(self):
|
||||
"""Test that text extensions list is defined."""
|
||||
assert len(TextFileHandler.text_extensions) > 0
|
||||
|
|
@ -338,6 +345,21 @@ class TestTextToDoclingWithFormat:
|
|||
assert "section_header" in labels
|
||||
assert "list_item" in labels
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dotfile_derived_name_parses_as_markdown(self):
|
||||
"""Docling ignores the extension of a stream name starting with a dot,
|
||||
so a name derived from a dotfile (".gitignore" -> ".gitignore.md") must
|
||||
be normalized or the whole document collapses to one plain-text block.
|
||||
"""
|
||||
config = AppConfig()
|
||||
converter = DoclingLocalConverter(config)
|
||||
|
||||
text = "Overview.\n\n## History\n\n- First item\n- Second item"
|
||||
doc = await converter.convert_text(text, name=".gitignore.md")
|
||||
labels = [str(getattr(item, "label", "")) for item, _ in doc.iterate_items()]
|
||||
assert "section_header" in labels
|
||||
assert "list_item" in labels
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_text_without_markdown_syntax_fallback(self):
|
||||
"""Test that plain text without markdown syntax falls back gracefully.
|
||||
|
|
@ -1305,6 +1327,48 @@ class TestDoclingServeConverter:
|
|||
call_kwargs = mock_client.post.call_args.kwargs
|
||||
assert "files" in call_kwargs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dotfile_uploads_with_detectable_name(self, converter):
|
||||
"""docling-serve runs the same format detection on the uploaded
|
||||
filename, which ignores the extension of any name starting with a dot.
|
||||
"""
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
await converter.convert_text("# Test", name=".gitignore.md")
|
||||
uploaded_name = mock_client.post.call_args.kwargs["files"]["files"][0]
|
||||
assert uploaded_name == "gitignore.md"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dotfile_path_uploads_with_detectable_name(self, converter, tmp_path):
|
||||
"""A dotfile with no extension is uploaded under a name docling can
|
||||
still probe by content rather than one it refuses to classify.
|
||||
"""
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
dotfile = tmp_path / ".customrc"
|
||||
dotfile.write_bytes(b"\x00binary")
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
await converter.convert_file(dotfile)
|
||||
uploaded_name = mock_client.post.call_args.kwargs["files"]["files"][0]
|
||||
assert uploaded_name == "customrc"
|
||||
|
||||
|
||||
class TestDoclingServeConverterPictureDescription:
|
||||
"""Tests for DoclingServeConverter picture description support."""
|
||||
|
|
|
|||
Loading…
Reference in a new issue