Strip leading dots from names handed to docling
This commit is contained in:
parent
ebd1fc7f3e
commit
8e7e494fd1
5 changed files with 94 additions and 8 deletions
|
|
@ -1,6 +1,10 @@
|
||||||
# Changelog
|
# Changelog
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Dotfiles (`.gitignore`, `.env`) are parsed as their actual format instead of a single unstructured text block. Docling ignores the extension of a name starting with a dot, so converters strip leading dots from the name they hand it.
|
||||||
|
|
||||||
## [0.71.0] - 2026-07-29
|
## [0.71.0] - 2026-07-29
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
from haiku.rag.config import AppConfig
|
from haiku.rag.config import AppConfig
|
||||||
from haiku.rag.converters.base import DocumentConverter, vlm_api_url
|
from haiku.rag.converters.base import DocumentConverter, vlm_api_url
|
||||||
from haiku.rag.converters.text_utils import TextFileHandler
|
from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from docling.datamodel.base_models import InputFormat
|
from docling.datamodel.base_models import InputFormat
|
||||||
|
|
@ -287,7 +287,9 @@ class DoclingLocalConverter(DocumentConverter):
|
||||||
f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}"
|
f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
doc_name = f"content.{format}" if name == "content.md" else name
|
doc_name = docling_safe_name(
|
||||||
|
f"content.{format}" if name == "content.md" else name
|
||||||
|
)
|
||||||
|
|
||||||
if format == "plain":
|
if format == "plain":
|
||||||
return TextFileHandler._create_simple_docling_document(text, doc_name)
|
return TextFileHandler._create_simple_docling_document(text, doc_name)
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
from haiku.rag.config import AppConfig
|
from haiku.rag.config import AppConfig
|
||||||
from haiku.rag.converters.base import DocumentConverter, vlm_api_url
|
from haiku.rag.converters.base import DocumentConverter, vlm_api_url
|
||||||
from haiku.rag.converters.text_utils import TextFileHandler
|
from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
|
||||||
from haiku.rag.providers.docling_serve import DoclingServeClient
|
from haiku.rag.providers.docling_serve import DoclingServeClient
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
|
@ -245,7 +245,13 @@ class DoclingServeConverter(DocumentConverter):
|
||||||
return f.read()
|
return f.read()
|
||||||
|
|
||||||
file_content = await asyncio.to_thread(read_file)
|
file_content = await asyncio.to_thread(read_file)
|
||||||
files = {"files": (path.name, file_content, "application/octet-stream")}
|
files = {
|
||||||
|
"files": (
|
||||||
|
docling_safe_name(path.name),
|
||||||
|
file_content,
|
||||||
|
"application/octet-stream",
|
||||||
|
)
|
||||||
|
}
|
||||||
return await self._make_request(files, path.name)
|
return await self._make_request(files, path.name)
|
||||||
|
|
||||||
SUPPORTED_FORMATS = ("md", "html", "plain")
|
SUPPORTED_FORMATS = ("md", "html", "plain")
|
||||||
|
|
@ -276,8 +282,6 @@ class DoclingServeConverter(DocumentConverter):
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If the text cannot be converted or format is unsupported.
|
ValueError: If the text cannot be converted or format is unsupported.
|
||||||
"""
|
"""
|
||||||
from haiku.rag.converters.text_utils import TextFileHandler
|
|
||||||
|
|
||||||
if format not in self.SUPPORTED_FORMATS:
|
if format not in self.SUPPORTED_FORMATS:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Unsupported format: {format}. "
|
f"Unsupported format: {format}. "
|
||||||
|
|
@ -285,7 +289,9 @@ class DoclingServeConverter(DocumentConverter):
|
||||||
)
|
)
|
||||||
|
|
||||||
# Derive document name from format to tell docling which parser to use
|
# Derive document name from format to tell docling which parser to use
|
||||||
doc_name = f"content.{format}" if name == "content.md" else name
|
doc_name = docling_safe_name(
|
||||||
|
f"content.{format}" if name == "content.md" else name
|
||||||
|
)
|
||||||
|
|
||||||
# Plain text doesn't need remote parsing - create document directly
|
# Plain text doesn't need remote parsing - create document directly
|
||||||
if format == "plain":
|
if format == "plain":
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,16 @@ if TYPE_CHECKING:
|
||||||
from docling_core.types.doc.document import DoclingDocument
|
from docling_core.types.doc.document import DoclingDocument
|
||||||
|
|
||||||
|
|
||||||
|
def docling_safe_name(name: str) -> str:
|
||||||
|
"""Strip leading dots from a filename handed to docling.
|
||||||
|
|
||||||
|
Docling ignores the extension of any name starting with a dot, so a name
|
||||||
|
derived from a dotfile (".gitignore" -> ".gitignore.md") is classified as
|
||||||
|
an unknown format and parsed as one unstructured text block.
|
||||||
|
"""
|
||||||
|
return name.lstrip(".") or name
|
||||||
|
|
||||||
|
|
||||||
class TextFileHandler:
|
class TextFileHandler:
|
||||||
"""Handles conversion of text files to DoclingDocument format.
|
"""Handles conversion of text files to DoclingDocument format.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ from haiku.rag.converters import get_converter
|
||||||
from haiku.rag.converters.base import vlm_api_url
|
from haiku.rag.converters.base import vlm_api_url
|
||||||
from haiku.rag.converters.docling_local import DoclingLocalConverter
|
from haiku.rag.converters.docling_local import DoclingLocalConverter
|
||||||
from haiku.rag.converters.docling_serve import DoclingServeConverter
|
from haiku.rag.converters.docling_serve import DoclingServeConverter
|
||||||
from haiku.rag.converters.text_utils import TextFileHandler
|
from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
|
||||||
|
|
||||||
|
|
||||||
class TestVlmApiUrl:
|
class TestVlmApiUrl:
|
||||||
|
|
@ -165,6 +165,13 @@ async def test_parse_zip_runs_off_event_loop_thread():
|
||||||
class TestTextFileHandler:
|
class TestTextFileHandler:
|
||||||
"""Tests for TextFileHandler utility class."""
|
"""Tests for TextFileHandler utility class."""
|
||||||
|
|
||||||
|
def test_docling_safe_name(self):
|
||||||
|
"""Leading dots are stripped, other names are untouched, and a name of
|
||||||
|
nothing but dots is kept rather than reduced to an empty filename."""
|
||||||
|
assert docling_safe_name(".gitignore.md") == "gitignore.md"
|
||||||
|
assert docling_safe_name("notes.md") == "notes.md"
|
||||||
|
assert docling_safe_name("...") == "..."
|
||||||
|
|
||||||
def test_text_extensions_defined(self):
|
def test_text_extensions_defined(self):
|
||||||
"""Test that text extensions list is defined."""
|
"""Test that text extensions list is defined."""
|
||||||
assert len(TextFileHandler.text_extensions) > 0
|
assert len(TextFileHandler.text_extensions) > 0
|
||||||
|
|
@ -338,6 +345,21 @@ class TestTextToDoclingWithFormat:
|
||||||
assert "section_header" in labels
|
assert "section_header" in labels
|
||||||
assert "list_item" in labels
|
assert "list_item" in labels
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dotfile_derived_name_parses_as_markdown(self):
|
||||||
|
"""Docling ignores the extension of a stream name starting with a dot,
|
||||||
|
so a name derived from a dotfile (".gitignore" -> ".gitignore.md") must
|
||||||
|
be normalized or the whole document collapses to one plain-text block.
|
||||||
|
"""
|
||||||
|
config = AppConfig()
|
||||||
|
converter = DoclingLocalConverter(config)
|
||||||
|
|
||||||
|
text = "Overview.\n\n## History\n\n- First item\n- Second item"
|
||||||
|
doc = await converter.convert_text(text, name=".gitignore.md")
|
||||||
|
labels = [str(getattr(item, "label", "")) for item, _ in doc.iterate_items()]
|
||||||
|
assert "section_header" in labels
|
||||||
|
assert "list_item" in labels
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_plain_text_without_markdown_syntax_fallback(self):
|
async def test_plain_text_without_markdown_syntax_fallback(self):
|
||||||
"""Test that plain text without markdown syntax falls back gracefully.
|
"""Test that plain text without markdown syntax falls back gracefully.
|
||||||
|
|
@ -1305,6 +1327,48 @@ class TestDoclingServeConverter:
|
||||||
call_kwargs = mock_client.post.call_args.kwargs
|
call_kwargs = mock_client.post.call_args.kwargs
|
||||||
assert "files" in call_kwargs
|
assert "files" in call_kwargs
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dotfile_uploads_with_detectable_name(self, converter):
|
||||||
|
"""docling-serve runs the same format detection on the uploaded
|
||||||
|
filename, which ignores the extension of any name starting with a dot.
|
||||||
|
"""
|
||||||
|
doc_json = create_mock_docling_document("test")
|
||||||
|
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||||
|
|
||||||
|
with patch("httpx.AsyncClient") as mock_client_class:
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||||
|
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||||
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||||
|
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
mock_client_class.return_value = mock_client
|
||||||
|
|
||||||
|
await converter.convert_text("# Test", name=".gitignore.md")
|
||||||
|
uploaded_name = mock_client.post.call_args.kwargs["files"]["files"][0]
|
||||||
|
assert uploaded_name == "gitignore.md"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dotfile_path_uploads_with_detectable_name(self, converter, tmp_path):
|
||||||
|
"""A dotfile with no extension is uploaded under a name docling can
|
||||||
|
still probe by content rather than one it refuses to classify.
|
||||||
|
"""
|
||||||
|
doc_json = create_mock_docling_document("test")
|
||||||
|
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||||
|
dotfile = tmp_path / ".customrc"
|
||||||
|
dotfile.write_bytes(b"\x00binary")
|
||||||
|
|
||||||
|
with patch("httpx.AsyncClient") as mock_client_class:
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||||
|
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||||
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||||
|
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
mock_client_class.return_value = mock_client
|
||||||
|
|
||||||
|
await converter.convert_file(dotfile)
|
||||||
|
uploaded_name = mock_client.post.call_args.kwargs["files"]["files"][0]
|
||||||
|
assert uploaded_name == "customrc"
|
||||||
|
|
||||||
|
|
||||||
class TestDoclingServeConverterPictureDescription:
|
class TestDoclingServeConverterPictureDescription:
|
||||||
"""Tests for DoclingServeConverter picture description support."""
|
"""Tests for DoclingServeConverter picture description support."""
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue