Prevent magic-byte sniffing from misrouting text conversion in docling-local

Docling guesses a DocumentStream's format from its first bytes before
considering the extension, so markdown/HTML content starting with a binary
magic signature ("BM" = BMP, "ID3" = MP3) was routed to an image or audio
backend and fell back to plain-text conversion. Prefix the encoded text
with a newline so the sniff finds nothing and the extension decides.
This commit is contained in:
Yiorgis Gozadinos 2026-07-15 19:18:15 +03:00
parent 5dd310a9b7
commit d2f5ecfc58
No known key found for this signature in database
3 changed files with 28 additions and 1 deletions

View file

@ -1,6 +1,10 @@
# Changelog
## [Unreleased]
### Fixed
- `docling-local` text conversion no longer misroutes markdown/HTML content whose first bytes collide with a binary magic signature (e.g. `BM`, `ID3`) to an image or audio backend.
## [0.66.0] - 2026-07-14
### Changed

View file

@ -311,7 +311,12 @@ class DoclingLocalConverter(DocumentConverter):
from docling.exceptions import ConversionError
from docling_core.types.io import DocumentStream
bytes_io = BytesIO(text.encode("utf-8"))
# Docling sniffs magic bytes before considering the extension, so text
# starting with e.g. "BM" (BMP) or "ID3" (MP3) gets routed to a binary
# backend. A leading newline defeats every magic signature (all match
# at offset 0) without changing the md/html parse, making docling fall
# back to the extension in doc_name, which encodes the known format.
bytes_io = BytesIO(b"\n" + text.encode("utf-8"))
doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
converter = DoclingDocConverter(
format_options=self._build_format_options(source_uri=source_uri)

View file

@ -320,6 +320,24 @@ class TestTextToDoclingWithFormat:
exported = doc.export_to_markdown()
assert "MZ Wallace" in exported
@pytest.mark.asyncio
async def test_text_starting_with_magic_bytes_parses_as_markdown(self):
"""Text whose first bytes collide with a binary magic signature
("BM" = BMP, "ID3" = MP3) must still be parsed as markdown, not
routed to an image/audio backend by content sniffing.
"""
config = AppConfig()
converter = DoclingLocalConverter(config)
for prefix in ("BMW", "ID3 Algorithm"):
text = f"{prefix} overview.\n\n## History\n\n- First item\n- Second item"
doc = await converter.convert_text(text, format="md")
labels = [
str(getattr(item, "label", "")) for item, _ in doc.iterate_items()
]
assert "section_header" in labels
assert "list_item" in labels
@pytest.mark.asyncio
async def test_plain_text_without_markdown_syntax_fallback(self):
"""Test that plain text without markdown syntax falls back gracefully.