Prevent magic-byte sniffing from misrouting text conversion in docling-local
Docling guesses a DocumentStream's format from its first bytes before
considering the extension, so markdown/HTML content starting with a binary
magic signature ("BM" = BMP, "ID3" = MP3) was routed to an image or audio
backend and fell back to plain-text conversion. Prefix the encoded text
with a newline so the sniff finds nothing and the extension decides.
This commit is contained in:
parent
5dd310a9b7
commit
d2f5ecfc58
3 changed files with 28 additions and 1 deletions
|
|
@ -1,6 +1,10 @@
|
||||||
# Changelog
|
# Changelog
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- `docling-local` text conversion no longer misroutes markdown/HTML content whose first bytes collide with a binary magic signature (e.g. `BM`, `ID3`) to an image or audio backend.
|
||||||
|
|
||||||
## [0.66.0] - 2026-07-14
|
## [0.66.0] - 2026-07-14
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
|
||||||
|
|
@ -311,7 +311,12 @@ class DoclingLocalConverter(DocumentConverter):
|
||||||
from docling.exceptions import ConversionError
|
from docling.exceptions import ConversionError
|
||||||
from docling_core.types.io import DocumentStream
|
from docling_core.types.io import DocumentStream
|
||||||
|
|
||||||
bytes_io = BytesIO(text.encode("utf-8"))
|
# Docling sniffs magic bytes before considering the extension, so text
|
||||||
|
# starting with e.g. "BM" (BMP) or "ID3" (MP3) gets routed to a binary
|
||||||
|
# backend. A leading newline defeats every magic signature (all match
|
||||||
|
# at offset 0) without changing the md/html parse, making docling fall
|
||||||
|
# back to the extension in doc_name, which encodes the known format.
|
||||||
|
bytes_io = BytesIO(b"\n" + text.encode("utf-8"))
|
||||||
doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
|
doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
|
||||||
converter = DoclingDocConverter(
|
converter = DoclingDocConverter(
|
||||||
format_options=self._build_format_options(source_uri=source_uri)
|
format_options=self._build_format_options(source_uri=source_uri)
|
||||||
|
|
|
||||||
|
|
@ -320,6 +320,24 @@ class TestTextToDoclingWithFormat:
|
||||||
exported = doc.export_to_markdown()
|
exported = doc.export_to_markdown()
|
||||||
assert "MZ Wallace" in exported
|
assert "MZ Wallace" in exported
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_text_starting_with_magic_bytes_parses_as_markdown(self):
|
||||||
|
"""Text whose first bytes collide with a binary magic signature
|
||||||
|
("BM" = BMP, "ID3" = MP3) must still be parsed as markdown, not
|
||||||
|
routed to an image/audio backend by content sniffing.
|
||||||
|
"""
|
||||||
|
config = AppConfig()
|
||||||
|
converter = DoclingLocalConverter(config)
|
||||||
|
|
||||||
|
for prefix in ("BMW", "ID3 Algorithm"):
|
||||||
|
text = f"{prefix} overview.\n\n## History\n\n- First item\n- Second item"
|
||||||
|
doc = await converter.convert_text(text, format="md")
|
||||||
|
labels = [
|
||||||
|
str(getattr(item, "label", "")) for item, _ in doc.iterate_items()
|
||||||
|
]
|
||||||
|
assert "section_header" in labels
|
||||||
|
assert "list_item" in labels
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_plain_text_without_markdown_syntax_fallback(self):
|
async def test_plain_text_without_markdown_syntax_fallback(self):
|
||||||
"""Test that plain text without markdown syntax falls back gracefully.
|
"""Test that plain text without markdown syntax falls back gracefully.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue