fix for issue with parsing pdf attachment such as *.joboptions files
This commit is contained in:
parent
88696cc6f1
commit
bce24e84a0
2 changed files with 63 additions and 5 deletions
|
|
@ -332,16 +332,30 @@ async def _ingest_fetch_result(
|
||||||
stored_uri: str,
|
stored_uri: str,
|
||||||
existing_doc: Document | None,
|
existing_doc: Document | None,
|
||||||
depth: int = 0,
|
depth: int = 0,
|
||||||
|
filename: str | None = None,
|
||||||
) -> Document:
|
) -> Document:
|
||||||
"""Convert / chunk / embed / store a fetched document. Replaces an
|
"""Convert / chunk / embed / store a fetched document. Replaces an
|
||||||
existing document if one is supplied. ``depth`` tracks position in an
|
existing document if one is supplied. ``depth`` tracks position in an
|
||||||
attachment chain so the reconciliation step can bound recursion."""
|
attachment chain so the reconciliation step can bound recursion.
|
||||||
|
|
||||||
|
``filename`` makes that name's suffix the authoritative source of the
|
||||||
|
file extension (hence the docling format). Callers pass it when the
|
||||||
|
result's ``uri`` cannot yield the right extension -- notably embedded
|
||||||
|
PDF attachments, whose synthetic ``...#attachment=<name>`` URI carries
|
||||||
|
the attachment name in the fragment, so the URL-suffix fallback would
|
||||||
|
otherwise inherit the *parent* PDF's ``.pdf`` and feed ASCII payloads
|
||||||
|
(e.g. ``Press Quality.joboptions``) to docling's PDF backend. An empty
|
||||||
|
or unsupported suffix then fails the guard below and is rejected,
|
||||||
|
instead of being silently misrouted."""
|
||||||
from haiku.rag.embeddings import embed_chunks
|
from haiku.rag.embeddings import embed_chunks
|
||||||
|
|
||||||
converter = get_converter(client._config)
|
converter = get_converter(client._config)
|
||||||
file_extension = get_extension_from_content_type_or_url(
|
if filename is not None:
|
||||||
result.uri, result.content_type
|
file_extension = Path(filename).suffix.lower()
|
||||||
)
|
else:
|
||||||
|
file_extension = get_extension_from_content_type_or_url(
|
||||||
|
result.uri, result.content_type
|
||||||
|
)
|
||||||
if file_extension not in converter.supported_extensions:
|
if file_extension not in converter.supported_extensions:
|
||||||
raise UnsupportedSourceError(
|
raise UnsupportedSourceError(
|
||||||
f"Unsupported content type/extension: {result.content_type}/{file_extension}"
|
f"Unsupported content type/extension: {result.content_type}/{file_extension}"
|
||||||
|
|
@ -508,12 +522,15 @@ async def _reconcile_pdf_attachments(
|
||||||
stored_uri=child_uri,
|
stored_uri=child_uri,
|
||||||
existing_doc=existing_child,
|
existing_doc=existing_child,
|
||||||
depth=depth + 1,
|
depth=depth + 1,
|
||||||
|
filename=name,
|
||||||
)
|
)
|
||||||
except UnsupportedSourceError:
|
except UnsupportedSourceError:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Skipping attachment %r in %s: unsupported content type %r",
|
"Skipping attachment %r in %s: unsupported extension %r "
|
||||||
|
"(content type %r)",
|
||||||
name,
|
name,
|
||||||
parent_doc.uri,
|
parent_doc.uri,
|
||||||
|
Path(name).suffix.lower(),
|
||||||
content_type,
|
content_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ async def fake_ingest_fetch_result(
|
||||||
stored_uri,
|
stored_uri,
|
||||||
existing_doc,
|
existing_doc,
|
||||||
depth=0,
|
depth=0,
|
||||||
|
filename=None,
|
||||||
):
|
):
|
||||||
"""A stand-in for ``_ingest_fetch_result`` that skips docling/embedder
|
"""A stand-in for ``_ingest_fetch_result`` that skips docling/embedder
|
||||||
entirely: it writes the document with content_type/md5/parent_uri set
|
entirely: it writes the document with content_type/md5/parent_uri set
|
||||||
|
|
@ -306,6 +307,7 @@ async def test_unsupported_attachment_continues_loop(temp_db_path, monkeypatch):
|
||||||
stored_uri,
|
stored_uri,
|
||||||
existing_doc,
|
existing_doc,
|
||||||
depth=0,
|
depth=0,
|
||||||
|
filename=None,
|
||||||
):
|
):
|
||||||
if stored_uri.endswith("unsupported.xyz"):
|
if stored_uri.endswith("unsupported.xyz"):
|
||||||
raise UnsupportedSourceError("nope")
|
raise UnsupportedSourceError("nope")
|
||||||
|
|
@ -329,6 +331,45 @@ async def test_unsupported_attachment_continues_loop(temp_db_path, monkeypatch):
|
||||||
assert {c.uri for c in children} == {f"{parent_uri}#attachment=ok.txt"}
|
assert {c.uri for c in children} == {f"{parent_uri}#attachment=ok.txt"}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_joboptions_attachment_skipped_not_routed_as_pdf(temp_db_path, caplog):
|
||||||
|
"""Regression: an Adobe ``.joboptions`` preset is ASCII text, embedded by
|
||||||
|
name only. Its synthetic ``...#attachment=Press%20Quality.joboptions`` URI
|
||||||
|
carries the name in a *fragment*, so the URL-suffix fallback used to inherit
|
||||||
|
the PARENT PDF's ``.pdf`` and hand the ASCII bytes to docling's PDF backend.
|
||||||
|
|
||||||
|
The attachment's own name must now drive the extension: ``.joboptions`` is
|
||||||
|
unsupported, so the child is skipped -- never ingested, never sent to
|
||||||
|
docling. This runs the REAL ``_ingest_fetch_result`` (no monkeypatch); its
|
||||||
|
unsupported-extension guard short-circuits before any converter/embedder
|
||||||
|
call, so the sole attachment needs no LLM."""
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from haiku.rag.client.processing import get_extension_from_content_type_or_url
|
||||||
|
|
||||||
|
pdf_bytes = build_pdf([("Press Quality.joboptions", b"/CompressObjects /Tags\n")])
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
parent_uri = "file:///fixtures/brochure.pdf"
|
||||||
|
parent = await _make_parent(client, parent_uri, pdf_bytes)
|
||||||
|
|
||||||
|
# The latent trap: octet-stream + the fragment URI still resolves to the
|
||||||
|
# parent's ``.pdf`` -- which is exactly what the fix must no longer use.
|
||||||
|
child_uri = f"{parent_uri}#attachment=Press%20Quality.joboptions"
|
||||||
|
assert (
|
||||||
|
get_extension_from_content_type_or_url(
|
||||||
|
child_uri, "application/octet-stream"
|
||||||
|
)
|
||||||
|
== ".pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING, logger="haiku.rag.client.documents"):
|
||||||
|
await _reconcile_pdf_attachments(client, parent, pdf_bytes, depth=0)
|
||||||
|
|
||||||
|
# No child created: the ASCII preset was skipped, not parsed as a PDF.
|
||||||
|
assert await client.list_documents(filter=parent_uri_filter(parent_uri)) == []
|
||||||
|
assert "unsupported extension" in caplog.text
|
||||||
|
assert ".joboptions" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
async def test_cascade_delete_removes_reconciled_children(temp_db_path, monkeypatch):
|
async def test_cascade_delete_removes_reconciled_children(temp_db_path, monkeypatch):
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"haiku.rag.client.documents._ingest_fetch_result",
|
"haiku.rag.client.documents._ingest_fetch_result",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue