da6cdfbc51
urlparse().path keeps the leading slash in front of a Windows drive, so file:///C:/docs/a.pdf read as \C:\docs\a.pdf and the ingester reported "File does not exist" for every file it discovered. url2pathname is the stdlib conversion that strips it, per platform. Four sites each decided both "is this local" and "what path is this": FSSource._uri_to_path and supports, resolve_adhoc_fetcher, create_document_from_source and check_source_accessible, and convert. is_local_uri and uri_to_path in haiku.rag.uri own those two decisions now, which closes two more cases of the same root cause. A bare C:\docs\a.pdf parses with scheme "c", so add-src raised "No source adapter for URI scheme 'c'" and convert silently treated the path as raw text. And convert and check_source_accessible never percent-decoded at all, so a file named a[b] c.md read as missing on Linux and macOS too. A file URI's host is reattached after conversion rather than passed to url2pathname, which as of 3.14 rejects a non-local authority off Windows. file:////server/share is the empty-authority spelling of a UNC path, its host being the first path segment, so that host is normalised into the authority before conversion. Output is identical on 3.12, 3.13 and 3.14. The ad-hoc FS fetcher roots at the path's own anchor rather than "/", which on Windows is only the current drive. test_uri.py runs on ubuntu, macos and windows across 3.13 and 3.14 without the project installed: --noconftest because the repo conftest imports dependencies that job does not need, and -o addopts= to drop the repository's -n auto. The Windows legs are what cover the drive conversion. Fixes #574.
277 lines
8.8 KiB
Python
277 lines
8.8 KiB
Python
"""Tests for haiku.rag.client.processing helpers."""
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from haiku.rag.client.processing import _warn_if_descriptions_missing, convert
|
|
from haiku.rag.config import AppConfig
|
|
from tests.conftest import capture_logs
|
|
|
|
|
|
def _doc_with_pictures(*, with_descriptions: bool):
|
|
"""Build a tiny DoclingDocument carrying one PictureItem.
|
|
|
|
When ``with_descriptions=True`` the picture's ``meta.description.text``
|
|
is populated, simulating a successful VLM call. Otherwise it's left
|
|
empty, simulating a silent VLM failure.
|
|
"""
|
|
from docling_core.types.doc.document import (
|
|
DescriptionMetaField,
|
|
DoclingDocument,
|
|
PictureItem,
|
|
PictureMeta,
|
|
)
|
|
from docling_core.types.doc.labels import DocItemLabel
|
|
|
|
doc = DoclingDocument(name="test")
|
|
pic = PictureItem(
|
|
self_ref="#/pictures/0",
|
|
label=DocItemLabel.PICTURE,
|
|
)
|
|
if with_descriptions:
|
|
pic.meta = PictureMeta(
|
|
description=DescriptionMetaField(text="A red square."),
|
|
)
|
|
doc.pictures.append(pic)
|
|
return doc
|
|
|
|
|
|
def _doc_without_pictures():
|
|
from docling_core.types.doc.document import DoclingDocument
|
|
|
|
return DoclingDocument(name="test")
|
|
|
|
|
|
@pytest.fixture
|
|
def caplog_warnings():
|
|
"""Capture WARNING-level records from the processing logger."""
|
|
from haiku.rag.client.processing import logger as proc_logger
|
|
|
|
with capture_logs(proc_logger, logging.WARNING) as records:
|
|
yield records
|
|
|
|
|
|
def test_no_warning_when_picture_description_disabled(caplog_warnings):
|
|
"""Default config has pictures='image' — even a document full of
|
|
pictures with no descriptions should not warn."""
|
|
config = AppConfig()
|
|
assert config.processing.pictures == "image"
|
|
doc = _doc_with_pictures(with_descriptions=False)
|
|
|
|
_warn_if_descriptions_missing(config, doc, "fake.pdf")
|
|
|
|
assert caplog_warnings == []
|
|
|
|
|
|
def test_no_warning_when_doc_has_no_pictures(caplog_warnings):
|
|
"""A picture-less document under enabled=True shouldn't warn — there
|
|
was simply nothing to describe."""
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
doc = _doc_without_pictures()
|
|
|
|
_warn_if_descriptions_missing(config, doc, "no-pictures.txt")
|
|
|
|
assert caplog_warnings == []
|
|
|
|
|
|
def test_warns_when_pictures_present_but_no_descriptions(caplog_warnings):
|
|
"""VLM was requested via ``processing.pictures='description'``, the
|
|
doc has pictures, but the converter returned zero descriptions
|
|
(docling-serve swallows VLM errors). Warn loudly so the user can fix
|
|
their VLM config before a long ingest."""
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
config.processing.conversion_options.picture_description.model.name = "qwen3.6"
|
|
config.processing.conversion_options.picture_description.model.base_url = (
|
|
"http://host.docker.internal:11434"
|
|
)
|
|
doc = _doc_with_pictures(with_descriptions=False)
|
|
|
|
_warn_if_descriptions_missing(config, doc, "doclaynet.pdf")
|
|
|
|
assert len(caplog_warnings) == 1
|
|
msg = caplog_warnings[0].getMessage()
|
|
assert "doclaynet.pdf" in msg
|
|
assert "1 pictures" in msg
|
|
assert "0 described" in msg
|
|
assert "qwen3.6" in msg
|
|
assert "host.docker.internal" in msg
|
|
|
|
|
|
def test_no_warning_when_at_least_one_description_came_back(caplog_warnings):
|
|
"""Partial coverage (some pictures described, some not) is acceptable
|
|
and does not warn — the user might have area-threshold filtering or
|
|
classification gating."""
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
doc = _doc_with_pictures(with_descriptions=True)
|
|
|
|
_warn_if_descriptions_missing(config, doc, "fine.pdf")
|
|
|
|
assert caplog_warnings == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_emits_warning_via_chokepoint(
|
|
monkeypatch, tmp_path, caplog_warnings
|
|
):
|
|
"""End-to-end: ``convert(...)`` runs the description-missing check
|
|
after the converter returns, so a VLM error swallowed inside
|
|
docling-serve still surfaces as a warning at the haiku.rag layer
|
|
regardless of which converter (local vs serve) ran."""
|
|
from haiku.rag.converters.base import DocumentConverter
|
|
|
|
pdf = tmp_path / "fake.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
class StubConverter(DocumentConverter):
|
|
@property
|
|
def supported_extensions(self) -> list[str]:
|
|
return [".pdf"]
|
|
|
|
async def convert_file(self, path: Path, source_uri: str | None = None):
|
|
return _doc_with_pictures(with_descriptions=False)
|
|
|
|
async def convert_text(
|
|
self,
|
|
text: str,
|
|
name: str = "content.md",
|
|
format: str = "md",
|
|
source_uri: str | None = None,
|
|
):
|
|
return _doc_without_pictures()
|
|
|
|
monkeypatch.setattr(
|
|
"haiku.rag.client.processing.get_converter", lambda config: StubConverter()
|
|
)
|
|
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
|
|
await convert(config, pdf)
|
|
|
|
assert any(
|
|
"0 described" in r.getMessage() and "fake.pdf" in r.getMessage()
|
|
for r in caplog_warnings
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings):
|
|
"""Raw text input (HTML, markdown) can still produce pictures via
|
|
docling, so the description-missing check must run on the
|
|
convert_text branch too — otherwise an HTML-with-images source
|
|
would never trigger the warning even when picture_description is
|
|
enabled and the VLM didn't actually run."""
|
|
from haiku.rag.converters.base import DocumentConverter
|
|
|
|
class StubConverter(DocumentConverter):
|
|
@property
|
|
def supported_extensions(self) -> list[str]:
|
|
return [".html"]
|
|
|
|
async def convert_file(self, path: Path, source_uri: str | None = None):
|
|
return _doc_without_pictures()
|
|
|
|
async def convert_text(
|
|
self,
|
|
text: str,
|
|
name: str = "content.md",
|
|
format: str = "md",
|
|
source_uri: str | None = None,
|
|
):
|
|
return _doc_with_pictures(with_descriptions=False)
|
|
|
|
monkeypatch.setattr(
|
|
"haiku.rag.client.processing.get_converter", lambda config: StubConverter()
|
|
)
|
|
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
|
|
# No URL scheme, no Path → drops into the convert_text branch.
|
|
await convert(config, "<html><img src='...'/></html>")
|
|
|
|
assert any("0 described" in r.getMessage() for r in caplog_warnings)
|
|
|
|
|
|
def test_merge_picture_chunks_no_pictures_returns_text_chunks():
|
|
"""When there are no picture chunks, _merge_picture_chunks returns
|
|
text chunks with order set."""
|
|
from haiku.rag.client.processing import _merge_picture_chunks
|
|
from haiku.rag.store.models.chunk import Chunk
|
|
|
|
doc = _doc_without_pictures()
|
|
text_chunks = [Chunk(content="a"), Chunk(content="b")]
|
|
|
|
result = _merge_picture_chunks(doc, text_chunks, None, None, 0)
|
|
|
|
assert result is text_chunks
|
|
assert [c.order for c in result] == [0, 1]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_dispatches_large_pdfs_through_split_and_merge(
|
|
tmp_path, monkeypatch
|
|
):
|
|
"""With split_pages configured, PDF conversion routes through the
|
|
split-and-merge helper rather than the converter directly."""
|
|
from docling_core.types.doc.document import DoclingDocument
|
|
|
|
config = AppConfig()
|
|
config.processing.split_pages = 2
|
|
|
|
pdf = tmp_path / "big.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4 stub")
|
|
called: dict = {}
|
|
|
|
async def fake_split(converter, path, uri, slice_size):
|
|
called["slice_size"] = slice_size
|
|
called["path"] = path
|
|
return DoclingDocument(name="merged")
|
|
|
|
monkeypatch.setattr(
|
|
"haiku.rag.converters.pdf_split.convert_pdf_with_splitting", fake_split
|
|
)
|
|
|
|
doc = await convert(config, pdf)
|
|
|
|
assert doc.name == "merged"
|
|
assert called["slice_size"] == 2
|
|
assert called["path"] == pdf
|
|
|
|
|
|
def _write_unsupported(directory):
|
|
target = directory / "thing.sqlite3"
|
|
target.write_bytes(b"binary")
|
|
return target.as_uri()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"make_source,match",
|
|
[
|
|
(lambda d: (d / "missing.md").as_uri(), "File does not exist"),
|
|
(_write_unsupported, "Unsupported file extension"),
|
|
],
|
|
ids=["missing_file", "unsupported_extension"],
|
|
)
|
|
async def test_convert_rejects_bad_file_uris(tmp_path, make_source, match):
|
|
from haiku.rag.client.exceptions import UnsupportedSourceError
|
|
|
|
with pytest.raises(UnsupportedSourceError, match=match):
|
|
await convert(AppConfig(), make_source(tmp_path))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_percent_encoded_file_uri(tmp_path):
|
|
"""`Path.as_uri()` encodes brackets and spaces; convert must decode them."""
|
|
target = tmp_path / "a[b] c.md"
|
|
target.write_text("# Heading")
|
|
|
|
doc = await convert(AppConfig(), target.as_uri())
|
|
|
|
assert "Heading" in doc.export_to_markdown()
|