f6acb65e95
Cover the remaining paths in the client, context, downloads, title generation, document tools and store models, and add fail_under=100 so uncovered lines fail CI. Six lines that no test can reach get a pragma with its reason: the docling import guard, the nameless PDF attachment, the FS symlink OSError guard that resolve(strict=False) absorbs, the docling bbox and LanceDB document-id shape guards, the tag-retention branch vacuum makes unreachable, and Monty's Rust-thread print callback. Fix test_find_config_file_user_config, which wrote its config into the cwd it had chdir'd to, so the cwd branch answered first and the user-directory lookup it names was never exercised.
266 lines
8.5 KiB
Python
266 lines
8.5 KiB
Python
"""Tests for haiku.rag.client.processing helpers."""
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from haiku.rag.client.processing import _warn_if_descriptions_missing, convert
|
|
from haiku.rag.config import AppConfig
|
|
from tests.conftest import capture_logs
|
|
|
|
|
|
def _doc_with_pictures(*, with_descriptions: bool):
|
|
"""Build a tiny DoclingDocument carrying one PictureItem.
|
|
|
|
When ``with_descriptions=True`` the picture's ``meta.description.text``
|
|
is populated, simulating a successful VLM call. Otherwise it's left
|
|
empty, simulating a silent VLM failure.
|
|
"""
|
|
from docling_core.types.doc.document import (
|
|
DescriptionMetaField,
|
|
DoclingDocument,
|
|
PictureItem,
|
|
PictureMeta,
|
|
)
|
|
from docling_core.types.doc.labels import DocItemLabel
|
|
|
|
doc = DoclingDocument(name="test")
|
|
pic = PictureItem(
|
|
self_ref="#/pictures/0",
|
|
label=DocItemLabel.PICTURE,
|
|
)
|
|
if with_descriptions:
|
|
pic.meta = PictureMeta(
|
|
description=DescriptionMetaField(text="A red square."),
|
|
)
|
|
doc.pictures.append(pic)
|
|
return doc
|
|
|
|
|
|
def _doc_without_pictures():
|
|
from docling_core.types.doc.document import DoclingDocument
|
|
|
|
return DoclingDocument(name="test")
|
|
|
|
|
|
@pytest.fixture
|
|
def caplog_warnings():
|
|
"""Capture WARNING-level records from the processing logger."""
|
|
from haiku.rag.client.processing import logger as proc_logger
|
|
|
|
with capture_logs(proc_logger, logging.WARNING) as records:
|
|
yield records
|
|
|
|
|
|
def test_no_warning_when_picture_description_disabled(caplog_warnings):
|
|
"""Default config has pictures='image' — even a document full of
|
|
pictures with no descriptions should not warn."""
|
|
config = AppConfig()
|
|
assert config.processing.pictures == "image"
|
|
doc = _doc_with_pictures(with_descriptions=False)
|
|
|
|
_warn_if_descriptions_missing(config, doc, "fake.pdf")
|
|
|
|
assert caplog_warnings == []
|
|
|
|
|
|
def test_no_warning_when_doc_has_no_pictures(caplog_warnings):
|
|
"""A picture-less document under enabled=True shouldn't warn — there
|
|
was simply nothing to describe."""
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
doc = _doc_without_pictures()
|
|
|
|
_warn_if_descriptions_missing(config, doc, "no-pictures.txt")
|
|
|
|
assert caplog_warnings == []
|
|
|
|
|
|
def test_warns_when_pictures_present_but_no_descriptions(caplog_warnings):
|
|
"""VLM was requested via ``processing.pictures='description'``, the
|
|
doc has pictures, but the converter returned zero descriptions
|
|
(docling-serve swallows VLM errors). Warn loudly so the user can fix
|
|
their VLM config before a long ingest."""
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
config.processing.conversion_options.picture_description.model.name = "qwen3.6"
|
|
config.processing.conversion_options.picture_description.model.base_url = (
|
|
"http://host.docker.internal:11434"
|
|
)
|
|
doc = _doc_with_pictures(with_descriptions=False)
|
|
|
|
_warn_if_descriptions_missing(config, doc, "doclaynet.pdf")
|
|
|
|
assert len(caplog_warnings) == 1
|
|
msg = caplog_warnings[0].getMessage()
|
|
assert "doclaynet.pdf" in msg
|
|
assert "1 pictures" in msg
|
|
assert "0 described" in msg
|
|
assert "qwen3.6" in msg
|
|
assert "host.docker.internal" in msg
|
|
|
|
|
|
def test_no_warning_when_at_least_one_description_came_back(caplog_warnings):
|
|
"""Partial coverage (some pictures described, some not) is acceptable
|
|
and does not warn — the user might have area-threshold filtering or
|
|
classification gating."""
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
doc = _doc_with_pictures(with_descriptions=True)
|
|
|
|
_warn_if_descriptions_missing(config, doc, "fine.pdf")
|
|
|
|
assert caplog_warnings == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_emits_warning_via_chokepoint(
|
|
monkeypatch, tmp_path, caplog_warnings
|
|
):
|
|
"""End-to-end: ``convert(...)`` runs the description-missing check
|
|
after the converter returns, so a VLM error swallowed inside
|
|
docling-serve still surfaces as a warning at the haiku.rag layer
|
|
regardless of which converter (local vs serve) ran."""
|
|
from haiku.rag.converters.base import DocumentConverter
|
|
|
|
pdf = tmp_path / "fake.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
class StubConverter(DocumentConverter):
|
|
@property
|
|
def supported_extensions(self) -> list[str]:
|
|
return [".pdf"]
|
|
|
|
async def convert_file(self, path: Path, source_uri: str | None = None):
|
|
return _doc_with_pictures(with_descriptions=False)
|
|
|
|
async def convert_text(
|
|
self,
|
|
text: str,
|
|
name: str = "content.md",
|
|
format: str = "md",
|
|
source_uri: str | None = None,
|
|
):
|
|
return _doc_without_pictures()
|
|
|
|
monkeypatch.setattr(
|
|
"haiku.rag.client.processing.get_converter", lambda config: StubConverter()
|
|
)
|
|
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
|
|
await convert(config, pdf)
|
|
|
|
assert any(
|
|
"0 described" in r.getMessage() and "fake.pdf" in r.getMessage()
|
|
for r in caplog_warnings
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings):
|
|
"""Raw text input (HTML, markdown) can still produce pictures via
|
|
docling, so the description-missing check must run on the
|
|
convert_text branch too — otherwise an HTML-with-images source
|
|
would never trigger the warning even when picture_description is
|
|
enabled and the VLM didn't actually run."""
|
|
from haiku.rag.converters.base import DocumentConverter
|
|
|
|
class StubConverter(DocumentConverter):
|
|
@property
|
|
def supported_extensions(self) -> list[str]:
|
|
return [".html"]
|
|
|
|
async def convert_file(self, path: Path, source_uri: str | None = None):
|
|
return _doc_without_pictures()
|
|
|
|
async def convert_text(
|
|
self,
|
|
text: str,
|
|
name: str = "content.md",
|
|
format: str = "md",
|
|
source_uri: str | None = None,
|
|
):
|
|
return _doc_with_pictures(with_descriptions=False)
|
|
|
|
monkeypatch.setattr(
|
|
"haiku.rag.client.processing.get_converter", lambda config: StubConverter()
|
|
)
|
|
|
|
config = AppConfig()
|
|
config.processing.pictures = "description"
|
|
|
|
# No URL scheme, no Path → drops into the convert_text branch.
|
|
await convert(config, "<html><img src='...'/></html>")
|
|
|
|
assert any("0 described" in r.getMessage() for r in caplog_warnings)
|
|
|
|
|
|
def test_merge_picture_chunks_no_pictures_returns_text_chunks():
|
|
"""When there are no picture chunks, _merge_picture_chunks returns
|
|
text chunks with order set."""
|
|
from haiku.rag.client.processing import _merge_picture_chunks
|
|
from haiku.rag.store.models.chunk import Chunk
|
|
|
|
doc = _doc_without_pictures()
|
|
text_chunks = [Chunk(content="a"), Chunk(content="b")]
|
|
|
|
result = _merge_picture_chunks(doc, text_chunks, None, None, 0)
|
|
|
|
assert result is text_chunks
|
|
assert [c.order for c in result] == [0, 1]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_dispatches_large_pdfs_through_split_and_merge(
|
|
tmp_path, monkeypatch
|
|
):
|
|
"""With split_pages configured, PDF conversion routes through the
|
|
split-and-merge helper rather than the converter directly."""
|
|
from docling_core.types.doc.document import DoclingDocument
|
|
|
|
config = AppConfig()
|
|
config.processing.split_pages = 2
|
|
|
|
pdf = tmp_path / "big.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4 stub")
|
|
called: dict = {}
|
|
|
|
async def fake_split(converter, path, uri, slice_size):
|
|
called["slice_size"] = slice_size
|
|
called["path"] = path
|
|
return DoclingDocument(name="merged")
|
|
|
|
monkeypatch.setattr(
|
|
"haiku.rag.converters.pdf_split.convert_pdf_with_splitting", fake_split
|
|
)
|
|
|
|
doc = await convert(config, pdf)
|
|
|
|
assert doc.name == "merged"
|
|
assert called["slice_size"] == 2
|
|
assert called["path"] == pdf
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"make_source,match",
|
|
[
|
|
(lambda d: (d / "missing.md").as_uri(), "File does not exist"),
|
|
(lambda d: _write_unsupported(d), "Unsupported file extension"),
|
|
],
|
|
ids=["missing_file", "unsupported_extension"],
|
|
)
|
|
async def test_convert_rejects_bad_file_uris(tmp_path, make_source, match):
|
|
from haiku.rag.client.exceptions import UnsupportedSourceError
|
|
|
|
with pytest.raises(UnsupportedSourceError, match=match):
|
|
await convert(AppConfig(), make_source(tmp_path))
|
|
|
|
|
|
def _write_unsupported(directory):
|
|
target = directory / "thing.sqlite3"
|
|
target.write_bytes(b"binary")
|
|
return target.as_uri()
|