"""Tests for haiku.rag.client.processing helpers.""" import logging from pathlib import Path import pytest from haiku.rag.client.processing import _warn_if_descriptions_missing, convert from haiku.rag.config import AppConfig from tests.conftest import capture_logs def _doc_with_pictures(*, with_descriptions: bool): """Build a tiny DoclingDocument carrying one PictureItem. When ``with_descriptions=True`` the picture's ``meta.description.text`` is populated, simulating a successful VLM call. Otherwise it's left empty, simulating a silent VLM failure. """ from docling_core.types.doc.document import ( DescriptionMetaField, DoclingDocument, PictureItem, PictureMeta, ) from docling_core.types.doc.labels import DocItemLabel doc = DoclingDocument(name="test") pic = PictureItem( self_ref="#/pictures/0", label=DocItemLabel.PICTURE, ) if with_descriptions: pic.meta = PictureMeta( description=DescriptionMetaField(text="A red square."), ) doc.pictures.append(pic) return doc def _doc_without_pictures(): from docling_core.types.doc.document import DoclingDocument return DoclingDocument(name="test") @pytest.fixture def caplog_warnings(): """Capture WARNING-level records from the processing logger.""" from haiku.rag.client.processing import logger as proc_logger with capture_logs(proc_logger, logging.WARNING) as records: yield records def test_no_warning_when_picture_description_disabled(caplog_warnings): """Default config has pictures='image' — even a document full of pictures with no descriptions should not warn.""" config = AppConfig() assert config.processing.pictures == "image" doc = _doc_with_pictures(with_descriptions=False) _warn_if_descriptions_missing(config, doc, "fake.pdf") assert caplog_warnings == [] def test_no_warning_when_doc_has_no_pictures(caplog_warnings): """A picture-less document under enabled=True shouldn't warn — there was simply nothing to describe.""" config = AppConfig() config.processing.pictures = "description" doc = _doc_without_pictures() _warn_if_descriptions_missing(config, doc, "no-pictures.txt") assert caplog_warnings == [] def test_warns_when_pictures_present_but_no_descriptions(caplog_warnings): """VLM was requested via ``processing.pictures='description'``, the doc has pictures, but the converter returned zero descriptions (docling-serve swallows VLM errors). Warn loudly so the user can fix their VLM config before a long ingest.""" config = AppConfig() config.processing.pictures = "description" config.processing.conversion_options.picture_description.model.name = "qwen3.6" config.processing.conversion_options.picture_description.model.base_url = ( "http://host.docker.internal:11434" ) doc = _doc_with_pictures(with_descriptions=False) _warn_if_descriptions_missing(config, doc, "doclaynet.pdf") assert len(caplog_warnings) == 1 msg = caplog_warnings[0].getMessage() assert "doclaynet.pdf" in msg assert "1 pictures" in msg assert "0 described" in msg assert "qwen3.6" in msg assert "host.docker.internal" in msg def test_no_warning_when_at_least_one_description_came_back(caplog_warnings): """Partial coverage (some pictures described, some not) is acceptable and does not warn — the user might have area-threshold filtering or classification gating.""" config = AppConfig() config.processing.pictures = "description" doc = _doc_with_pictures(with_descriptions=True) _warn_if_descriptions_missing(config, doc, "fine.pdf") assert caplog_warnings == [] @pytest.mark.asyncio async def test_convert_emits_warning_via_chokepoint( monkeypatch, tmp_path, caplog_warnings ): """End-to-end: ``convert(...)`` runs the description-missing check after the converter returns, so a VLM error swallowed inside docling-serve still surfaces as a warning at the haiku.rag layer regardless of which converter (local vs serve) ran.""" from haiku.rag.converters.base import DocumentConverter pdf = tmp_path / "fake.pdf" pdf.write_bytes(b"%PDF-1.4 stub") class StubConverter(DocumentConverter): @property def supported_extensions(self) -> list[str]: return [".pdf"] async def convert_file(self, path: Path, source_uri: str | None = None): return _doc_with_pictures(with_descriptions=False) async def convert_text( self, text: str, name: str = "content.md", format: str = "md", source_uri: str | None = None, ): return _doc_without_pictures() monkeypatch.setattr( "haiku.rag.client.processing.get_converter", lambda config: StubConverter() ) config = AppConfig() config.processing.pictures = "description" await convert(config, pdf) assert any( "0 described" in r.getMessage() and "fake.pdf" in r.getMessage() for r in caplog_warnings ) @pytest.mark.asyncio async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings): """Raw text input (HTML, markdown) can still produce pictures via docling, so the description-missing check must run on the convert_text branch too — otherwise an HTML-with-images source would never trigger the warning even when picture_description is enabled and the VLM didn't actually run.""" from haiku.rag.converters.base import DocumentConverter class StubConverter(DocumentConverter): @property def supported_extensions(self) -> list[str]: return [".html"] async def convert_file(self, path: Path, source_uri: str | None = None): return _doc_without_pictures() async def convert_text( self, text: str, name: str = "content.md", format: str = "md", source_uri: str | None = None, ): return _doc_with_pictures(with_descriptions=False) monkeypatch.setattr( "haiku.rag.client.processing.get_converter", lambda config: StubConverter() ) config = AppConfig() config.processing.pictures = "description" # No URL scheme, no Path → drops into the convert_text branch. await convert(config, "") assert any("0 described" in r.getMessage() for r in caplog_warnings) def test_merge_picture_chunks_no_pictures_returns_text_chunks(): """When there are no picture chunks, _merge_picture_chunks returns text chunks with order set.""" from haiku.rag.client.processing import _merge_picture_chunks from haiku.rag.store.models.chunk import Chunk doc = _doc_without_pictures() text_chunks = [Chunk(content="a"), Chunk(content="b")] result = _merge_picture_chunks(doc, text_chunks, None, None, 0) assert result is text_chunks assert [c.order for c in result] == [0, 1] @pytest.mark.asyncio async def test_convert_dispatches_large_pdfs_through_split_and_merge( tmp_path, monkeypatch ): """With split_pages configured, PDF conversion routes through the split-and-merge helper rather than the converter directly.""" from docling_core.types.doc.document import DoclingDocument config = AppConfig() config.processing.split_pages = 2 pdf = tmp_path / "big.pdf" pdf.write_bytes(b"%PDF-1.4 stub") called: dict = {} async def fake_split(converter, path, uri, slice_size): called["slice_size"] = slice_size called["path"] = path return DoclingDocument(name="merged") monkeypatch.setattr( "haiku.rag.converters.pdf_split.convert_pdf_with_splitting", fake_split ) doc = await convert(config, pdf) assert doc.name == "merged" assert called["slice_size"] == 2 assert called["path"] == pdf def _write_unsupported(directory): target = directory / "thing.sqlite3" target.write_bytes(b"binary") return target.as_uri() @pytest.mark.asyncio @pytest.mark.parametrize( "make_source,match", [ (lambda d: (d / "missing.md").as_uri(), "File does not exist"), (_write_unsupported, "Unsupported file extension"), ], ids=["missing_file", "unsupported_extension"], ) async def test_convert_rejects_bad_file_uris(tmp_path, make_source, match): from haiku.rag.client.exceptions import UnsupportedSourceError with pytest.raises(UnsupportedSourceError, match=match): await convert(AppConfig(), make_source(tmp_path)) @pytest.mark.asyncio async def test_convert_percent_encoded_file_uri(tmp_path): """`Path.as_uri()` encodes brackets and spaces; convert must decode them.""" target = tmp_path / "a[b] c.md" target.write_text("# Heading") doc = await convert(AppConfig(), target.as_uri()) assert "Heading" in doc.export_to_markdown()