diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb1a98d..4e6f7760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,17 @@ ### Added +- **`processing.conversion_options.fetch_remote_images`** (default `true`). Controls whether docling fetches images referenced by URL in HTML and Markdown inputs. docling-local only — docling-serve cannot fetch external images via its API regardless of this flag. - **`s3://` is a first-class document source.** `create_document_from_source`, the CLI `haiku-rag add-src`, and the MCP `add_document_from_url` tool all dispatch on the `s3` URL scheme. Two-stage change detection keeps `metadata["md5"]` semantically uniform across all sources: HEAD ETag matching the stored `metadata["etag"]` short-circuits without GET; if ETag differs but bytes hash to the same MD5 (multipart re-upload, server-side `CopyObject`, SSE mode change), only the etag refreshes — no re-chunk or re-embed. Closes #357. - **S3 / object-storage monitoring.** `monitor.s3: list[S3MonitorEntry]` adds a polling watcher per bucket prefix alongside the existing local-directory watcher. Each entry has its own `poll_interval`, `include_patterns`, `ignore_patterns`, `delete_orphans`, and `storage_options`. The same `serve --monitor` flag enables both. Orphan deletion is per-entry (scoped via `uri LIKE 's3://bucket/prefix/%'`); other buckets and prefixes are never touched. - **`[s3]` optional extra** (`obstore>=0.9`). Required for `s3://` sources and the S3 watcher. Uses obstore — the Python binding to the same Rust `object_store` crate that LanceDB uses internally — so `monitor.s3[*].storage_options` accepts the same dict shape as `lancedb.storage_options`. Empty/missing options fall back to the AWS default credential chain. - **`scripts/run-integration-tests.sh`** — wraps `docker compose up --wait`, `pytest -m integration`, and tear-down so the SeaweedFS-backed integration suite is a one-liner. +### Fixed + +- **Conversion options now apply to non-PDF formats.** `DoclingLocalConverter` previously wired its `PdfPipelineOptions` only to `InputFormat.PDF`, so user settings (OCR knobs, `picture_description.enabled`, `images_scale`, etc.) silently no-op'd for HTML, Markdown, DOCX, PPTX, and IMAGE inputs. The converter now shares a single `PdfPipelineOptions` instance across PDF, IMAGE, HTML, MD, DOCX, and PPTX `FormatOption`s. SimplePipeline-backed formats ignore the PDF-specific fields; `ConvertPipelineOptions`-level enrichments (picture description / classification / chart extraction) now run uniformly. HTML and Markdown additionally receive `HTMLBackendOptions` / `MarkdownBackendOptions` gated on `fetch_remote_images`. +- **HTML text ingest path picks up converter options.** `convert_text(format="html"/"md")` previously used a bare `DoclingDocConverter()` with zero format options — the wix corpus ingest path. It now uses the same shared `_build_format_options()` helper as the file path. + ### Documentation - New "S3 / Object Storage Monitoring" section in `docs/server.md` and `docs/configuration/processing.md` covering the `[s3]` extra, polling cadence, ETag semantics, credentials, and CLI usage. diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index a46cedc3..c9b6adf6 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -161,6 +161,10 @@ class ConversionOptions(BaseModel): images_scale: float = 2.0 generate_page_images: bool = True + # Fetch images referenced by URL in HTML and Markdown inputs. + # docling-local only — docling-serve cannot fetch external images. + fetch_remote_images: bool = True + picture_description: PictureDescriptionConfig = Field( default_factory=PictureDescriptionConfig ) diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index baf7456e..a63564b8 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -2,13 +2,15 @@ import asyncio from pathlib import Path -from typing import TYPE_CHECKING, ClassVar, cast +from typing import TYPE_CHECKING, ClassVar from haiku.rag.config import AppConfig from haiku.rag.converters.base import DocumentConverter from haiku.rag.converters.text_utils import TextFileHandler if TYPE_CHECKING: + from docling.datamodel.base_models import InputFormat + from docling.document_converter import FormatOption from docling_core.types.doc.document import DoclingDocument from haiku.rag.config.models import ConversionOptions, ModelConfig @@ -103,23 +105,17 @@ class DoclingLocalConverter(DocumentConverter): case _: # "auto" or any other value return OcrAutoOptions(force_full_page_ocr=force_ocr, lang=lang) - def _sync_convert_docling_file(self, path: Path) -> "DoclingDocument": - """Synchronous conversion of docling-supported files.""" - from docling.backend.docling_parse_backend import DoclingParseDocumentBackend - from docling.datamodel.base_models import InputFormat + def _build_pipeline_options(self): + """Build the shared PdfPipelineOptions instance applied to every wired + FormatOption. SimplePipeline-backed formats ignore the PDF-specific + fields; the ConvertPipelineOptions-level picture description / + classification / chart-extraction settings apply uniformly.""" from docling.datamodel.pipeline_options import ( PdfPipelineOptions, PictureDescriptionApiOptions, TableFormerMode, TableStructureOptions, ) - from docling.document_converter import ( - DocumentConverter as DoclingDocConverter, - ) - from docling.document_converter import ( - FormatOption, - PdfFormatOption, - ) opts = self.config.processing.conversion_options pic_desc = opts.picture_description @@ -146,8 +142,6 @@ class DoclingLocalConverter(DocumentConverter): if runs_vlm: from pydantic import AnyUrl - prompt = self.config.prompts.picture_description - pipeline_options.enable_remote_services = True pipeline_options.picture_description_options = PictureDescriptionApiOptions( url=AnyUrl(self._get_vlm_api_url(pic_desc.model)), @@ -155,21 +149,70 @@ class DoclingLocalConverter(DocumentConverter): model=pic_desc.model.name, max_completion_tokens=pic_desc.max_tokens, ), - prompt=prompt, + prompt=self.config.prompts.picture_description, timeout=pic_desc.timeout, ) - format_options = cast( - dict[InputFormat, FormatOption], - { - InputFormat.PDF: PdfFormatOption( - pipeline_options=pipeline_options, - backend=DoclingParseDocumentBackend, - ) - }, + return pipeline_options + + def _build_format_options(self) -> "dict[InputFormat, FormatOption]": + """Per-format options shared between file and text conversion paths. + + Every wired FormatOption gets the same `PdfPipelineOptions` instance so + picture-description / classification / chart settings apply uniformly + across PDF, IMAGE, HTML, MD, DOCX, PPTX. HTML and Markdown additionally + receive backend options gated on `fetch_remote_images`. + """ + from docling.backend.docling_parse_backend import DoclingParseDocumentBackend + from docling.datamodel.backend_options import ( + HTMLBackendOptions, + MarkdownBackendOptions, + ) + from docling.datamodel.base_models import InputFormat + from docling.document_converter import ( + HTMLFormatOption, + ImageFormatOption, + MarkdownFormatOption, + PdfFormatOption, + PowerpointFormatOption, + WordFormatOption, ) - converter = DoclingDocConverter(format_options=format_options) + opts = self.config.processing.conversion_options + pipeline_options = self._build_pipeline_options() + fetch = opts.fetch_remote_images + + return { + InputFormat.PDF: PdfFormatOption( + pipeline_options=pipeline_options, + backend=DoclingParseDocumentBackend, + ), + InputFormat.IMAGE: ImageFormatOption(pipeline_options=pipeline_options), + InputFormat.HTML: HTMLFormatOption( + pipeline_options=pipeline_options, + backend_options=HTMLBackendOptions( + fetch_images=fetch, + enable_remote_fetch=fetch, + ), + ), + InputFormat.MD: MarkdownFormatOption( + pipeline_options=pipeline_options, + backend_options=MarkdownBackendOptions( + fetch_images=fetch, + enable_remote_fetch=fetch, + ), + ), + InputFormat.DOCX: WordFormatOption(pipeline_options=pipeline_options), + InputFormat.PPTX: PowerpointFormatOption(pipeline_options=pipeline_options), + } + + def _sync_convert_docling_file(self, path: Path) -> "DoclingDocument": + """Synchronous conversion of docling-supported files.""" + from docling.document_converter import ( + DocumentConverter as DoclingDocConverter, + ) + + converter = DoclingDocConverter(format_options=self._build_format_options()) result = converter.convert(path) return result.document @@ -219,4 +262,39 @@ class DoclingLocalConverter(DocumentConverter): Raises: ValueError: If the text cannot be converted or format is unsupported. """ - return await TextFileHandler.text_to_docling_document(text, name, format) + if format not in TextFileHandler.SUPPORTED_FORMATS: + raise ValueError( + f"Unsupported format: {format}. " + f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}" + ) + + doc_name = f"content.{format}" if name == "content.md" else name + + if format == "plain": + return TextFileHandler._create_simple_docling_document(text, doc_name) + + try: + return await asyncio.to_thread( + self._sync_convert_docling_text, text, doc_name + ) + except Exception as e: + raise ValueError(f"Failed to convert text to DoclingDocument: {e}") + + def _sync_convert_docling_text(self, text: str, doc_name: str) -> "DoclingDocument": + """Synchronous text-to-DoclingDocument using the shared format options.""" + from io import BytesIO + + from docling.document_converter import ( + DocumentConverter as DoclingDocConverter, + ) + from docling.exceptions import ConversionError + from docling_core.types.io import DocumentStream + + bytes_io = BytesIO(text.encode("utf-8")) + doc_stream = DocumentStream(name=doc_name, stream=bytes_io) + converter = DoclingDocConverter(format_options=self._build_format_options()) + try: + result = converter.convert(doc_stream) + return result.document + except ConversionError: + return TextFileHandler._create_simple_docling_document(text, doc_name) diff --git a/haiku_rag_slim/haiku/rag/converters/text_utils.py b/haiku_rag_slim/haiku/rag/converters/text_utils.py index 270ddf1f..c51a945d 100644 --- a/haiku_rag_slim/haiku/rag/converters/text_utils.py +++ b/haiku_rag_slim/haiku/rag/converters/text_utils.py @@ -1,7 +1,5 @@ """Shared utilities for text file handling in converters.""" -import asyncio -from io import BytesIO from typing import TYPE_CHECKING, ClassVar if TYPE_CHECKING: @@ -183,61 +181,3 @@ class TextFileHandler: doc = DoclingDocument(name=doc_name) doc.add_text(label=DocItemLabel.TEXT, text=text) return doc - - @staticmethod - def _sync_text_to_docling_document( - text: str, name: str = "content.md", format: str = "md" - ) -> "DoclingDocument": - """Synchronous implementation of text to DoclingDocument conversion.""" - from docling.document_converter import DocumentConverter as DoclingDocConverter - from docling.exceptions import ConversionError - from docling_core.types.io import DocumentStream - - if format not in TextFileHandler.SUPPORTED_FORMATS: - raise ValueError( - f"Unsupported format: {format}. " - f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}" - ) - - # Derive document name from format to tell docling which parser to use - doc_name = f"content.{format}" if name == "content.md" else name - - # Plain text doesn't need parsing - create document directly - if format == "plain": - return TextFileHandler._create_simple_docling_document(text, doc_name) - - bytes_io = BytesIO(text.encode("utf-8")) - doc_stream = DocumentStream(name=doc_name, stream=bytes_io) - converter = DoclingDocConverter() - try: - result = converter.convert(doc_stream) - return result.document - except ConversionError: - # Docling's format detection fails for plain text without markdown syntax. - # Fall back to creating a simple document directly. - return TextFileHandler._create_simple_docling_document(text, doc_name) - - @staticmethod - async def text_to_docling_document( - text: str, name: str = "content.md", format: str = "md" - ) -> "DoclingDocument": - """Convert text to DoclingDocument using docling's parser. - - Args: - text: The text content to convert. - name: The name to use for the document. - format: The format of the text content ("md", "html", or "plain"). - Defaults to "md". Use "plain" for plain text without parsing. - - Returns: - DoclingDocument representation of the text. - - Raises: - ValueError: If the conversion fails or format is unsupported. - """ - try: - return await asyncio.to_thread( - TextFileHandler._sync_text_to_docling_document, text, name, format - ) - except Exception as e: - raise ValueError(f"Failed to convert text to DoclingDocument: {e}") diff --git a/tests/test_config.py b/tests/test_config.py index 2f778e58..95eab0b3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -272,3 +272,31 @@ processing: assert cfg.processing.conversion_options.picture_description.enabled is False assert "generate_picture_images" not in data["processing"]["conversion_options"] assert any("generate_picture_images" in r.getMessage() for r in handler.records) + + +def test_fetch_remote_images_default_true(): + """`fetch_remote_images` defaults to True and round-trips through YAML.""" + from haiku.rag.config.models import ConversionOptions + + assert ConversionOptions().fetch_remote_images is True + + cfg = AppConfig() + assert cfg.processing.conversion_options.fetch_remote_images is True + + data = generate_default_config() + assert data["processing"]["conversion_options"]["fetch_remote_images"] is True + + +def test_fetch_remote_images_override_via_yaml(tmp_path): + """User can disable image fetching via YAML.""" + config_file = _write( + tmp_path, + """ +processing: + conversion_options: + fetch_remote_images: false +""", + ) + data = load_yaml_config(config_file) + cfg = AppConfig.model_validate(data) + assert cfg.processing.conversion_options.fetch_remote_images is False diff --git a/tests/test_converters.py b/tests/test_converters.py index de2fc97b..d5e17645 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -349,6 +349,130 @@ class TestDoclingLocalConverter: converter.config.processing.conversion_options.generate_page_images is False ) + @pytest.mark.asyncio + async def test_convert_text_html_fetches_data_uri_image(self, config): + """`fetch_remote_images=True` decodes inline `data:` URIs into picture + bytes via the HTML backend. Default behavior.""" + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk" + "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + html = f'

before

dot

after

' + converter = DoclingLocalConverter(config) + + doc = await converter.convert_text(html, format="html") + + assert doc.pictures, "HTML with should yield picture items" + pics_with_image = [p for p in doc.pictures if p.image is not None] + assert len(pics_with_image) == len(doc.pictures), ( + "All with valid data: URIs should have decoded bytes" + ) + + @pytest.mark.asyncio + async def test_convert_text_html_no_fetch_when_disabled(self, config): + """`fetch_remote_images=False` produces placeholder pictures with no + bytes — even for inline `data:` URIs (docling's `fetch_images` gates + all image decoding, not just remote fetches).""" + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk" + "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + html = f'' + config.processing.conversion_options.fetch_remote_images = False + converter = DoclingLocalConverter(config) + + doc = await converter.convert_text(html, format="html") + + assert doc.pictures, "Picture placeholders are still emitted" + for pic in doc.pictures: + assert pic.image is None, ( + "fetch_remote_images=False must leave picture.image=None" + ) + + @pytest.mark.asyncio + async def test_convert_text_md_html_block_fetches_data_uri_image(self, config): + """Markdown with an embedded `` HTML block produces picture bytes + — proves the MarkdownBackendOptions wiring delegates to the HTML + backend with our `fetch_images` / `enable_remote_fetch` settings. + + Note: docling's md backend does NOT fetch images from native + `![alt](url)` syntax — only from embedded HTML blocks. That's an + upstream limitation, not something this PR can address. + """ + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk" + "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + md = ( + f"# Title\n\nIntro paragraph.\n\n" + f'dot\n\n' + f"Trailing paragraph.\n" + ) + converter = DoclingLocalConverter(config) + + doc = await converter.convert_text(md, format="md") + + assert doc.pictures, "MD with HTML block should yield picture items" + pics_with_image = [p for p in doc.pictures if p.image is not None] + assert len(pics_with_image) == len(doc.pictures) + + def test_build_format_options_covers_pdf_image_html_md_docx_pptx(self, config): + """`_build_format_options()` registers every format we care about and + shares the same `PdfPipelineOptions` instance across them so + picture-description / classification / chart settings apply uniformly.""" + from docling.datamodel.base_models import InputFormat + + converter = DoclingLocalConverter(config) + options = converter._build_format_options() + + wired = { + InputFormat.PDF, + InputFormat.IMAGE, + InputFormat.HTML, + InputFormat.MD, + InputFormat.DOCX, + InputFormat.PPTX, + } + assert wired <= set(options.keys()), ( + f"Missing format options: {wired - set(options.keys())}" + ) + + pdf_opts = options[InputFormat.PDF].pipeline_options + for fmt in wired: + assert options[fmt].pipeline_options is pdf_opts, ( + f"{fmt} must share the PDF pipeline_options instance" + ) + + def test_build_format_options_propagates_fetch_remote_images(self, config): + """HTML and Markdown FormatOptions reflect `fetch_remote_images`.""" + from docling.datamodel.backend_options import ( + HTMLBackendOptions, + MarkdownBackendOptions, + ) + from docling.datamodel.base_models import InputFormat + + config.processing.conversion_options.fetch_remote_images = True + opts = DoclingLocalConverter(config)._build_format_options() + html_bo = opts[InputFormat.HTML].backend_options + md_bo = opts[InputFormat.MD].backend_options + assert isinstance(html_bo, HTMLBackendOptions) + assert html_bo.fetch_images is True + assert html_bo.enable_remote_fetch is True + assert isinstance(md_bo, MarkdownBackendOptions) + assert md_bo.fetch_images is True + assert md_bo.enable_remote_fetch is True + + config.processing.conversion_options.fetch_remote_images = False + opts = DoclingLocalConverter(config)._build_format_options() + html_bo = opts[InputFormat.HTML].backend_options + md_bo = opts[InputFormat.MD].backend_options + assert isinstance(html_bo, HTMLBackendOptions) + assert html_bo.fetch_images is False + assert html_bo.enable_remote_fetch is False + assert isinstance(md_bo, MarkdownBackendOptions) + assert md_bo.fetch_images is False + assert md_bo.enable_remote_fetch is False + @pytest.mark.asyncio async def test_convert_pdf_with_picture_images(self, config): """Picture bytes are produced by the local converter for PDFs that