From afd4665517ce29639ca3ab21ed03ac5fbbf3abba Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 12 May 2026 15:13:02 +0300 Subject: [PATCH] Thread source_uri through URL ingest --- CHANGELOG.md | 1 + haiku_rag_slim/haiku/rag/client/__init__.py | 14 +++-- haiku_rag_slim/haiku/rag/client/documents.py | 4 +- haiku_rag_slim/haiku/rag/client/processing.py | 21 +++++-- haiku_rag_slim/haiku/rag/converters/base.py | 17 ++++- .../haiku/rag/converters/docling_local.py | 59 +++++++++++++---- .../haiku/rag/converters/docling_serve.py | 16 ++++- tests/test_converters.py | 63 +++++++++++++++++++ tests/test_processing.py | 16 +++-- 9 files changed, 181 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6f7760..f88ae7d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - **Conversion options now apply to non-PDF formats.** `DoclingLocalConverter` previously wired its `PdfPipelineOptions` only to `InputFormat.PDF`, so user settings (OCR knobs, `picture_description.enabled`, `images_scale`, etc.) silently no-op'd for HTML, Markdown, DOCX, PPTX, and IMAGE inputs. The converter now shares a single `PdfPipelineOptions` instance across PDF, IMAGE, HTML, MD, DOCX, and PPTX `FormatOption`s. SimplePipeline-backed formats ignore the PDF-specific fields; `ConvertPipelineOptions`-level enrichments (picture description / classification / chart extraction) now run uniformly. HTML and Markdown additionally receive `HTMLBackendOptions` / `MarkdownBackendOptions` gated on `fetch_remote_images`. - **HTML text ingest path picks up converter options.** `convert_text(format="html"/"md")` previously used a bare `DoclingDocConverter()` with zero format options — the wix corpus ingest path. It now uses the same shared `_build_format_options()` helper as the file path. +- **Relative `` paths resolve during URL ingest.** `HaikuRAG.convert()` and the converter `convert_file` / `convert_text` methods now thread a `source_uri` through to `HTMLBackendOptions.source_uri` / `MarkdownBackendOptions.source_uri`. URL ingest uses the originating URL; file ingest uses `file://`; raw text accepts an optional override. docling-serve accepts the kwarg as a no-op (its API has no equivalent option). ### Documentation diff --git a/haiku_rag_slim/haiku/rag/client/__init__.py b/haiku_rag_slim/haiku/rag/client/__init__.py index 1413757f..3a26a533 100644 --- a/haiku_rag_slim/haiku/rag/client/__init__.py +++ b/haiku_rag_slim/haiku/rag/client/__init__.py @@ -139,19 +139,25 @@ class HaikuRAG: # ========================================================================= @overload - async def convert(self, source: Path) -> "DoclingDocument": ... + async def convert( + self, source: Path, *, source_uri: str | None = None + ) -> "DoclingDocument": ... @overload async def convert( - self, source: str, *, format: str = "md" + self, source: str, *, format: str = "md", source_uri: str | None = None ) -> "DoclingDocument": ... async def convert( - self, source: Path | str, *, format: str = "md" + self, + source: Path | str, + *, + format: str = "md", + source_uri: str | None = None, ) -> "DoclingDocument": from haiku.rag.client.processing import convert - return await convert(self._config, source, format=format) + return await convert(self._config, source, format=format, source_uri=source_uri) async def chunk( self, diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index 100bc51a..605c8950 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -411,7 +411,7 @@ async def _create_or_update_document_from_url( temp_path = Path(temp_file.name) try: - docling_document = await client.convert(temp_path) + docling_document = await client.convert(temp_path, source_uri=url) chunks = await client.chunk(docling_document) embedded_chunks = await embed_chunks(chunks, client._config) finally: @@ -547,7 +547,7 @@ async def _create_or_update_document_from_s3( return existing_doc try: - docling_document = await client.convert(temp_path) + docling_document = await client.convert(temp_path, source_uri=url) chunks = await client.chunk(docling_document) embedded_chunks = await embed_chunks(chunks, client._config) finally: diff --git a/haiku_rag_slim/haiku/rag/client/processing.py b/haiku_rag_slim/haiku/rag/client/processing.py index 8ccec652..b1051323 100644 --- a/haiku_rag_slim/haiku/rag/client/processing.py +++ b/haiku_rag_slim/haiku/rag/client/processing.py @@ -52,7 +52,11 @@ def _warn_if_descriptions_missing( async def convert( - config: AppConfig, source: Path | str, *, format: str = "md" + config: AppConfig, + source: Path | str, + *, + format: str = "md", + source_uri: str | None = None, ) -> "DoclingDocument": """Convert a file, URL, or text to DoclingDocument. @@ -66,6 +70,10 @@ async def convert( Defaults to "md". Use "plain" for plain text without parsing. Only used when source is raw text (not a file path or URL). Files and URLs determine format from extension/content-type. + source_uri: Origin URI used by docling's HTML/Markdown backends to + resolve relative `` references. When omitted, + defaults to the URL (URL ingest) or `file://` URI (file ingest); + raw text input has no origin so no default is derived. Returns: DoclingDocument from the converted source. @@ -82,7 +90,8 @@ async def convert( raise ValueError(f"File does not exist: {source}") if source.suffix.lower() not in converter.supported_extensions: raise ValueError(f"Unsupported file extension: {source.suffix}") - doc = await converter.convert_file(source) + effective_uri = source_uri or source.absolute().as_uri() + doc = await converter.convert_file(source, source_uri=effective_uri) _warn_if_descriptions_missing(config, doc, str(source)) return doc @@ -113,7 +122,8 @@ async def convert( temp_path = Path(temp_file.name) try: - doc = await converter.convert_file(temp_path) + effective_uri = source_uri or source + doc = await converter.convert_file(temp_path, source_uri=effective_uri) _warn_if_descriptions_missing(config, doc, source) return doc finally: @@ -126,14 +136,15 @@ async def convert( raise ValueError(f"File does not exist: {file_path}") if file_path.suffix.lower() not in converter.supported_extensions: raise ValueError(f"Unsupported file extension: {file_path.suffix}") - doc = await converter.convert_file(file_path) + effective_uri = source_uri or file_path.absolute().as_uri() + doc = await converter.convert_file(file_path, source_uri=effective_uri) _warn_if_descriptions_missing(config, doc, str(file_path)) return doc else: # Raw text content — HTML and markdown can still embed pictures # via /![](...) so the same description check applies. - doc = await converter.convert_text(source, format=format) + doc = await converter.convert_text(source, format=format, source_uri=source_uri) _warn_if_descriptions_missing(config, doc, "") return doc diff --git a/haiku_rag_slim/haiku/rag/converters/base.py b/haiku_rag_slim/haiku/rag/converters/base.py index e1ccbc7a..d6aea337 100644 --- a/haiku_rag_slim/haiku/rag/converters/base.py +++ b/haiku_rag_slim/haiku/rag/converters/base.py @@ -26,11 +26,18 @@ class DocumentConverter(ABC): pass @abstractmethod - async def convert_file(self, path: Path) -> "DoclingDocument": + async def convert_file( + self, path: Path, source_uri: str | None = None + ) -> "DoclingDocument": """Convert a file to DoclingDocument format. Args: path: Path to the file to convert. + source_uri: Optional origin URI (e.g. the URL the file was + downloaded from) used by docling's HTML/Markdown backends to + resolve relative `` references. Ignored by + converters that have no equivalent backend option (notably + docling-serve). Returns: DoclingDocument representation of the file. @@ -44,7 +51,11 @@ class DocumentConverter(ABC): @abstractmethod async def convert_text( - self, text: str, name: str = "content.md", format: str = "md" + self, + text: str, + name: str = "content.md", + format: str = "md", + source_uri: str | None = None, ) -> "DoclingDocument": """Convert text content to DoclingDocument format. @@ -53,6 +64,8 @@ class DocumentConverter(ABC): name: The name to use for the document (defaults to "content.md"). format: The format of the text content ("md", "html", or "plain"). Defaults to "md". Use "plain" for plain text without parsing. + source_uri: Optional origin URI used by docling's HTML/Markdown + backends to resolve relative image references. Returns: DoclingDocument representation of the text. diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index a63564b8..3028b59c 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -155,13 +155,20 @@ class DoclingLocalConverter(DocumentConverter): return pipeline_options - def _build_format_options(self) -> "dict[InputFormat, FormatOption]": + def _build_format_options( + self, source_uri: str | None = None + ) -> "dict[InputFormat, FormatOption]": """Per-format options shared between file and text conversion paths. Every wired FormatOption gets the same `PdfPipelineOptions` instance so picture-description / classification / chart settings apply uniformly across PDF, IMAGE, HTML, MD, DOCX, PPTX. HTML and Markdown additionally receive backend options gated on `fetch_remote_images`. + + Args: + source_uri: Origin URI used by the HTML and Markdown backends to + resolve relative `` references (e.g. when + ingesting a downloaded HTML page). """ from docling.backend.docling_parse_backend import DoclingParseDocumentBackend from docling.datamodel.backend_options import ( @@ -177,10 +184,12 @@ class DoclingLocalConverter(DocumentConverter): PowerpointFormatOption, WordFormatOption, ) + from pydantic import AnyUrl opts = self.config.processing.conversion_options pipeline_options = self._build_pipeline_options() fetch = opts.fetch_remote_images + source_url = AnyUrl(source_uri) if source_uri else None return { InputFormat.PDF: PdfFormatOption( @@ -193,6 +202,7 @@ class DoclingLocalConverter(DocumentConverter): backend_options=HTMLBackendOptions( fetch_images=fetch, enable_remote_fetch=fetch, + source_uri=source_url, ), ), InputFormat.MD: MarkdownFormatOption( @@ -200,27 +210,36 @@ class DoclingLocalConverter(DocumentConverter): backend_options=MarkdownBackendOptions( fetch_images=fetch, enable_remote_fetch=fetch, + source_uri=source_url, ), ), InputFormat.DOCX: WordFormatOption(pipeline_options=pipeline_options), InputFormat.PPTX: PowerpointFormatOption(pipeline_options=pipeline_options), } - def _sync_convert_docling_file(self, path: Path) -> "DoclingDocument": + def _sync_convert_docling_file( + self, path: Path, source_uri: str | None = None + ) -> "DoclingDocument": """Synchronous conversion of docling-supported files.""" from docling.document_converter import ( DocumentConverter as DoclingDocConverter, ) - converter = DoclingDocConverter(format_options=self._build_format_options()) + converter = DoclingDocConverter( + format_options=self._build_format_options(source_uri=source_uri) + ) result = converter.convert(path) return result.document - async def convert_file(self, path: Path) -> "DoclingDocument": + async def convert_file( + self, path: Path, source_uri: str | None = None + ) -> "DoclingDocument": """Convert a file to DoclingDocument using local docling. Args: path: Path to the file to convert. + source_uri: Optional origin URI used by docling's HTML/Markdown + backends to resolve relative image references. Returns: DoclingDocument representation of the file. @@ -232,21 +251,33 @@ class DoclingLocalConverter(DocumentConverter): file_extension = path.suffix.lower() if file_extension in self.docling_extensions: - return await asyncio.to_thread(self._sync_convert_docling_file, path) + return await asyncio.to_thread( + self._sync_convert_docling_file, path, source_uri + ) elif file_extension in TextFileHandler.text_extensions: content = await asyncio.to_thread(path.read_text, encoding="utf-8") prepared_content = TextFileHandler.prepare_text_content( content, file_extension ) - return await self.convert_text(prepared_content, name=f"{path.stem}.md") + return await self.convert_text( + prepared_content, + name=f"{path.stem}.md", + source_uri=source_uri, + ) else: content = await asyncio.to_thread(path.read_text, encoding="utf-8") - return await self.convert_text(content, name=f"{path.stem}.md") + return await self.convert_text( + content, name=f"{path.stem}.md", source_uri=source_uri + ) except Exception: raise ValueError(f"Failed to parse file: {path}") async def convert_text( - self, text: str, name: str = "content.md", format: str = "md" + self, + text: str, + name: str = "content.md", + format: str = "md", + source_uri: str | None = None, ) -> "DoclingDocument": """Convert text content to DoclingDocument using local docling. @@ -255,6 +286,8 @@ class DoclingLocalConverter(DocumentConverter): name: The name to use for the document (defaults to "content.md"). format: The format of the text content ("md", "html", or "plain"). Defaults to "md". Use "plain" for plain text without parsing. + source_uri: Optional origin URI used by docling's HTML/Markdown + backends to resolve relative image references. Returns: DoclingDocument representation of the text. @@ -275,12 +308,14 @@ class DoclingLocalConverter(DocumentConverter): try: return await asyncio.to_thread( - self._sync_convert_docling_text, text, doc_name + self._sync_convert_docling_text, text, doc_name, source_uri ) except Exception as e: raise ValueError(f"Failed to convert text to DoclingDocument: {e}") - def _sync_convert_docling_text(self, text: str, doc_name: str) -> "DoclingDocument": + def _sync_convert_docling_text( + self, text: str, doc_name: str, source_uri: str | None = None + ) -> "DoclingDocument": """Synchronous text-to-DoclingDocument using the shared format options.""" from io import BytesIO @@ -292,7 +327,9 @@ class DoclingLocalConverter(DocumentConverter): bytes_io = BytesIO(text.encode("utf-8")) doc_stream = DocumentStream(name=doc_name, stream=bytes_io) - converter = DoclingDocConverter(format_options=self._build_format_options()) + converter = DoclingDocConverter( + format_options=self._build_format_options(source_uri=source_uri) + ) try: result = converter.convert(doc_stream) return result.document diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py index ea3fe173..68b6876a 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py @@ -226,11 +226,16 @@ class DoclingServeConverter(DocumentConverter): ) return self._parse_zip_to_docling(zip_bytes, name) - async def convert_file(self, path: Path) -> "DoclingDocument": + async def convert_file( + self, path: Path, source_uri: str | None = None + ) -> "DoclingDocument": """Convert a file to DoclingDocument using docling-serve. Args: path: Path to the file to convert. + source_uri: Ignored. docling-serve has no API path for + ``HTMLBackendOptions.source_uri``; the kwarg is accepted for + interface compatibility with docling-local. Returns: DoclingDocument representation of the file. @@ -261,7 +266,11 @@ class DoclingServeConverter(DocumentConverter): SUPPORTED_FORMATS = ("md", "html", "plain") async def convert_text( - self, text: str, name: str = "content.md", format: str = "md" + self, + text: str, + name: str = "content.md", + format: str = "md", + source_uri: str | None = None, ) -> "DoclingDocument": """Convert text content to DoclingDocument via docling-serve. @@ -272,6 +281,9 @@ class DoclingServeConverter(DocumentConverter): name: The name to use for the document (defaults to "content.md"). format: The format of the text content ("md", "html", or "plain"). Defaults to "md". Use "plain" for plain text without parsing. + source_uri: Ignored. docling-serve has no API path for + ``HTMLBackendOptions.source_uri``; the kwarg is accepted for + interface compatibility with docling-local. Returns: DoclingDocument representation of the text. diff --git a/tests/test_converters.py b/tests/test_converters.py index d5e17645..15c396a4 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -473,6 +473,69 @@ class TestDoclingLocalConverter: assert md_bo.fetch_images is False assert md_bo.enable_remote_fetch is False + def test_build_format_options_threads_source_uri(self, config): + """`_build_format_options(source_uri=...)` plumbs the URI into the + HTML and Markdown backend options so docling can resolve relative + `` paths during URL ingest.""" + from docling.datamodel.backend_options import ( + HTMLBackendOptions, + MarkdownBackendOptions, + ) + from docling.datamodel.base_models import InputFormat + + converter = DoclingLocalConverter(config) + opts = converter._build_format_options(source_uri="https://example.com/article") + + html_bo = opts[InputFormat.HTML].backend_options + md_bo = opts[InputFormat.MD].backend_options + assert isinstance(html_bo, HTMLBackendOptions) + assert str(html_bo.source_uri) == "https://example.com/article" + assert isinstance(md_bo, MarkdownBackendOptions) + assert str(md_bo.source_uri) == "https://example.com/article" + + # No source_uri ⇒ both stay None + opts_no_uri = converter._build_format_options() + html_bo = opts_no_uri[InputFormat.HTML].backend_options + md_bo = opts_no_uri[InputFormat.MD].backend_options + assert isinstance(html_bo, HTMLBackendOptions) + assert html_bo.source_uri is None + assert isinstance(md_bo, MarkdownBackendOptions) + assert md_bo.source_uri is None + + @pytest.mark.asyncio + async def test_convert_text_html_source_uri_resolves_relative_img( + self, config, monkeypatch + ): + """`convert_text(..., source_uri=...)` lets docling resolve a relative + `` against the source URL. We patch the docling HTML + backend's `_load_image_data` to capture the resolved absolute URL + instead of doing a real network fetch.""" + captured: list[str] = [] + + def fake_load_image_data(self, src_loc: str): + captured.append(src_loc) + return None # docling treats as a fetch failure → placeholder + + from docling.backend import html_backend as html_backend_module + + monkeypatch.setattr( + html_backend_module.HTMLDocumentBackend, + "_load_image_data", + fake_load_image_data, + ) + + html = '' + converter = DoclingLocalConverter(config) + + await converter.convert_text( + html, format="html", source_uri="https://example.com/article" + ) + + assert captured, "_load_image_data should have been invoked" + assert captured[0] == "https://example.com/static/cat.jpg", ( + f"Expected absolute URL resolved via source_uri, got {captured[0]!r}" + ) + @pytest.mark.asyncio async def test_convert_pdf_with_picture_images(self, config): """Picture bytes are produced by the local converter for PDFs that diff --git a/tests/test_processing.py b/tests/test_processing.py index b02624fe..f6000a74 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -143,11 +143,15 @@ async def test_convert_emits_warning_via_chokepoint( def supported_extensions(self) -> list[str]: return [".pdf"] - async def convert_file(self, path: Path): + async def convert_file(self, path: Path, source_uri: str | None = None): return _doc_with_pictures(with_descriptions=False) async def convert_text( - self, text: str, name: str = "content.md", format: str = "md" + self, + text: str, + name: str = "content.md", + format: str = "md", + source_uri: str | None = None, ): return _doc_without_pictures() @@ -180,11 +184,15 @@ async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings): def supported_extensions(self) -> list[str]: return [".html"] - async def convert_file(self, path: Path): + async def convert_file(self, path: Path, source_uri: str | None = None): return _doc_without_pictures() async def convert_text( - self, text: str, name: str = "content.md", format: str = "md" + self, + text: str, + name: str = "content.md", + format: str = "md", + source_uri: str | None = None, ): return _doc_with_pictures(with_descriptions=False)