diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e6f7760..f88ae7d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@
- **Conversion options now apply to non-PDF formats.** `DoclingLocalConverter` previously wired its `PdfPipelineOptions` only to `InputFormat.PDF`, so user settings (OCR knobs, `picture_description.enabled`, `images_scale`, etc.) silently no-op'd for HTML, Markdown, DOCX, PPTX, and IMAGE inputs. The converter now shares a single `PdfPipelineOptions` instance across PDF, IMAGE, HTML, MD, DOCX, and PPTX `FormatOption`s. SimplePipeline-backed formats ignore the PDF-specific fields; `ConvertPipelineOptions`-level enrichments (picture description / classification / chart extraction) now run uniformly. HTML and Markdown additionally receive `HTMLBackendOptions` / `MarkdownBackendOptions` gated on `fetch_remote_images`.
- **HTML text ingest path picks up converter options.** `convert_text(format="html"/"md")` previously used a bare `DoclingDocConverter()` with zero format options — the wix corpus ingest path. It now uses the same shared `_build_format_options()` helper as the file path.
+- **Relative `
` paths resolve during URL ingest.** `HaikuRAG.convert()` and the converter `convert_file` / `convert_text` methods now thread a `source_uri` through to `HTMLBackendOptions.source_uri` / `MarkdownBackendOptions.source_uri`. URL ingest uses the originating URL; file ingest uses `file://`; raw text accepts an optional override. docling-serve accepts the kwarg as a no-op (its API has no equivalent option).
### Documentation
diff --git a/haiku_rag_slim/haiku/rag/client/__init__.py b/haiku_rag_slim/haiku/rag/client/__init__.py
index 1413757f..3a26a533 100644
--- a/haiku_rag_slim/haiku/rag/client/__init__.py
+++ b/haiku_rag_slim/haiku/rag/client/__init__.py
@@ -139,19 +139,25 @@ class HaikuRAG:
# =========================================================================
@overload
- async def convert(self, source: Path) -> "DoclingDocument": ...
+ async def convert(
+ self, source: Path, *, source_uri: str | None = None
+ ) -> "DoclingDocument": ...
@overload
async def convert(
- self, source: str, *, format: str = "md"
+ self, source: str, *, format: str = "md", source_uri: str | None = None
) -> "DoclingDocument": ...
async def convert(
- self, source: Path | str, *, format: str = "md"
+ self,
+ source: Path | str,
+ *,
+ format: str = "md",
+ source_uri: str | None = None,
) -> "DoclingDocument":
from haiku.rag.client.processing import convert
- return await convert(self._config, source, format=format)
+ return await convert(self._config, source, format=format, source_uri=source_uri)
async def chunk(
self,
diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py
index 100bc51a..605c8950 100644
--- a/haiku_rag_slim/haiku/rag/client/documents.py
+++ b/haiku_rag_slim/haiku/rag/client/documents.py
@@ -411,7 +411,7 @@ async def _create_or_update_document_from_url(
temp_path = Path(temp_file.name)
try:
- docling_document = await client.convert(temp_path)
+ docling_document = await client.convert(temp_path, source_uri=url)
chunks = await client.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, client._config)
finally:
@@ -547,7 +547,7 @@ async def _create_or_update_document_from_s3(
return existing_doc
try:
- docling_document = await client.convert(temp_path)
+ docling_document = await client.convert(temp_path, source_uri=url)
chunks = await client.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, client._config)
finally:
diff --git a/haiku_rag_slim/haiku/rag/client/processing.py b/haiku_rag_slim/haiku/rag/client/processing.py
index 8ccec652..b1051323 100644
--- a/haiku_rag_slim/haiku/rag/client/processing.py
+++ b/haiku_rag_slim/haiku/rag/client/processing.py
@@ -52,7 +52,11 @@ def _warn_if_descriptions_missing(
async def convert(
- config: AppConfig, source: Path | str, *, format: str = "md"
+ config: AppConfig,
+ source: Path | str,
+ *,
+ format: str = "md",
+ source_uri: str | None = None,
) -> "DoclingDocument":
"""Convert a file, URL, or text to DoclingDocument.
@@ -66,6 +70,10 @@ async def convert(
Defaults to "md". Use "plain" for plain text without parsing.
Only used when source is raw text (not a file path or URL).
Files and URLs determine format from extension/content-type.
+ source_uri: Origin URI used by docling's HTML/Markdown backends to
+ resolve relative `
` references. When omitted,
+ defaults to the URL (URL ingest) or `file://` URI (file ingest);
+ raw text input has no origin so no default is derived.
Returns:
DoclingDocument from the converted source.
@@ -82,7 +90,8 @@ async def convert(
raise ValueError(f"File does not exist: {source}")
if source.suffix.lower() not in converter.supported_extensions:
raise ValueError(f"Unsupported file extension: {source.suffix}")
- doc = await converter.convert_file(source)
+ effective_uri = source_uri or source.absolute().as_uri()
+ doc = await converter.convert_file(source, source_uri=effective_uri)
_warn_if_descriptions_missing(config, doc, str(source))
return doc
@@ -113,7 +122,8 @@ async def convert(
temp_path = Path(temp_file.name)
try:
- doc = await converter.convert_file(temp_path)
+ effective_uri = source_uri or source
+ doc = await converter.convert_file(temp_path, source_uri=effective_uri)
_warn_if_descriptions_missing(config, doc, source)
return doc
finally:
@@ -126,14 +136,15 @@ async def convert(
raise ValueError(f"File does not exist: {file_path}")
if file_path.suffix.lower() not in converter.supported_extensions:
raise ValueError(f"Unsupported file extension: {file_path.suffix}")
- doc = await converter.convert_file(file_path)
+ effective_uri = source_uri or file_path.absolute().as_uri()
+ doc = await converter.convert_file(file_path, source_uri=effective_uri)
_warn_if_descriptions_missing(config, doc, str(file_path))
return doc
else:
# Raw text content — HTML and markdown can still embed pictures
# via
/ so the same description check applies.
- doc = await converter.convert_text(source, format=format)
+ doc = await converter.convert_text(source, format=format, source_uri=source_uri)
_warn_if_descriptions_missing(config, doc, "")
return doc
diff --git a/haiku_rag_slim/haiku/rag/converters/base.py b/haiku_rag_slim/haiku/rag/converters/base.py
index e1ccbc7a..d6aea337 100644
--- a/haiku_rag_slim/haiku/rag/converters/base.py
+++ b/haiku_rag_slim/haiku/rag/converters/base.py
@@ -26,11 +26,18 @@ class DocumentConverter(ABC):
pass
@abstractmethod
- async def convert_file(self, path: Path) -> "DoclingDocument":
+ async def convert_file(
+ self, path: Path, source_uri: str | None = None
+ ) -> "DoclingDocument":
"""Convert a file to DoclingDocument format.
Args:
path: Path to the file to convert.
+ source_uri: Optional origin URI (e.g. the URL the file was
+ downloaded from) used by docling's HTML/Markdown backends to
+ resolve relative `
` references. Ignored by
+ converters that have no equivalent backend option (notably
+ docling-serve).
Returns:
DoclingDocument representation of the file.
@@ -44,7 +51,11 @@ class DocumentConverter(ABC):
@abstractmethod
async def convert_text(
- self, text: str, name: str = "content.md", format: str = "md"
+ self,
+ text: str,
+ name: str = "content.md",
+ format: str = "md",
+ source_uri: str | None = None,
) -> "DoclingDocument":
"""Convert text content to DoclingDocument format.
@@ -53,6 +64,8 @@ class DocumentConverter(ABC):
name: The name to use for the document (defaults to "content.md").
format: The format of the text content ("md", "html", or "plain").
Defaults to "md". Use "plain" for plain text without parsing.
+ source_uri: Optional origin URI used by docling's HTML/Markdown
+ backends to resolve relative image references.
Returns:
DoclingDocument representation of the text.
diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py
index a63564b8..3028b59c 100644
--- a/haiku_rag_slim/haiku/rag/converters/docling_local.py
+++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py
@@ -155,13 +155,20 @@ class DoclingLocalConverter(DocumentConverter):
return pipeline_options
- def _build_format_options(self) -> "dict[InputFormat, FormatOption]":
+ def _build_format_options(
+ self, source_uri: str | None = None
+ ) -> "dict[InputFormat, FormatOption]":
"""Per-format options shared between file and text conversion paths.
Every wired FormatOption gets the same `PdfPipelineOptions` instance so
picture-description / classification / chart settings apply uniformly
across PDF, IMAGE, HTML, MD, DOCX, PPTX. HTML and Markdown additionally
receive backend options gated on `fetch_remote_images`.
+
+ Args:
+ source_uri: Origin URI used by the HTML and Markdown backends to
+ resolve relative `
` references (e.g. when
+ ingesting a downloaded HTML page).
"""
from docling.backend.docling_parse_backend import DoclingParseDocumentBackend
from docling.datamodel.backend_options import (
@@ -177,10 +184,12 @@ class DoclingLocalConverter(DocumentConverter):
PowerpointFormatOption,
WordFormatOption,
)
+ from pydantic import AnyUrl
opts = self.config.processing.conversion_options
pipeline_options = self._build_pipeline_options()
fetch = opts.fetch_remote_images
+ source_url = AnyUrl(source_uri) if source_uri else None
return {
InputFormat.PDF: PdfFormatOption(
@@ -193,6 +202,7 @@ class DoclingLocalConverter(DocumentConverter):
backend_options=HTMLBackendOptions(
fetch_images=fetch,
enable_remote_fetch=fetch,
+ source_uri=source_url,
),
),
InputFormat.MD: MarkdownFormatOption(
@@ -200,27 +210,36 @@ class DoclingLocalConverter(DocumentConverter):
backend_options=MarkdownBackendOptions(
fetch_images=fetch,
enable_remote_fetch=fetch,
+ source_uri=source_url,
),
),
InputFormat.DOCX: WordFormatOption(pipeline_options=pipeline_options),
InputFormat.PPTX: PowerpointFormatOption(pipeline_options=pipeline_options),
}
- def _sync_convert_docling_file(self, path: Path) -> "DoclingDocument":
+ def _sync_convert_docling_file(
+ self, path: Path, source_uri: str | None = None
+ ) -> "DoclingDocument":
"""Synchronous conversion of docling-supported files."""
from docling.document_converter import (
DocumentConverter as DoclingDocConverter,
)
- converter = DoclingDocConverter(format_options=self._build_format_options())
+ converter = DoclingDocConverter(
+ format_options=self._build_format_options(source_uri=source_uri)
+ )
result = converter.convert(path)
return result.document
- async def convert_file(self, path: Path) -> "DoclingDocument":
+ async def convert_file(
+ self, path: Path, source_uri: str | None = None
+ ) -> "DoclingDocument":
"""Convert a file to DoclingDocument using local docling.
Args:
path: Path to the file to convert.
+ source_uri: Optional origin URI used by docling's HTML/Markdown
+ backends to resolve relative image references.
Returns:
DoclingDocument representation of the file.
@@ -232,21 +251,33 @@ class DoclingLocalConverter(DocumentConverter):
file_extension = path.suffix.lower()
if file_extension in self.docling_extensions:
- return await asyncio.to_thread(self._sync_convert_docling_file, path)
+ return await asyncio.to_thread(
+ self._sync_convert_docling_file, path, source_uri
+ )
elif file_extension in TextFileHandler.text_extensions:
content = await asyncio.to_thread(path.read_text, encoding="utf-8")
prepared_content = TextFileHandler.prepare_text_content(
content, file_extension
)
- return await self.convert_text(prepared_content, name=f"{path.stem}.md")
+ return await self.convert_text(
+ prepared_content,
+ name=f"{path.stem}.md",
+ source_uri=source_uri,
+ )
else:
content = await asyncio.to_thread(path.read_text, encoding="utf-8")
- return await self.convert_text(content, name=f"{path.stem}.md")
+ return await self.convert_text(
+ content, name=f"{path.stem}.md", source_uri=source_uri
+ )
except Exception:
raise ValueError(f"Failed to parse file: {path}")
async def convert_text(
- self, text: str, name: str = "content.md", format: str = "md"
+ self,
+ text: str,
+ name: str = "content.md",
+ format: str = "md",
+ source_uri: str | None = None,
) -> "DoclingDocument":
"""Convert text content to DoclingDocument using local docling.
@@ -255,6 +286,8 @@ class DoclingLocalConverter(DocumentConverter):
name: The name to use for the document (defaults to "content.md").
format: The format of the text content ("md", "html", or "plain").
Defaults to "md". Use "plain" for plain text without parsing.
+ source_uri: Optional origin URI used by docling's HTML/Markdown
+ backends to resolve relative image references.
Returns:
DoclingDocument representation of the text.
@@ -275,12 +308,14 @@ class DoclingLocalConverter(DocumentConverter):
try:
return await asyncio.to_thread(
- self._sync_convert_docling_text, text, doc_name
+ self._sync_convert_docling_text, text, doc_name, source_uri
)
except Exception as e:
raise ValueError(f"Failed to convert text to DoclingDocument: {e}")
- def _sync_convert_docling_text(self, text: str, doc_name: str) -> "DoclingDocument":
+ def _sync_convert_docling_text(
+ self, text: str, doc_name: str, source_uri: str | None = None
+ ) -> "DoclingDocument":
"""Synchronous text-to-DoclingDocument using the shared format options."""
from io import BytesIO
@@ -292,7 +327,9 @@ class DoclingLocalConverter(DocumentConverter):
bytes_io = BytesIO(text.encode("utf-8"))
doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
- converter = DoclingDocConverter(format_options=self._build_format_options())
+ converter = DoclingDocConverter(
+ format_options=self._build_format_options(source_uri=source_uri)
+ )
try:
result = converter.convert(doc_stream)
return result.document
diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py
index ea3fe173..68b6876a 100644
--- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py
+++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py
@@ -226,11 +226,16 @@ class DoclingServeConverter(DocumentConverter):
)
return self._parse_zip_to_docling(zip_bytes, name)
- async def convert_file(self, path: Path) -> "DoclingDocument":
+ async def convert_file(
+ self, path: Path, source_uri: str | None = None
+ ) -> "DoclingDocument":
"""Convert a file to DoclingDocument using docling-serve.
Args:
path: Path to the file to convert.
+ source_uri: Ignored. docling-serve has no API path for
+ ``HTMLBackendOptions.source_uri``; the kwarg is accepted for
+ interface compatibility with docling-local.
Returns:
DoclingDocument representation of the file.
@@ -261,7 +266,11 @@ class DoclingServeConverter(DocumentConverter):
SUPPORTED_FORMATS = ("md", "html", "plain")
async def convert_text(
- self, text: str, name: str = "content.md", format: str = "md"
+ self,
+ text: str,
+ name: str = "content.md",
+ format: str = "md",
+ source_uri: str | None = None,
) -> "DoclingDocument":
"""Convert text content to DoclingDocument via docling-serve.
@@ -272,6 +281,9 @@ class DoclingServeConverter(DocumentConverter):
name: The name to use for the document (defaults to "content.md").
format: The format of the text content ("md", "html", or "plain").
Defaults to "md". Use "plain" for plain text without parsing.
+ source_uri: Ignored. docling-serve has no API path for
+ ``HTMLBackendOptions.source_uri``; the kwarg is accepted for
+ interface compatibility with docling-local.
Returns:
DoclingDocument representation of the text.
diff --git a/tests/test_converters.py b/tests/test_converters.py
index d5e17645..15c396a4 100644
--- a/tests/test_converters.py
+++ b/tests/test_converters.py
@@ -473,6 +473,69 @@ class TestDoclingLocalConverter:
assert md_bo.fetch_images is False
assert md_bo.enable_remote_fetch is False
+ def test_build_format_options_threads_source_uri(self, config):
+ """`_build_format_options(source_uri=...)` plumbs the URI into the
+ HTML and Markdown backend options so docling can resolve relative
+ `
` paths during URL ingest."""
+ from docling.datamodel.backend_options import (
+ HTMLBackendOptions,
+ MarkdownBackendOptions,
+ )
+ from docling.datamodel.base_models import InputFormat
+
+ converter = DoclingLocalConverter(config)
+ opts = converter._build_format_options(source_uri="https://example.com/article")
+
+ html_bo = opts[InputFormat.HTML].backend_options
+ md_bo = opts[InputFormat.MD].backend_options
+ assert isinstance(html_bo, HTMLBackendOptions)
+ assert str(html_bo.source_uri) == "https://example.com/article"
+ assert isinstance(md_bo, MarkdownBackendOptions)
+ assert str(md_bo.source_uri) == "https://example.com/article"
+
+ # No source_uri ⇒ both stay None
+ opts_no_uri = converter._build_format_options()
+ html_bo = opts_no_uri[InputFormat.HTML].backend_options
+ md_bo = opts_no_uri[InputFormat.MD].backend_options
+ assert isinstance(html_bo, HTMLBackendOptions)
+ assert html_bo.source_uri is None
+ assert isinstance(md_bo, MarkdownBackendOptions)
+ assert md_bo.source_uri is None
+
+ @pytest.mark.asyncio
+ async def test_convert_text_html_source_uri_resolves_relative_img(
+ self, config, monkeypatch
+ ):
+ """`convert_text(..., source_uri=...)` lets docling resolve a relative
+ `
` against the source URL. We patch the docling HTML
+ backend's `_load_image_data` to capture the resolved absolute URL
+ instead of doing a real network fetch."""
+ captured: list[str] = []
+
+ def fake_load_image_data(self, src_loc: str):
+ captured.append(src_loc)
+ return None # docling treats as a fetch failure → placeholder
+
+ from docling.backend import html_backend as html_backend_module
+
+ monkeypatch.setattr(
+ html_backend_module.HTMLDocumentBackend,
+ "_load_image_data",
+ fake_load_image_data,
+ )
+
+ html = '
'
+ converter = DoclingLocalConverter(config)
+
+ await converter.convert_text(
+ html, format="html", source_uri="https://example.com/article"
+ )
+
+ assert captured, "_load_image_data should have been invoked"
+ assert captured[0] == "https://example.com/static/cat.jpg", (
+ f"Expected absolute URL resolved via source_uri, got {captured[0]!r}"
+ )
+
@pytest.mark.asyncio
async def test_convert_pdf_with_picture_images(self, config):
"""Picture bytes are produced by the local converter for PDFs that
diff --git a/tests/test_processing.py b/tests/test_processing.py
index b02624fe..f6000a74 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -143,11 +143,15 @@ async def test_convert_emits_warning_via_chokepoint(
def supported_extensions(self) -> list[str]:
return [".pdf"]
- async def convert_file(self, path: Path):
+ async def convert_file(self, path: Path, source_uri: str | None = None):
return _doc_with_pictures(with_descriptions=False)
async def convert_text(
- self, text: str, name: str = "content.md", format: str = "md"
+ self,
+ text: str,
+ name: str = "content.md",
+ format: str = "md",
+ source_uri: str | None = None,
):
return _doc_without_pictures()
@@ -180,11 +184,15 @@ async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings):
def supported_extensions(self) -> list[str]:
return [".html"]
- async def convert_file(self, path: Path):
+ async def convert_file(self, path: Path, source_uri: str | None = None):
return _doc_without_pictures()
async def convert_text(
- self, text: str, name: str = "content.md", format: str = "md"
+ self,
+ text: str,
+ name: str = "content.md",
+ format: str = "md",
+ source_uri: str | None = None,
):
return _doc_with_pictures(with_descriptions=False)