diff --git a/CHANGELOG.md b/CHANGELOG.md
index d271b75c..385ef08a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,12 @@
### Added
+- **Format Parameter for Text Conversion**: New `format` parameter for `convert()` and `create_document()` to specify content type
+ - Supports `"md"` (default) for markdown and `"html"` for HTML content
+ - HTML format preserves document structure (headings, lists, sections) in DoclingDocument
+ - Enables proper parsing of HTML content that was previously treated as plain text
+ - Document content is stored as markdown export for consistent display (original preserved in `docling_document_json`)
+ - Wix evaluation dataset now uses `html_content` with `format="html"` for better document structure
- **DoclingDocument Storage**: Full DoclingDocument JSON is now stored with each document, enabling rich context and visual grounding
- Documents store the complete DoclingDocument structure (JSON) and schema version
- Chunks store metadata with JSON pointer references (`doc_item_refs`), semantic labels, section headings, and page numbers
diff --git a/docs/custom-pipelines.md b/docs/custom-pipelines.md
index bfcd3365..afe7dc4d 100644
--- a/docs/custom-pipelines.md
+++ b/docs/custom-pipelines.md
@@ -53,13 +53,25 @@ docling_doc = await client.convert(Path("/absolute/path/to/file.docx"))
# From URL (downloads and converts)
docling_doc = await client.convert("https://example.com/paper.pdf")
-# From plain text
-docling_doc = await client.convert("Your text content here")
+# From plain text (parsed as markdown by default)
+docling_doc = await client.convert("# Title\n\nYour text content here")
+
+# From HTML text (use format parameter to preserve structure)
+html_content = "
Title
Paragraph
"
+docling_doc = await client.convert(html_content, format="html")
# From file:// URI
docling_doc = await client.convert("file:///path/to/document.md")
```
+The `format` parameter controls how text content is parsed:
+
+- `"md"` (default) - Parse as Markdown
+- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
+
+!!! note
+ The `format` parameter only applies to text content. Files and URLs determine their format from the file extension or content-type header.
+
Supported formats depend on your converter configuration (docling-local or docling-serve). Common formats include PDF, DOCX, HTML, Markdown, and images.
## Chunk
diff --git a/docs/python.md b/docs/python.md
index 645813bc..ff75d953 100644
--- a/docs/python.md
+++ b/docs/python.md
@@ -36,6 +36,24 @@ doc = await client.create_document(
)
```
+From HTML content (preserves document structure):
+```python
+html_content = "Title
Paragraph
"
+doc = await client.create_document(
+ content=html_content,
+ uri="doc://html-example",
+ format="html" # parse as HTML instead of markdown
+)
+```
+
+The `format` parameter controls how text content is parsed:
+
+- `"md"` (default) - Parse as Markdown
+- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
+
+!!! note
+ The document's `content` field stores the markdown export of the parsed document for consistent display. The original input is preserved in the `docling_document_json` field.
+
From file:
```python
doc = await client.create_document_from_source(
diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py
index da7fbbc1..9cc391cf 100644
--- a/evaluations/evaluations/benchmark.py
+++ b/evaluations/evaluations/benchmark.py
@@ -98,6 +98,7 @@ async def populate_db(
uri=payload.uri,
title=payload.title,
metadata=payload.metadata,
+ format=payload.format,
)
progress.advance(task)
diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py
index f791cbef..acd71eb1 100644
--- a/evaluations/evaluations/config.py
+++ b/evaluations/evaluations/config.py
@@ -14,6 +14,7 @@ class DocumentPayload:
content: str
title: str | None = None
metadata: dict[str, Any] | None = None
+ format: str = "md"
@dataclass
diff --git a/evaluations/evaluations/datasets/wix.py b/evaluations/evaluations/datasets/wix.py
index 6b4358c9..c6f01c3f 100644
--- a/evaluations/evaluations/datasets/wix.py
+++ b/evaluations/evaluations/datasets/wix.py
@@ -27,9 +27,10 @@ def map_wix_document(doc: Mapping[str, Any]) -> DocumentPayload:
return DocumentPayload(
uri=uri,
- content=doc["contents"],
+ content=doc["html_content"],
title=doc.get("title"),
metadata=metadata or None,
+ format="html",
)
diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py
index a890ca2d..264d3dbc 100644
--- a/haiku_rag_slim/haiku/rag/client.py
+++ b/haiku_rag_slim/haiku/rag/client.py
@@ -100,9 +100,13 @@ class HaikuRAG:
async def convert(self, source: Path) -> "DoclingDocument": ...
@overload
- async def convert(self, source: str) -> "DoclingDocument": ...
+ async def convert(
+ self, source: str, *, format: str = "md"
+ ) -> "DoclingDocument": ...
- async def convert(self, source: Path | str) -> "DoclingDocument":
+ async def convert(
+ self, source: Path | str, *, format: str = "md"
+ ) -> "DoclingDocument":
"""Convert a file, URL, or text to DoclingDocument.
Args:
@@ -110,6 +114,9 @@ class HaikuRAG:
- Path: Local file path to convert
- str (URL): HTTP/HTTPS URL to download and convert
- str (text): Raw text content to convert
+ format: The format of text content ("md" or "html"). Defaults to "md".
+ Only used when source is raw text (not a file path or URL).
+ Files and URLs determine format from extension/content-type.
Returns:
DoclingDocument from the converted source.
@@ -170,7 +177,7 @@ class HaikuRAG:
else:
# Treat as text content
- return await converter.convert_text(source)
+ return await converter.convert_text(source, format=format)
async def chunk(self, docling_document: "DoclingDocument") -> list[Chunk]:
"""Chunk a DoclingDocument into Chunks.
@@ -329,6 +336,7 @@ class HaikuRAG:
uri: str | None = None,
title: str | None = None,
metadata: dict | None = None,
+ format: str = "md",
) -> Document:
"""Create a new document from text content.
@@ -339,6 +347,8 @@ class HaikuRAG:
uri: Optional URI identifier for the document.
title: Optional title for the document.
metadata: Optional metadata dictionary.
+ format: The format of the content ("md" or "html"). Defaults to "md".
+ This determines which parser is used to interpret the content structure.
Returns:
The created Document instance.
@@ -346,13 +356,17 @@ class HaikuRAG:
from haiku.rag.embeddings import embed_chunks
# Convert → Chunk → Embed using primitives
- docling_document = await self.convert(content)
+ docling_document = await self.convert(content, format=format)
chunks = await self.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, self._config)
+ # Store markdown export as content for better display/readability
+ # The original content is preserved in docling_document_json
+ stored_content = docling_document.export_to_markdown()
+
# Create document model
document = Document(
- content=content,
+ content=stored_content,
uri=uri,
title=title,
metadata=metadata or {},
diff --git a/haiku_rag_slim/haiku/rag/converters/base.py b/haiku_rag_slim/haiku/rag/converters/base.py
index 713cbd94..1a6b883f 100644
--- a/haiku_rag_slim/haiku/rag/converters/base.py
+++ b/haiku_rag_slim/haiku/rag/converters/base.py
@@ -40,20 +40,24 @@ class DocumentConverter(ABC):
"""
pass
+ SUPPORTED_FORMATS = ("md", "html")
+
@abstractmethod
async def convert_text(
- self, text: str, name: str = "content.md"
+ self, text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text content to DoclingDocument format.
Args:
text: The text content to convert.
name: The name to use for the document (defaults to "content.md").
+ format: The format of the text content ("md" or "html"). Defaults to "md".
+ This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
- ValueError: If the text cannot be converted.
+ ValueError: If the text cannot be converted or format is unsupported.
"""
pass
diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py
index f8b6a4ec..64d817b5 100644
--- a/haiku_rag_slim/haiku/rag/converters/docling_local.py
+++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py
@@ -137,18 +137,20 @@ class DoclingLocalConverter(DocumentConverter):
raise ValueError(f"Failed to parse file: {path}")
async def convert_text(
- self, text: str, name: str = "content.md"
+ self, text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text content to DoclingDocument using local docling.
Args:
text: The text content to convert.
name: The name to use for the document (defaults to "content.md").
+ format: The format of the text content ("md" or "html"). Defaults to "md".
+ This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
- ValueError: If the text cannot be converted.
+ ValueError: If the text cannot be converted or format is unsupported.
"""
- return await TextFileHandler.text_to_docling_document(text, name)
+ return await TextFileHandler.text_to_docling_document(text, name, format)
diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py
index 09232af2..5f61d7bb 100644
--- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py
+++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py
@@ -176,23 +176,37 @@ class DoclingServeConverter(DocumentConverter):
files = {"files": (path.name, file_content, "application/octet-stream")}
return await self._make_request(files, path.name)
+ SUPPORTED_FORMATS = ("md", "html")
+
async def convert_text(
- self, text: str, name: str = "content.md"
+ self, text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text content to DoclingDocument via docling-serve.
- Sends the text as a markdown file to docling-serve for conversion.
+ Sends the text to docling-serve for conversion using the specified format.
Args:
text: The text content to convert.
name: The name to use for the document (defaults to "content.md").
+ format: The format of the text content ("md" or "html"). Defaults to "md".
+ This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
- ValueError: If the text cannot be converted.
+ ValueError: If the text cannot be converted or format is unsupported.
"""
+ if format not in self.SUPPORTED_FORMATS:
+ raise ValueError(
+ f"Unsupported format: {format}. "
+ f"Supported formats: {', '.join(self.SUPPORTED_FORMATS)}"
+ )
+
+ # Derive document name from format to tell docling which parser to use
+ doc_name = f"content.{format}" if name == "content.md" else name
+ mime_type = "text/html" if format == "html" else "text/markdown"
+
text_bytes = text.encode("utf-8")
- files = {"files": (name, text_bytes, "text/markdown")}
- return await self._make_request(files, name)
+ files = {"files": (doc_name, text_bytes, mime_type)}
+ return await self._make_request(files, doc_name)
diff --git a/haiku_rag_slim/haiku/rag/converters/text_utils.py b/haiku_rag_slim/haiku/rag/converters/text_utils.py
index a149297c..36263155 100644
--- a/haiku_rag_slim/haiku/rag/converters/text_utils.py
+++ b/haiku_rag_slim/haiku/rag/converters/text_utils.py
@@ -89,39 +89,52 @@ class TextFileHandler:
return f"```{language}\n{content}\n```"
return content
+ SUPPORTED_FORMATS = ("md", "html")
+
@staticmethod
def _sync_text_to_docling_document(
- text: str, name: str = "content.md"
+ text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Synchronous implementation of text to DoclingDocument conversion."""
from docling.document_converter import DocumentConverter as DoclingDocConverter
from docling_core.types.io import DocumentStream
+ if format not in TextFileHandler.SUPPORTED_FORMATS:
+ raise ValueError(
+ f"Unsupported format: {format}. "
+ f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}"
+ )
+
+ # Derive document name from format to tell docling which parser to use
+ doc_name = f"content.{format}" if name == "content.md" else name
+
bytes_io = BytesIO(text.encode("utf-8"))
- doc_stream = DocumentStream(name=name, stream=bytes_io)
+ doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
converter = DoclingDocConverter()
result = converter.convert(doc_stream)
return result.document
@staticmethod
async def text_to_docling_document(
- text: str, name: str = "content.md"
+ text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
- """Convert text to DoclingDocument using docling's markdown parser.
+ """Convert text to DoclingDocument using docling's parser.
Args:
text: The text content to convert.
name: The name to use for the document.
+ format: The format of the text content ("md" or "html"). Defaults to "md".
+ This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
- ValueError: If the conversion fails.
+ ValueError: If the conversion fails or format is unsupported.
"""
try:
return await asyncio.to_thread(
- TextFileHandler._sync_text_to_docling_document, text, name
+ TextFileHandler._sync_text_to_docling_document, text, name, format
)
except Exception as e:
raise ValueError(f"Failed to convert text to DoclingDocument: {e}")
diff --git a/tests/test_client.py b/tests/test_client.py
index 2a3ca4c3..eabff47a 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -28,7 +28,8 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path):
)
assert created_doc.id is not None
- assert created_doc.content == document_text
+ # Content is stored as markdown export, check key text is preserved
+ assert "Jakarta" in created_doc.content
assert created_doc.uri == test_uri
assert created_doc.metadata == test_metadata
@@ -36,14 +37,14 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path):
retrieved_doc = await client.get_document_by_id(created_doc.id)
assert retrieved_doc is not None
assert retrieved_doc.id == created_doc.id
- assert retrieved_doc.content == document_text
+ assert "Jakarta" in retrieved_doc.content
assert retrieved_doc.uri == test_uri
# Test get_document_by_uri
retrieved_by_uri = await client.get_document_by_uri(test_uri)
assert retrieved_by_uri is not None
assert retrieved_by_uri.id == created_doc.id
- assert retrieved_by_uri.content == document_text
+ assert "Jakarta" in retrieved_by_uri.content
# Test get_document_by_uri with non-existent URI
non_existent = await client.get_document_by_uri("file:///non/existent.txt")
@@ -1380,3 +1381,52 @@ async def test_update_document_embeds_chunks_without_embeddings(temp_db_path):
results = await client.search("Updated chunk", search_type="vector")
assert len(results) > 0
assert results[0].content == "Updated chunk without embedding"
+
+
+@pytest.mark.asyncio
+async def test_client_create_document_with_html_format(temp_db_path):
+ """Test create_document with HTML format preserves document structure."""
+ async with HaikuRAG(temp_db_path, create=True) as client:
+ html_content = """
+ Main Title
+ Introduction paragraph.
+ Section Header
+
+ """
+
+ doc = await client.create_document(
+ content=html_content,
+ uri="test://html-doc",
+ format="html",
+ )
+
+ assert doc.id is not None
+ assert doc.docling_document_json is not None
+
+ # Verify the DoclingDocument has proper structure
+ docling_doc = doc.get_docling_document()
+ assert docling_doc is not None
+
+ items = list(docling_doc.iterate_items())
+ labels = [str(getattr(item, "label", "")) for item, _ in items]
+
+ # HTML format should preserve headers and list items
+ assert "title" in labels or "section_header" in labels
+ assert "list_item" in labels
+
+
+@pytest.mark.asyncio
+async def test_client_convert_with_html_format(temp_db_path):
+ """Test convert with HTML format."""
+ async with HaikuRAG(temp_db_path, create=True) as client:
+ html_content = "Title
Text
"
+
+ docling_doc = await client.convert(html_content, format="html")
+
+ items = list(docling_doc.iterate_items())
+ labels = [str(getattr(item, "label", "")) for item, _ in items]
+
+ assert "title" in labels
diff --git a/tests/test_converters.py b/tests/test_converters.py
index 9bf9e873..c8fd2f85 100644
--- a/tests/test_converters.py
+++ b/tests/test_converters.py
@@ -116,6 +116,73 @@ class TestConverterFactory:
get_converter(config)
+class TestTextToDoclingWithFormat:
+ """Tests for format parameter in text to DoclingDocument conversion."""
+
+ @pytest.mark.asyncio
+ async def test_html_format_preserves_structure(self):
+ """Test that HTML content parsed with html format preserves document structure."""
+ html_content = """
+ Main Title
+ Introduction paragraph.
+ Section Header
+
+ """
+ config = AppConfig()
+ converter = DoclingLocalConverter(config)
+
+ # With html format, should get proper structure
+ doc = await converter.convert_text(
+ html_content, name="content.html", format="html"
+ )
+
+ items = list(doc.iterate_items())
+ labels = [str(getattr(item, "label", "")) for item, _ in items]
+
+ assert "title" in labels or "section_header" in labels
+ assert "list_item" in labels
+ assert len(items) > 3
+
+ @pytest.mark.asyncio
+ async def test_md_format_is_default(self):
+ """Test that md format is used by default."""
+ config = AppConfig()
+ converter = DoclingLocalConverter(config)
+
+ # Plain text should work with default format
+ doc = await converter.convert_text("# Heading\n\nParagraph text.")
+ items = list(doc.iterate_items())
+
+ assert len(items) >= 2
+
+ @pytest.mark.asyncio
+ async def test_html_as_md_loses_structure(self):
+ """Test that HTML parsed as markdown loses semantic structure."""
+ html_content = "Title
Text
"
+ config = AppConfig()
+ converter = DoclingLocalConverter(config)
+
+ # With md format (default), HTML tags are treated as text
+ doc = await converter.convert_text(html_content, format="md")
+ items = list(doc.iterate_items())
+
+ # Should still parse but with different structure
+ # (markdown parser will interpret some HTML)
+ assert len(items) >= 1
+
+ @pytest.mark.asyncio
+ async def test_invalid_format_raises_error(self):
+ """Test that invalid format raises ValueError."""
+ config = AppConfig()
+ converter = DoclingLocalConverter(config)
+
+ with pytest.raises(ValueError, match="Unsupported format"):
+ await converter.convert_text("content", format="invalid")
+
+
class TestDoclingLocalConverter:
"""Tests for DoclingLocalConverter."""