danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add format parameter for text-to-DoclingDocument conversion

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-12-08 12:54:54 +02:00
parent 1b14a4643d
commit 06b4f6a224
No known key found for this signature in database
13 changed files with 230 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
CHANGELOG.md
View file

@ -10,6 +10,12 @@
### Added
- **Format Parameter for Text Conversion**: New `format` parameter for `convert()` and `create_document()` to specify content type
- Supports `"md"` (default) for markdown and `"html"` for HTML content
- HTML format preserves document structure (headings, lists, sections) in DoclingDocument
- Enables proper parsing of HTML content that was previously treated as plain text
- Document content is stored as markdown export for consistent display (original preserved in `docling_document_json`)
- Wix evaluation dataset now uses `html_content` with `format="html"` for better document structure
- **DoclingDocument Storage**: Full DoclingDocument JSON is now stored with each document, enabling rich context and visual grounding
- Documents store the complete DoclingDocument structure (JSON) and schema version
- Chunks store metadata with JSON pointer references (`doc_item_refs`), semantic labels, section headings, and page numbers

16
docs/custom-pipelines.md
View file

@ -53,13 +53,25 @@ docling_doc = await client.convert(Path("/absolute/path/to/file.docx"))
# From URL (downloads and converts)
docling_doc = await client.convert("https://example.com/paper.pdf")
# From plain text
docling_doc = await client.convert("Your text content here")
# From plain text (parsed as markdown by default)
docling_doc = await client.convert("# Title\n\nYour text content here")
# From HTML text (use format parameter to preserve structure)
html_content = "<h1>Title</h1><p>Paragraph</p><ul><li>Item</li></ul>"
docling_doc = await client.convert(html_content, format="html")
# From file:// URI
docling_doc = await client.convert("file:///path/to/document.md")
```
The `format` parameter controls how text content is parsed:
- `"md"` (default) - Parse as Markdown
- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
!!! note
The `format` parameter only applies to text content. Files and URLs determine their format from the file extension or content-type header.
Supported formats depend on your converter configuration (docling-local or docling-serve). Common formats include PDF, DOCX, HTML, Markdown, and images.
## Chunk

18
docs/python.md
View file

@ -36,6 +36,24 @@ doc = await client.create_document(
)
```
From HTML content (preserves document structure):
```python
html_content = "<h1>Title</h1><p>Paragraph</p><ul><li>Item 1</li></ul>"
doc = await client.create_document(
content=html_content,
uri="doc://html-example",
format="html" # parse as HTML instead of markdown
)
```
The `format` parameter controls how text content is parsed:
- `"md"` (default) - Parse as Markdown
- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
!!! note
The document's `content` field stores the markdown export of the parsed document for consistent display. The original input is preserved in the `docling_document_json` field.
From file:
```python
doc = await client.create_document_from_source(

1
evaluations/evaluations/benchmark.py
View file

@ -98,6 +98,7 @@ async def populate_db(
uri=payload.uri,
title=payload.title,
metadata=payload.metadata,
format=payload.format,
)
progress.advance(task)

1
evaluations/evaluations/config.py
View file

@ -14,6 +14,7 @@ class DocumentPayload:
content: str
title: str | None = None
metadata: dict[str, Any] | None = None
format: str = "md"
@dataclass

3
evaluations/evaluations/datasets/wix.py
View file

@ -27,9 +27,10 @@ def map_wix_document(doc: Mapping[str, Any]) -> DocumentPayload:
return DocumentPayload(
uri=uri,
content=doc["contents"],
content=doc["html_content"],
title=doc.get("title"),
metadata=metadata or None,
format="html",
)

24
haiku_rag_slim/haiku/rag/client.py
View file

@ -100,9 +100,13 @@ class HaikuRAG:
async def convert(self, source: Path) -> "DoclingDocument": ...
@overload
async def convert(self, source: str) -> "DoclingDocument": ...
async def convert(
self, source: str, *, format: str = "md"
) -> "DoclingDocument": ...
async def convert(self, source: Path | str) -> "DoclingDocument":
async def convert(
self, source: Path | str, *, format: str = "md"
) -> "DoclingDocument":
"""Convert a file, URL, or text to DoclingDocument.
Args:
@ -110,6 +114,9 @@ class HaikuRAG:
- Path: Local file path to convert
- str (URL): HTTP/HTTPS URL to download and convert
- str (text): Raw text content to convert
format: The format of text content ("md" or "html"). Defaults to "md".
Only used when source is raw text (not a file path or URL).
Files and URLs determine format from extension/content-type.
Returns:
DoclingDocument from the converted source.
@ -170,7 +177,7 @@ class HaikuRAG:
else:
# Treat as text content
return await converter.convert_text(source)
return await converter.convert_text(source, format=format)
async def chunk(self, docling_document: "DoclingDocument") -> list[Chunk]:
"""Chunk a DoclingDocument into Chunks.
@ -329,6 +336,7 @@ class HaikuRAG:
uri: str | None = None,
title: str | None = None,
metadata: dict | None = None,
format: str = "md",
) -> Document:
"""Create a new document from text content.
@ -339,6 +347,8 @@ class HaikuRAG:
uri: Optional URI identifier for the document.
title: Optional title for the document.
metadata: Optional metadata dictionary.
format: The format of the content ("md" or "html"). Defaults to "md".
This determines which parser is used to interpret the content structure.
Returns:
The created Document instance.
@ -346,13 +356,17 @@ class HaikuRAG:
from haiku.rag.embeddings import embed_chunks
# Convert → Chunk → Embed using primitives
docling_document = await self.convert(content)
docling_document = await self.convert(content, format=format)
chunks = await self.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, self._config)
# Store markdown export as content for better display/readability
# The original content is preserved in docling_document_json
stored_content = docling_document.export_to_markdown()
# Create document model
document = Document(
content=content,
content=stored_content,
uri=uri,
title=title,
metadata=metadata or {},

8
haiku_rag_slim/haiku/rag/converters/base.py
View file

@ -40,20 +40,24 @@ class DocumentConverter(ABC):
"""
pass
SUPPORTED_FORMATS = ("md", "html")
@abstractmethod
async def convert_text(
self, text: str, name: str = "content.md"
self, text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text content to DoclingDocument format.
Args:
text: The text content to convert.
name: The name to use for the document (defaults to "content.md").
format: The format of the text content ("md" or "html"). Defaults to "md".
This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
ValueError: If the text cannot be converted.
ValueError: If the text cannot be converted or format is unsupported.
"""
pass

8
haiku_rag_slim/haiku/rag/converters/docling_local.py
View file

@ -137,18 +137,20 @@ class DoclingLocalConverter(DocumentConverter):
raise ValueError(f"Failed to parse file: {path}")
async def convert_text(
self, text: str, name: str = "content.md"
self, text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text content to DoclingDocument using local docling.
Args:
text: The text content to convert.
name: The name to use for the document (defaults to "content.md").
format: The format of the text content ("md" or "html"). Defaults to "md".
This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
ValueError: If the text cannot be converted.
ValueError: If the text cannot be converted or format is unsupported.
"""
return await TextFileHandler.text_to_docling_document(text, name)
return await TextFileHandler.text_to_docling_document(text, name, format)

24
haiku_rag_slim/haiku/rag/converters/docling_serve.py
View file

@ -176,23 +176,37 @@ class DoclingServeConverter(DocumentConverter):
files = {"files": (path.name, file_content, "application/octet-stream")}
return await self._make_request(files, path.name)
SUPPORTED_FORMATS = ("md", "html")
async def convert_text(
self, text: str, name: str = "content.md"
self, text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text content to DoclingDocument via docling-serve.
Sends the text as a markdown file to docling-serve for conversion.
Sends the text to docling-serve for conversion using the specified format.
Args:
text: The text content to convert.
name: The name to use for the document (defaults to "content.md").
format: The format of the text content ("md" or "html"). Defaults to "md".
This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
ValueError: If the text cannot be converted.
ValueError: If the text cannot be converted or format is unsupported.
"""
if format not in self.SUPPORTED_FORMATS:
raise ValueError(
f"Unsupported format: {format}. "
f"Supported formats: {', '.join(self.SUPPORTED_FORMATS)}"
)
# Derive document name from format to tell docling which parser to use
doc_name = f"content.{format}" if name == "content.md" else name
mime_type = "text/html" if format == "html" else "text/markdown"
text_bytes = text.encode("utf-8")
files = {"files": (name, text_bytes, "text/markdown")}
return await self._make_request(files, name)
files = {"files": (doc_name, text_bytes, mime_type)}
return await self._make_request(files, doc_name)

25
haiku_rag_slim/haiku/rag/converters/text_utils.py
View file

@ -89,39 +89,52 @@ class TextFileHandler:
return f"```{language}\n{content}\n```"
return content
SUPPORTED_FORMATS = ("md", "html")
@staticmethod
def _sync_text_to_docling_document(
text: str, name: str = "content.md"
text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Synchronous implementation of text to DoclingDocument conversion."""
from docling.document_converter import DocumentConverter as DoclingDocConverter
from docling_core.types.io import DocumentStream
if format not in TextFileHandler.SUPPORTED_FORMATS:
raise ValueError(
f"Unsupported format: {format}. "
f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}"
)
# Derive document name from format to tell docling which parser to use
doc_name = f"content.{format}" if name == "content.md" else name
bytes_io = BytesIO(text.encode("utf-8"))
doc_stream = DocumentStream(name=name, stream=bytes_io)
doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
converter = DoclingDocConverter()
result = converter.convert(doc_stream)
return result.document
@staticmethod
async def text_to_docling_document(
text: str, name: str = "content.md"
text: str, name: str = "content.md", format: str = "md"
) -> "DoclingDocument":
"""Convert text to DoclingDocument using docling's markdown parser.
"""Convert text to DoclingDocument using docling's parser.
Args:
text: The text content to convert.
name: The name to use for the document.
format: The format of the text content ("md" or "html"). Defaults to "md".
This determines which parser docling uses to interpret the content.
Returns:
DoclingDocument representation of the text.
Raises:
ValueError: If the conversion fails.
ValueError: If the conversion fails or format is unsupported.
"""
try:
return await asyncio.to_thread(
TextFileHandler._sync_text_to_docling_document, text, name
TextFileHandler._sync_text_to_docling_document, text, name, format
)
except Exception as e:
raise ValueError(f"Failed to convert text to DoclingDocument: {e}")

56
tests/test_client.py
View file

@ -28,7 +28,8 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path):
)
assert created_doc.id is not None
assert created_doc.content == document_text
# Content is stored as markdown export, check key text is preserved
assert "Jakarta" in created_doc.content
assert created_doc.uri == test_uri
assert created_doc.metadata == test_metadata
@ -36,14 +37,14 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path):
retrieved_doc = await client.get_document_by_id(created_doc.id)
assert retrieved_doc is not None
assert retrieved_doc.id == created_doc.id
assert retrieved_doc.content == document_text
assert "Jakarta" in retrieved_doc.content
assert retrieved_doc.uri == test_uri
# Test get_document_by_uri
retrieved_by_uri = await client.get_document_by_uri(test_uri)
assert retrieved_by_uri is not None
assert retrieved_by_uri.id == created_doc.id
assert retrieved_by_uri.content == document_text
assert "Jakarta" in retrieved_by_uri.content
# Test get_document_by_uri with non-existent URI
non_existent = await client.get_document_by_uri("file:///non/existent.txt")
@ -1380,3 +1381,52 @@ async def test_update_document_embeds_chunks_without_embeddings(temp_db_path):
results = await client.search("Updated chunk", search_type="vector")
assert len(results) > 0
assert results[0].content == "Updated chunk without embedding"
@pytest.mark.asyncio
async def test_client_create_document_with_html_format(temp_db_path):
"""Test create_document with HTML format preserves document structure."""
async with HaikuRAG(temp_db_path, create=True) as client:
html_content = """
<h1>Main Title</h1>
<p>Introduction paragraph.</p>
<h2>Section Header</h2>
<ul>
<li>Item 1</li>
<li>Item 2</li>
</ul>
"""
doc = await client.create_document(
content=html_content,
uri="test://html-doc",
format="html",
)
assert doc.id is not None
assert doc.docling_document_json is not None
# Verify the DoclingDocument has proper structure
docling_doc = doc.get_docling_document()
assert docling_doc is not None
items = list(docling_doc.iterate_items())
labels = [str(getattr(item, "label", "")) for item, _ in items]
# HTML format should preserve headers and list items
assert "title" in labels or "section_header" in labels
assert "list_item" in labels
@pytest.mark.asyncio
async def test_client_convert_with_html_format(temp_db_path):
"""Test convert with HTML format."""
async with HaikuRAG(temp_db_path, create=True) as client:
html_content = "<h1>Title</h1><p>Text</p>"
docling_doc = await client.convert(html_content, format="html")
items = list(docling_doc.iterate_items())
labels = [str(getattr(item, "label", "")) for item, _ in items]
assert "title" in labels

67
tests/test_converters.py
View file

@ -116,6 +116,73 @@ class TestConverterFactory:
get_converter(config)
class TestTextToDoclingWithFormat:
"""Tests for format parameter in text to DoclingDocument conversion."""
@pytest.mark.asyncio
async def test_html_format_preserves_structure(self):
"""Test that HTML content parsed with html format preserves document structure."""
html_content = """
<h1>Main Title</h1>
<p>Introduction paragraph.</p>
<h2>Section Header</h2>
<ul>
<li>Item 1</li>
<li>Item 2</li>
</ul>
"""
config = AppConfig()
converter = DoclingLocalConverter(config)
# With html format, should get proper structure
doc = await converter.convert_text(
html_content, name="content.html", format="html"
)
items = list(doc.iterate_items())
labels = [str(getattr(item, "label", "")) for item, _ in items]
assert "title" in labels or "section_header" in labels
assert "list_item" in labels
assert len(items) > 3
@pytest.mark.asyncio
async def test_md_format_is_default(self):
"""Test that md format is used by default."""
config = AppConfig()
converter = DoclingLocalConverter(config)
# Plain text should work with default format
doc = await converter.convert_text("# Heading\n\nParagraph text.")
items = list(doc.iterate_items())
assert len(items) >= 2
@pytest.mark.asyncio
async def test_html_as_md_loses_structure(self):
"""Test that HTML parsed as markdown loses semantic structure."""
html_content = "<h1>Title</h1><p>Text</p><ul><li>Item</li></ul>"
config = AppConfig()
converter = DoclingLocalConverter(config)
# With md format (default), HTML tags are treated as text
doc = await converter.convert_text(html_content, format="md")
items = list(doc.iterate_items())
# Should still parse but with different structure
# (markdown parser will interpret some HTML)
assert len(items) >= 1
@pytest.mark.asyncio
async def test_invalid_format_raises_error(self):
"""Test that invalid format raises ValueError."""
config = AppConfig()
converter = DoclingLocalConverter(config)
with pytest.raises(ValueError, match="Unsupported format"):
await converter.convert_text("content", format="invalid")
class TestDoclingLocalConverter:
"""Tests for DoclingLocalConverter."""