Add format parameter for text-to-DoclingDocument conversion

CHANGELOG.md

@@ -10,6 +10,12 @@
 
 ### Added
 
+- **Format Parameter for Text Conversion**: New `format` parameter for `convert()` and `create_document()` to specify content type
+  - Supports `"md"` (default) for markdown and `"html"` for HTML content
+  - HTML format preserves document structure (headings, lists, sections) in DoclingDocument
+  - Enables proper parsing of HTML content that was previously treated as plain text
+  - Document content is stored as markdown export for consistent display (original preserved in `docling_document_json`)
+  - Wix evaluation dataset now uses `html_content` with `format="html"` for better document structure
 - **DoclingDocument Storage**: Full DoclingDocument JSON is now stored with each document, enabling rich context and visual grounding
   - Documents store the complete DoclingDocument structure (JSON) and schema version
   - Chunks store metadata with JSON pointer references (`doc_item_refs`), semantic labels, section headings, and page numbers

docs/custom-pipelines.md

@@ -53,13 +53,25 @@ docling_doc = await client.convert(Path("/absolute/path/to/file.docx"))
 # From URL (downloads and converts)
 docling_doc = await client.convert("https://example.com/paper.pdf")
 
-# From plain text
+# From plain text (parsed as markdown by default)
-docling_doc = await client.convert("Your text content here")
+docling_doc = await client.convert("# Title\n\nYour text content here")
+
+# From HTML text (use format parameter to preserve structure)
+html_content = "<h1>Title</h1><p>Paragraph</p><ul><li>Item</li></ul>"
+docling_doc = await client.convert(html_content, format="html")
 
 # From file:// URI
 docling_doc = await client.convert("file:///path/to/document.md")
 ```
 
+The `format` parameter controls how text content is parsed:
+
+- `"md"` (default) - Parse as Markdown
+- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
+
+!!! note
+    The `format` parameter only applies to text content. Files and URLs determine their format from the file extension or content-type header.
+
 Supported formats depend on your converter configuration (docling-local or docling-serve). Common formats include PDF, DOCX, HTML, Markdown, and images.
 
 ## Chunk

docs/python.md

@@ -36,6 +36,24 @@ doc = await client.create_document(
 )
 ```
 
+From HTML content (preserves document structure):
+```python
+html_content = "<h1>Title</h1><p>Paragraph</p><ul><li>Item 1</li></ul>"
+doc = await client.create_document(
+    content=html_content,
+    uri="doc://html-example",
+    format="html"  # parse as HTML instead of markdown
+)
+```
+
+The `format` parameter controls how text content is parsed:
+
+- `"md"` (default) - Parse as Markdown
+- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
+
+!!! note
+    The document's `content` field stores the markdown export of the parsed document for consistent display. The original input is preserved in the `docling_document_json` field.
+
 From file:
 ```python
 doc = await client.create_document_from_source(

evaluations/evaluations/benchmark.py

@@ -98,6 +98,7 @@ async def populate_db(
                     uri=payload.uri,
                     title=payload.title,
                     metadata=payload.metadata,
+                    format=payload.format,
                 )
                 progress.advance(task)
 

evaluations/evaluations/config.py

@@ -14,6 +14,7 @@ class DocumentPayload:
     content: str
     title: str | None = None
     metadata: dict[str, Any] | None = None
+    format: str = "md"
 
 
 @dataclass

evaluations/evaluations/datasets/wix.py

@@ -27,9 +27,10 @@ def map_wix_document(doc: Mapping[str, Any]) -> DocumentPayload:
 
     return DocumentPayload(
         uri=uri,
-        content=doc["contents"],
+        content=doc["html_content"],
         title=doc.get("title"),
         metadata=metadata or None,
+        format="html",
     )
 
 

haiku_rag_slim/haiku/rag/client.py

@@ -100,9 +100,13 @@ class HaikuRAG:
     async def convert(self, source: Path) -> "DoclingDocument": ...
 
     @overload
-    async def convert(self, source: str) -> "DoclingDocument": ...
+    async def convert(
+        self, source: str, *, format: str = "md"
+    ) -> "DoclingDocument": ...
 
-    async def convert(self, source: Path | str) -> "DoclingDocument":
+    async def convert(
+        self, source: Path | str, *, format: str = "md"
+    ) -> "DoclingDocument":
         """Convert a file, URL, or text to DoclingDocument.
 
         Args:
@@ -110,6 +114,9 @@ class HaikuRAG:
                 - Path: Local file path to convert
                 - str (URL): HTTP/HTTPS URL to download and convert
                 - str (text): Raw text content to convert
+            format: The format of text content ("md" or "html"). Defaults to "md".
+                Only used when source is raw text (not a file path or URL).
+                Files and URLs determine format from extension/content-type.
 
         Returns:
             DoclingDocument from the converted source.
@@ -170,7 +177,7 @@ class HaikuRAG:
 
         else:
             # Treat as text content
-            return await converter.convert_text(source)
+            return await converter.convert_text(source, format=format)
 
     async def chunk(self, docling_document: "DoclingDocument") -> list[Chunk]:
         """Chunk a DoclingDocument into Chunks.
@@ -329,6 +336,7 @@ class HaikuRAG:
         uri: str | None = None,
         title: str | None = None,
         metadata: dict | None = None,
+        format: str = "md",
     ) -> Document:
         """Create a new document from text content.
 
@@ -339,6 +347,8 @@ class HaikuRAG:
             uri: Optional URI identifier for the document.
             title: Optional title for the document.
             metadata: Optional metadata dictionary.
+            format: The format of the content ("md" or "html"). Defaults to "md".
+                This determines which parser is used to interpret the content structure.
 
         Returns:
             The created Document instance.
@@ -346,13 +356,17 @@ class HaikuRAG:
         from haiku.rag.embeddings import embed_chunks
 
         # Convert → Chunk → Embed using primitives
-        docling_document = await self.convert(content)
+        docling_document = await self.convert(content, format=format)
         chunks = await self.chunk(docling_document)
         embedded_chunks = await embed_chunks(chunks, self._config)
 
+        # Store markdown export as content for better display/readability
+        # The original content is preserved in docling_document_json
+        stored_content = docling_document.export_to_markdown()
+
         # Create document model
         document = Document(
-            content=content,
+            content=stored_content,
             uri=uri,
             title=title,
             metadata=metadata or {},

haiku_rag_slim/haiku/rag/converters/base.py

@@ -40,20 +40,24 @@ class DocumentConverter(ABC):
         """
         pass
 
+    SUPPORTED_FORMATS = ("md", "html")
+
     @abstractmethod
     async def convert_text(
-        self, text: str, name: str = "content.md"
+        self, text: str, name: str = "content.md", format: str = "md"
     ) -> "DoclingDocument":
         """Convert text content to DoclingDocument format.
 
         Args:
             text: The text content to convert.
             name: The name to use for the document (defaults to "content.md").
+            format: The format of the text content ("md" or "html"). Defaults to "md".
+                This determines which parser docling uses to interpret the content.
 
         Returns:
             DoclingDocument representation of the text.
 
         Raises:
-            ValueError: If the text cannot be converted.
+            ValueError: If the text cannot be converted or format is unsupported.
         """
         pass

haiku_rag_slim/haiku/rag/converters/docling_local.py

@@ -137,18 +137,20 @@ class DoclingLocalConverter(DocumentConverter):
             raise ValueError(f"Failed to parse file: {path}")
 
     async def convert_text(
-        self, text: str, name: str = "content.md"
+        self, text: str, name: str = "content.md", format: str = "md"
     ) -> "DoclingDocument":
         """Convert text content to DoclingDocument using local docling.
 
         Args:
             text: The text content to convert.
             name: The name to use for the document (defaults to "content.md").
+            format: The format of the text content ("md" or "html"). Defaults to "md".
+                This determines which parser docling uses to interpret the content.
 
         Returns:
             DoclingDocument representation of the text.
 
         Raises:
-            ValueError: If the text cannot be converted.
+            ValueError: If the text cannot be converted or format is unsupported.
         """
-        return await TextFileHandler.text_to_docling_document(text, name)
+        return await TextFileHandler.text_to_docling_document(text, name, format)

haiku_rag_slim/haiku/rag/converters/docling_serve.py

@@ -176,23 +176,37 @@ class DoclingServeConverter(DocumentConverter):
         files = {"files": (path.name, file_content, "application/octet-stream")}
         return await self._make_request(files, path.name)
 
+    SUPPORTED_FORMATS = ("md", "html")
+
     async def convert_text(
-        self, text: str, name: str = "content.md"
+        self, text: str, name: str = "content.md", format: str = "md"
     ) -> "DoclingDocument":
         """Convert text content to DoclingDocument via docling-serve.
 
-        Sends the text as a markdown file to docling-serve for conversion.
+        Sends the text to docling-serve for conversion using the specified format.
 
         Args:
             text: The text content to convert.
             name: The name to use for the document (defaults to "content.md").
+            format: The format of the text content ("md" or "html"). Defaults to "md".
+                This determines which parser docling uses to interpret the content.
 
         Returns:
             DoclingDocument representation of the text.
 
         Raises:
-            ValueError: If the text cannot be converted.
+            ValueError: If the text cannot be converted or format is unsupported.
         """
+        if format not in self.SUPPORTED_FORMATS:
+            raise ValueError(
+                f"Unsupported format: {format}. "
+                f"Supported formats: {', '.join(self.SUPPORTED_FORMATS)}"
+            )
+
+        # Derive document name from format to tell docling which parser to use
+        doc_name = f"content.{format}" if name == "content.md" else name
+        mime_type = "text/html" if format == "html" else "text/markdown"
+
         text_bytes = text.encode("utf-8")
-        files = {"files": (name, text_bytes, "text/markdown")}
+        files = {"files": (doc_name, text_bytes, mime_type)}
-        return await self._make_request(files, name)
+        return await self._make_request(files, doc_name)

haiku_rag_slim/haiku/rag/converters/text_utils.py

@@ -89,39 +89,52 @@ class TextFileHandler:
             return f"```{language}\n{content}\n```"
         return content
 
+    SUPPORTED_FORMATS = ("md", "html")
+
     @staticmethod
     def _sync_text_to_docling_document(
-        text: str, name: str = "content.md"
+        text: str, name: str = "content.md", format: str = "md"
     ) -> "DoclingDocument":
         """Synchronous implementation of text to DoclingDocument conversion."""
         from docling.document_converter import DocumentConverter as DoclingDocConverter
         from docling_core.types.io import DocumentStream
 
+        if format not in TextFileHandler.SUPPORTED_FORMATS:
+            raise ValueError(
+                f"Unsupported format: {format}. "
+                f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}"
+            )
+
+        # Derive document name from format to tell docling which parser to use
+        doc_name = f"content.{format}" if name == "content.md" else name
+
         bytes_io = BytesIO(text.encode("utf-8"))
-        doc_stream = DocumentStream(name=name, stream=bytes_io)
+        doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
         converter = DoclingDocConverter()
         result = converter.convert(doc_stream)
         return result.document
 
     @staticmethod
     async def text_to_docling_document(
-        text: str, name: str = "content.md"
+        text: str, name: str = "content.md", format: str = "md"
     ) -> "DoclingDocument":
-        """Convert text to DoclingDocument using docling's markdown parser.
+        """Convert text to DoclingDocument using docling's parser.
 
         Args:
             text: The text content to convert.
             name: The name to use for the document.
+            format: The format of the text content ("md" or "html"). Defaults to "md".
+                This determines which parser docling uses to interpret the content.
 
         Returns:
             DoclingDocument representation of the text.
 
         Raises:
-            ValueError: If the conversion fails.
+            ValueError: If the conversion fails or format is unsupported.
         """
         try:
             return await asyncio.to_thread(
-                TextFileHandler._sync_text_to_docling_document, text, name
+                TextFileHandler._sync_text_to_docling_document, text, name, format
             )
         except Exception as e:
             raise ValueError(f"Failed to convert text to DoclingDocument: {e}")

tests/test_client.py

@@ -28,7 +28,8 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path):
         )
 
         assert created_doc.id is not None
-        assert created_doc.content == document_text
+        # Content is stored as markdown export, check key text is preserved
+        assert "Jakarta" in created_doc.content
         assert created_doc.uri == test_uri
         assert created_doc.metadata == test_metadata
 
@@ -36,14 +37,14 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path):
         retrieved_doc = await client.get_document_by_id(created_doc.id)
         assert retrieved_doc is not None
         assert retrieved_doc.id == created_doc.id
-        assert retrieved_doc.content == document_text
+        assert "Jakarta" in retrieved_doc.content
         assert retrieved_doc.uri == test_uri
 
         # Test get_document_by_uri
         retrieved_by_uri = await client.get_document_by_uri(test_uri)
         assert retrieved_by_uri is not None
         assert retrieved_by_uri.id == created_doc.id
-        assert retrieved_by_uri.content == document_text
+        assert "Jakarta" in retrieved_by_uri.content
 
         # Test get_document_by_uri with non-existent URI
         non_existent = await client.get_document_by_uri("file:///non/existent.txt")
@@ -1380,3 +1381,52 @@ async def test_update_document_embeds_chunks_without_embeddings(temp_db_path):
         results = await client.search("Updated chunk", search_type="vector")
         assert len(results) > 0
         assert results[0].content == "Updated chunk without embedding"
+
+
+@pytest.mark.asyncio
+async def test_client_create_document_with_html_format(temp_db_path):
+    """Test create_document with HTML format preserves document structure."""
+    async with HaikuRAG(temp_db_path, create=True) as client:
+        html_content = """
+        <h1>Main Title</h1>
+        <p>Introduction paragraph.</p>
+        <h2>Section Header</h2>
+        <ul>
+            <li>Item 1</li>
+            <li>Item 2</li>
+        </ul>
+        """
+
+        doc = await client.create_document(
+            content=html_content,
+            uri="test://html-doc",
+            format="html",
+        )
+
+        assert doc.id is not None
+        assert doc.docling_document_json is not None
+
+        # Verify the DoclingDocument has proper structure
+        docling_doc = doc.get_docling_document()
+        assert docling_doc is not None
+
+        items = list(docling_doc.iterate_items())
+        labels = [str(getattr(item, "label", "")) for item, _ in items]
+
+        # HTML format should preserve headers and list items
+        assert "title" in labels or "section_header" in labels
+        assert "list_item" in labels
+
+
+@pytest.mark.asyncio
+async def test_client_convert_with_html_format(temp_db_path):
+    """Test convert with HTML format."""
+    async with HaikuRAG(temp_db_path, create=True) as client:
+        html_content = "<h1>Title</h1><p>Text</p>"
+
+        docling_doc = await client.convert(html_content, format="html")
+
+        items = list(docling_doc.iterate_items())
+        labels = [str(getattr(item, "label", "")) for item, _ in items]
+
+        assert "title" in labels

tests/test_converters.py

@@ -116,6 +116,73 @@ class TestConverterFactory:
             get_converter(config)
 
 
+class TestTextToDoclingWithFormat:
+    """Tests for format parameter in text to DoclingDocument conversion."""
+
+    @pytest.mark.asyncio
+    async def test_html_format_preserves_structure(self):
+        """Test that HTML content parsed with html format preserves document structure."""
+        html_content = """
+        <h1>Main Title</h1>
+        <p>Introduction paragraph.</p>
+        <h2>Section Header</h2>
+        <ul>
+            <li>Item 1</li>
+            <li>Item 2</li>
+        </ul>
+        """
+        config = AppConfig()
+        converter = DoclingLocalConverter(config)
+
+        # With html format, should get proper structure
+        doc = await converter.convert_text(
+            html_content, name="content.html", format="html"
+        )
+
+        items = list(doc.iterate_items())
+        labels = [str(getattr(item, "label", "")) for item, _ in items]
+
+        assert "title" in labels or "section_header" in labels
+        assert "list_item" in labels
+        assert len(items) > 3
+
+    @pytest.mark.asyncio
+    async def test_md_format_is_default(self):
+        """Test that md format is used by default."""
+        config = AppConfig()
+        converter = DoclingLocalConverter(config)
+
+        # Plain text should work with default format
+        doc = await converter.convert_text("# Heading\n\nParagraph text.")
+        items = list(doc.iterate_items())
+
+        assert len(items) >= 2
+
+    @pytest.mark.asyncio
+    async def test_html_as_md_loses_structure(self):
+        """Test that HTML parsed as markdown loses semantic structure."""
+        html_content = "<h1>Title</h1><p>Text</p><ul><li>Item</li></ul>"
+        config = AppConfig()
+        converter = DoclingLocalConverter(config)
+
+        # With md format (default), HTML tags are treated as text
+        doc = await converter.convert_text(html_content, format="md")
+        items = list(doc.iterate_items())
+
+        # Should still parse but with different structure
+        # (markdown parser will interpret some HTML)
+        assert len(items) >= 1
+
+    @pytest.mark.asyncio
+    async def test_invalid_format_raises_error(self):
+        """Test that invalid format raises ValueError."""
+        config = AppConfig()
+        converter = DoclingLocalConverter(config)
+
+        with pytest.raises(ValueError, match="Unsupported format"):
+            await converter.convert_text("content", format="invalid")
+
+
 class TestDoclingLocalConverter:
     """Tests for DoclingLocalConverter."""