From 06b4f6a2245790b9bbce34fdb8264dc335d89179 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 8 Dec 2025 12:54:54 +0200 Subject: [PATCH] Add format parameter for text-to-DoclingDocument conversion --- CHANGELOG.md | 6 ++ docs/custom-pipelines.md | 16 ++++- docs/python.md | 18 +++++ evaluations/evaluations/benchmark.py | 1 + evaluations/evaluations/config.py | 1 + evaluations/evaluations/datasets/wix.py | 3 +- haiku_rag_slim/haiku/rag/client.py | 24 +++++-- haiku_rag_slim/haiku/rag/converters/base.py | 8 ++- .../haiku/rag/converters/docling_local.py | 8 ++- .../haiku/rag/converters/docling_serve.py | 24 +++++-- .../haiku/rag/converters/text_utils.py | 25 +++++-- tests/test_client.py | 56 +++++++++++++++- tests/test_converters.py | 67 +++++++++++++++++++ 13 files changed, 230 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d271b75c..385ef08a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ ### Added +- **Format Parameter for Text Conversion**: New `format` parameter for `convert()` and `create_document()` to specify content type + - Supports `"md"` (default) for markdown and `"html"` for HTML content + - HTML format preserves document structure (headings, lists, sections) in DoclingDocument + - Enables proper parsing of HTML content that was previously treated as plain text + - Document content is stored as markdown export for consistent display (original preserved in `docling_document_json`) + - Wix evaluation dataset now uses `html_content` with `format="html"` for better document structure - **DoclingDocument Storage**: Full DoclingDocument JSON is now stored with each document, enabling rich context and visual grounding - Documents store the complete DoclingDocument structure (JSON) and schema version - Chunks store metadata with JSON pointer references (`doc_item_refs`), semantic labels, section headings, and page numbers diff --git a/docs/custom-pipelines.md b/docs/custom-pipelines.md index bfcd3365..afe7dc4d 100644 --- a/docs/custom-pipelines.md +++ b/docs/custom-pipelines.md @@ -53,13 +53,25 @@ docling_doc = await client.convert(Path("/absolute/path/to/file.docx")) # From URL (downloads and converts) docling_doc = await client.convert("https://example.com/paper.pdf") -# From plain text -docling_doc = await client.convert("Your text content here") +# From plain text (parsed as markdown by default) +docling_doc = await client.convert("# Title\n\nYour text content here") + +# From HTML text (use format parameter to preserve structure) +html_content = "

Title

Paragraph

" +docling_doc = await client.convert(html_content, format="html") # From file:// URI docling_doc = await client.convert("file:///path/to/document.md") ``` +The `format` parameter controls how text content is parsed: + +- `"md"` (default) - Parse as Markdown +- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables) + +!!! note + The `format` parameter only applies to text content. Files and URLs determine their format from the file extension or content-type header. + Supported formats depend on your converter configuration (docling-local or docling-serve). Common formats include PDF, DOCX, HTML, Markdown, and images. ## Chunk diff --git a/docs/python.md b/docs/python.md index 645813bc..ff75d953 100644 --- a/docs/python.md +++ b/docs/python.md @@ -36,6 +36,24 @@ doc = await client.create_document( ) ``` +From HTML content (preserves document structure): +```python +html_content = "

Title

Paragraph

" +doc = await client.create_document( + content=html_content, + uri="doc://html-example", + format="html" # parse as HTML instead of markdown +) +``` + +The `format` parameter controls how text content is parsed: + +- `"md"` (default) - Parse as Markdown +- `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables) + +!!! note + The document's `content` field stores the markdown export of the parsed document for consistent display. The original input is preserved in the `docling_document_json` field. + From file: ```python doc = await client.create_document_from_source( diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index da7fbbc1..9cc391cf 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -98,6 +98,7 @@ async def populate_db( uri=payload.uri, title=payload.title, metadata=payload.metadata, + format=payload.format, ) progress.advance(task) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index f791cbef..acd71eb1 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -14,6 +14,7 @@ class DocumentPayload: content: str title: str | None = None metadata: dict[str, Any] | None = None + format: str = "md" @dataclass diff --git a/evaluations/evaluations/datasets/wix.py b/evaluations/evaluations/datasets/wix.py index 6b4358c9..c6f01c3f 100644 --- a/evaluations/evaluations/datasets/wix.py +++ b/evaluations/evaluations/datasets/wix.py @@ -27,9 +27,10 @@ def map_wix_document(doc: Mapping[str, Any]) -> DocumentPayload: return DocumentPayload( uri=uri, - content=doc["contents"], + content=doc["html_content"], title=doc.get("title"), metadata=metadata or None, + format="html", ) diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index a890ca2d..264d3dbc 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -100,9 +100,13 @@ class HaikuRAG: async def convert(self, source: Path) -> "DoclingDocument": ... @overload - async def convert(self, source: str) -> "DoclingDocument": ... + async def convert( + self, source: str, *, format: str = "md" + ) -> "DoclingDocument": ... - async def convert(self, source: Path | str) -> "DoclingDocument": + async def convert( + self, source: Path | str, *, format: str = "md" + ) -> "DoclingDocument": """Convert a file, URL, or text to DoclingDocument. Args: @@ -110,6 +114,9 @@ class HaikuRAG: - Path: Local file path to convert - str (URL): HTTP/HTTPS URL to download and convert - str (text): Raw text content to convert + format: The format of text content ("md" or "html"). Defaults to "md". + Only used when source is raw text (not a file path or URL). + Files and URLs determine format from extension/content-type. Returns: DoclingDocument from the converted source. @@ -170,7 +177,7 @@ class HaikuRAG: else: # Treat as text content - return await converter.convert_text(source) + return await converter.convert_text(source, format=format) async def chunk(self, docling_document: "DoclingDocument") -> list[Chunk]: """Chunk a DoclingDocument into Chunks. @@ -329,6 +336,7 @@ class HaikuRAG: uri: str | None = None, title: str | None = None, metadata: dict | None = None, + format: str = "md", ) -> Document: """Create a new document from text content. @@ -339,6 +347,8 @@ class HaikuRAG: uri: Optional URI identifier for the document. title: Optional title for the document. metadata: Optional metadata dictionary. + format: The format of the content ("md" or "html"). Defaults to "md". + This determines which parser is used to interpret the content structure. Returns: The created Document instance. @@ -346,13 +356,17 @@ class HaikuRAG: from haiku.rag.embeddings import embed_chunks # Convert → Chunk → Embed using primitives - docling_document = await self.convert(content) + docling_document = await self.convert(content, format=format) chunks = await self.chunk(docling_document) embedded_chunks = await embed_chunks(chunks, self._config) + # Store markdown export as content for better display/readability + # The original content is preserved in docling_document_json + stored_content = docling_document.export_to_markdown() + # Create document model document = Document( - content=content, + content=stored_content, uri=uri, title=title, metadata=metadata or {}, diff --git a/haiku_rag_slim/haiku/rag/converters/base.py b/haiku_rag_slim/haiku/rag/converters/base.py index 713cbd94..1a6b883f 100644 --- a/haiku_rag_slim/haiku/rag/converters/base.py +++ b/haiku_rag_slim/haiku/rag/converters/base.py @@ -40,20 +40,24 @@ class DocumentConverter(ABC): """ pass + SUPPORTED_FORMATS = ("md", "html") + @abstractmethod async def convert_text( - self, text: str, name: str = "content.md" + self, text: str, name: str = "content.md", format: str = "md" ) -> "DoclingDocument": """Convert text content to DoclingDocument format. Args: text: The text content to convert. name: The name to use for the document (defaults to "content.md"). + format: The format of the text content ("md" or "html"). Defaults to "md". + This determines which parser docling uses to interpret the content. Returns: DoclingDocument representation of the text. Raises: - ValueError: If the text cannot be converted. + ValueError: If the text cannot be converted or format is unsupported. """ pass diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index f8b6a4ec..64d817b5 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -137,18 +137,20 @@ class DoclingLocalConverter(DocumentConverter): raise ValueError(f"Failed to parse file: {path}") async def convert_text( - self, text: str, name: str = "content.md" + self, text: str, name: str = "content.md", format: str = "md" ) -> "DoclingDocument": """Convert text content to DoclingDocument using local docling. Args: text: The text content to convert. name: The name to use for the document (defaults to "content.md"). + format: The format of the text content ("md" or "html"). Defaults to "md". + This determines which parser docling uses to interpret the content. Returns: DoclingDocument representation of the text. Raises: - ValueError: If the text cannot be converted. + ValueError: If the text cannot be converted or format is unsupported. """ - return await TextFileHandler.text_to_docling_document(text, name) + return await TextFileHandler.text_to_docling_document(text, name, format) diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py index 09232af2..5f61d7bb 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py @@ -176,23 +176,37 @@ class DoclingServeConverter(DocumentConverter): files = {"files": (path.name, file_content, "application/octet-stream")} return await self._make_request(files, path.name) + SUPPORTED_FORMATS = ("md", "html") + async def convert_text( - self, text: str, name: str = "content.md" + self, text: str, name: str = "content.md", format: str = "md" ) -> "DoclingDocument": """Convert text content to DoclingDocument via docling-serve. - Sends the text as a markdown file to docling-serve for conversion. + Sends the text to docling-serve for conversion using the specified format. Args: text: The text content to convert. name: The name to use for the document (defaults to "content.md"). + format: The format of the text content ("md" or "html"). Defaults to "md". + This determines which parser docling uses to interpret the content. Returns: DoclingDocument representation of the text. Raises: - ValueError: If the text cannot be converted. + ValueError: If the text cannot be converted or format is unsupported. """ + if format not in self.SUPPORTED_FORMATS: + raise ValueError( + f"Unsupported format: {format}. " + f"Supported formats: {', '.join(self.SUPPORTED_FORMATS)}" + ) + + # Derive document name from format to tell docling which parser to use + doc_name = f"content.{format}" if name == "content.md" else name + mime_type = "text/html" if format == "html" else "text/markdown" + text_bytes = text.encode("utf-8") - files = {"files": (name, text_bytes, "text/markdown")} - return await self._make_request(files, name) + files = {"files": (doc_name, text_bytes, mime_type)} + return await self._make_request(files, doc_name) diff --git a/haiku_rag_slim/haiku/rag/converters/text_utils.py b/haiku_rag_slim/haiku/rag/converters/text_utils.py index a149297c..36263155 100644 --- a/haiku_rag_slim/haiku/rag/converters/text_utils.py +++ b/haiku_rag_slim/haiku/rag/converters/text_utils.py @@ -89,39 +89,52 @@ class TextFileHandler: return f"```{language}\n{content}\n```" return content + SUPPORTED_FORMATS = ("md", "html") + @staticmethod def _sync_text_to_docling_document( - text: str, name: str = "content.md" + text: str, name: str = "content.md", format: str = "md" ) -> "DoclingDocument": """Synchronous implementation of text to DoclingDocument conversion.""" from docling.document_converter import DocumentConverter as DoclingDocConverter from docling_core.types.io import DocumentStream + if format not in TextFileHandler.SUPPORTED_FORMATS: + raise ValueError( + f"Unsupported format: {format}. " + f"Supported formats: {', '.join(TextFileHandler.SUPPORTED_FORMATS)}" + ) + + # Derive document name from format to tell docling which parser to use + doc_name = f"content.{format}" if name == "content.md" else name + bytes_io = BytesIO(text.encode("utf-8")) - doc_stream = DocumentStream(name=name, stream=bytes_io) + doc_stream = DocumentStream(name=doc_name, stream=bytes_io) converter = DoclingDocConverter() result = converter.convert(doc_stream) return result.document @staticmethod async def text_to_docling_document( - text: str, name: str = "content.md" + text: str, name: str = "content.md", format: str = "md" ) -> "DoclingDocument": - """Convert text to DoclingDocument using docling's markdown parser. + """Convert text to DoclingDocument using docling's parser. Args: text: The text content to convert. name: The name to use for the document. + format: The format of the text content ("md" or "html"). Defaults to "md". + This determines which parser docling uses to interpret the content. Returns: DoclingDocument representation of the text. Raises: - ValueError: If the conversion fails. + ValueError: If the conversion fails or format is unsupported. """ try: return await asyncio.to_thread( - TextFileHandler._sync_text_to_docling_document, text, name + TextFileHandler._sync_text_to_docling_document, text, name, format ) except Exception as e: raise ValueError(f"Failed to convert text to DoclingDocument: {e}") diff --git a/tests/test_client.py b/tests/test_client.py index 2a3ca4c3..eabff47a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -28,7 +28,8 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path): ) assert created_doc.id is not None - assert created_doc.content == document_text + # Content is stored as markdown export, check key text is preserved + assert "Jakarta" in created_doc.content assert created_doc.uri == test_uri assert created_doc.metadata == test_metadata @@ -36,14 +37,14 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path): retrieved_doc = await client.get_document_by_id(created_doc.id) assert retrieved_doc is not None assert retrieved_doc.id == created_doc.id - assert retrieved_doc.content == document_text + assert "Jakarta" in retrieved_doc.content assert retrieved_doc.uri == test_uri # Test get_document_by_uri retrieved_by_uri = await client.get_document_by_uri(test_uri) assert retrieved_by_uri is not None assert retrieved_by_uri.id == created_doc.id - assert retrieved_by_uri.content == document_text + assert "Jakarta" in retrieved_by_uri.content # Test get_document_by_uri with non-existent URI non_existent = await client.get_document_by_uri("file:///non/existent.txt") @@ -1380,3 +1381,52 @@ async def test_update_document_embeds_chunks_without_embeddings(temp_db_path): results = await client.search("Updated chunk", search_type="vector") assert len(results) > 0 assert results[0].content == "Updated chunk without embedding" + + +@pytest.mark.asyncio +async def test_client_create_document_with_html_format(temp_db_path): + """Test create_document with HTML format preserves document structure.""" + async with HaikuRAG(temp_db_path, create=True) as client: + html_content = """ +

Main Title

+

Introduction paragraph.

+

Section Header

+ + """ + + doc = await client.create_document( + content=html_content, + uri="test://html-doc", + format="html", + ) + + assert doc.id is not None + assert doc.docling_document_json is not None + + # Verify the DoclingDocument has proper structure + docling_doc = doc.get_docling_document() + assert docling_doc is not None + + items = list(docling_doc.iterate_items()) + labels = [str(getattr(item, "label", "")) for item, _ in items] + + # HTML format should preserve headers and list items + assert "title" in labels or "section_header" in labels + assert "list_item" in labels + + +@pytest.mark.asyncio +async def test_client_convert_with_html_format(temp_db_path): + """Test convert with HTML format.""" + async with HaikuRAG(temp_db_path, create=True) as client: + html_content = "

Title

Text

" + + docling_doc = await client.convert(html_content, format="html") + + items = list(docling_doc.iterate_items()) + labels = [str(getattr(item, "label", "")) for item, _ in items] + + assert "title" in labels diff --git a/tests/test_converters.py b/tests/test_converters.py index 9bf9e873..c8fd2f85 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -116,6 +116,73 @@ class TestConverterFactory: get_converter(config) +class TestTextToDoclingWithFormat: + """Tests for format parameter in text to DoclingDocument conversion.""" + + @pytest.mark.asyncio + async def test_html_format_preserves_structure(self): + """Test that HTML content parsed with html format preserves document structure.""" + html_content = """ +

Main Title

+

Introduction paragraph.

+

Section Header

+ + """ + config = AppConfig() + converter = DoclingLocalConverter(config) + + # With html format, should get proper structure + doc = await converter.convert_text( + html_content, name="content.html", format="html" + ) + + items = list(doc.iterate_items()) + labels = [str(getattr(item, "label", "")) for item, _ in items] + + assert "title" in labels or "section_header" in labels + assert "list_item" in labels + assert len(items) > 3 + + @pytest.mark.asyncio + async def test_md_format_is_default(self): + """Test that md format is used by default.""" + config = AppConfig() + converter = DoclingLocalConverter(config) + + # Plain text should work with default format + doc = await converter.convert_text("# Heading\n\nParagraph text.") + items = list(doc.iterate_items()) + + assert len(items) >= 2 + + @pytest.mark.asyncio + async def test_html_as_md_loses_structure(self): + """Test that HTML parsed as markdown loses semantic structure.""" + html_content = "

Title

Text

" + config = AppConfig() + converter = DoclingLocalConverter(config) + + # With md format (default), HTML tags are treated as text + doc = await converter.convert_text(html_content, format="md") + items = list(doc.iterate_items()) + + # Should still parse but with different structure + # (markdown parser will interpret some HTML) + assert len(items) >= 1 + + @pytest.mark.asyncio + async def test_invalid_format_raises_error(self): + """Test that invalid format raises ValueError.""" + config = AppConfig() + converter = DoclingLocalConverter(config) + + with pytest.raises(ValueError, match="Unsupported format"): + await converter.convert_text("content", format="invalid") + + class TestDoclingLocalConverter: """Tests for DoclingLocalConverter."""