From 3149552a6189ffb0a3e924580d58596e9b1b4ee5 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 12 Dec 2025 17:43:09 +0200 Subject: [PATCH] Add 'plain' format and fallback for text conversion --- CHANGELOG.md | 4 ++ docs/python.md | 1 + haiku_rag_slim/haiku/rag/client.py | 7 ++-- haiku_rag_slim/haiku/rag/converters/base.py | 6 +-- .../haiku/rag/converters/docling_local.py | 4 +- .../haiku/rag/converters/docling_serve.py | 13 +++++-- .../haiku/rag/converters/text_utils.py | 35 +++++++++++++++--- tests/test_converters.py | 37 +++++++++++++++++++ 8 files changed, 91 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df7d0b9..bb0a1ba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ - Extracts unique documents from validation set context paragraphs - Uses MAP for retrieval evaluation (multiple supporting documents per question) - Run with `evaluations hotpotqa` +- **Plain Text Format**: Added `format="plain"` for text conversion + - Use when content is plain text without markdown/HTML structure + - Falls back gracefully when docling cannot detect markdown format in content + - Supported in `create_document()`, `convert()`, and all converter classes ### Changed diff --git a/docs/python.md b/docs/python.md index 12e2048b..f459ae1b 100644 --- a/docs/python.md +++ b/docs/python.md @@ -50,6 +50,7 @@ The `format` parameter controls how text content is parsed: - `"md"` (default) - Parse as Markdown - `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables) +- `"plain"` - Plain text, no parsing (creates a simple text document) !!! note The document's `content` field stores the markdown export of the parsed document for consistent display. The original input is preserved in the `docling_document_json` field. diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index 53ceff30..ee533c3e 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -115,7 +115,8 @@ class HaikuRAG: - Path: Local file path to convert - str (URL): HTTP/HTTPS URL to download and convert - str (text): Raw text content to convert - format: The format of text content ("md" or "html"). Defaults to "md". + format: The format of text content ("md", "html", or "plain"). + Defaults to "md". Use "plain" for plain text without parsing. Only used when source is raw text (not a file path or URL). Files and URLs determine format from extension/content-type. @@ -350,8 +351,8 @@ class HaikuRAG: uri: Optional URI identifier for the document. title: Optional title for the document. metadata: Optional metadata dictionary. - format: The format of the content ("md" or "html"). Defaults to "md". - This determines which parser is used to interpret the content structure. + format: The format of the content ("md", "html", or "plain"). + Defaults to "md". Use "plain" for plain text without parsing. Returns: The created Document instance. diff --git a/haiku_rag_slim/haiku/rag/converters/base.py b/haiku_rag_slim/haiku/rag/converters/base.py index 1a6b883f..e1ccbc7a 100644 --- a/haiku_rag_slim/haiku/rag/converters/base.py +++ b/haiku_rag_slim/haiku/rag/converters/base.py @@ -40,7 +40,7 @@ class DocumentConverter(ABC): """ pass - SUPPORTED_FORMATS = ("md", "html") + SUPPORTED_FORMATS = ("md", "html", "plain") @abstractmethod async def convert_text( @@ -51,8 +51,8 @@ class DocumentConverter(ABC): Args: text: The text content to convert. name: The name to use for the document (defaults to "content.md"). - format: The format of the text content ("md" or "html"). Defaults to "md". - This determines which parser docling uses to interpret the content. + format: The format of the text content ("md", "html", or "plain"). + Defaults to "md". Use "plain" for plain text without parsing. Returns: DoclingDocument representation of the text. diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index 594090a8..8c105912 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -145,8 +145,8 @@ class DoclingLocalConverter(DocumentConverter): Args: text: The text content to convert. name: The name to use for the document (defaults to "content.md"). - format: The format of the text content ("md" or "html"). Defaults to "md". - This determines which parser docling uses to interpret the content. + format: The format of the text content ("md", "html", or "plain"). + Defaults to "md". Use "plain" for plain text without parsing. Returns: DoclingDocument representation of the text. diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py index 4cdb8257..46cf570b 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py @@ -177,7 +177,7 @@ class DoclingServeConverter(DocumentConverter): files = {"files": (path.name, file_content, "application/octet-stream")} return await self._make_request(files, path.name) - SUPPORTED_FORMATS = ("md", "html") + SUPPORTED_FORMATS = ("md", "html", "plain") async def convert_text( self, text: str, name: str = "content.md", format: str = "md" @@ -189,8 +189,8 @@ class DoclingServeConverter(DocumentConverter): Args: text: The text content to convert. name: The name to use for the document (defaults to "content.md"). - format: The format of the text content ("md" or "html"). Defaults to "md". - This determines which parser docling uses to interpret the content. + format: The format of the text content ("md", "html", or "plain"). + Defaults to "md". Use "plain" for plain text without parsing. Returns: DoclingDocument representation of the text. @@ -198,6 +198,8 @@ class DoclingServeConverter(DocumentConverter): Raises: ValueError: If the text cannot be converted or format is unsupported. """ + from haiku.rag.converters.text_utils import TextFileHandler + if format not in self.SUPPORTED_FORMATS: raise ValueError( f"Unsupported format: {format}. " @@ -206,6 +208,11 @@ class DoclingServeConverter(DocumentConverter): # Derive document name from format to tell docling which parser to use doc_name = f"content.{format}" if name == "content.md" else name + + # Plain text doesn't need remote parsing - create document directly + if format == "plain": + return TextFileHandler._create_simple_docling_document(text, doc_name) + mime_type = "text/html" if format == "html" else "text/markdown" text_bytes = text.encode("utf-8") diff --git a/haiku_rag_slim/haiku/rag/converters/text_utils.py b/haiku_rag_slim/haiku/rag/converters/text_utils.py index 36263155..64994e31 100644 --- a/haiku_rag_slim/haiku/rag/converters/text_utils.py +++ b/haiku_rag_slim/haiku/rag/converters/text_utils.py @@ -89,7 +89,22 @@ class TextFileHandler: return f"```{language}\n{content}\n```" return content - SUPPORTED_FORMATS = ("md", "html") + SUPPORTED_FORMATS = ("md", "html", "plain") + + @staticmethod + def _create_simple_docling_document(text: str, name: str) -> "DoclingDocument": + """Create a simple DoclingDocument directly from text. + + Used as fallback when docling's format detection fails for plain text + that doesn't contain markdown syntax. + """ + from docling_core.types.doc.document import DoclingDocument + from docling_core.types.doc.labels import DocItemLabel + + doc_name = name.rsplit(".", 1)[0] if "." in name else name + doc = DoclingDocument(name=doc_name) + doc.add_text(label=DocItemLabel.TEXT, text=text) + return doc @staticmethod def _sync_text_to_docling_document( @@ -97,6 +112,7 @@ class TextFileHandler: ) -> "DoclingDocument": """Synchronous implementation of text to DoclingDocument conversion.""" from docling.document_converter import DocumentConverter as DoclingDocConverter + from docling.exceptions import ConversionError from docling_core.types.io import DocumentStream if format not in TextFileHandler.SUPPORTED_FORMATS: @@ -108,11 +124,20 @@ class TextFileHandler: # Derive document name from format to tell docling which parser to use doc_name = f"content.{format}" if name == "content.md" else name + # Plain text doesn't need parsing - create document directly + if format == "plain": + return TextFileHandler._create_simple_docling_document(text, doc_name) + bytes_io = BytesIO(text.encode("utf-8")) doc_stream = DocumentStream(name=doc_name, stream=bytes_io) converter = DoclingDocConverter() - result = converter.convert(doc_stream) - return result.document + try: + result = converter.convert(doc_stream) + return result.document + except ConversionError: + # Docling's format detection fails for plain text without markdown syntax. + # Fall back to creating a simple document directly. + return TextFileHandler._create_simple_docling_document(text, doc_name) @staticmethod async def text_to_docling_document( @@ -123,8 +148,8 @@ class TextFileHandler: Args: text: The text content to convert. name: The name to use for the document. - format: The format of the text content ("md" or "html"). Defaults to "md". - This determines which parser docling uses to interpret the content. + format: The format of the text content ("md", "html", or "plain"). + Defaults to "md". Use "plain" for plain text without parsing. Returns: DoclingDocument representation of the text. diff --git a/tests/test_converters.py b/tests/test_converters.py index 595831bb..0ec18652 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -182,6 +182,43 @@ class TestTextToDoclingWithFormat: with pytest.raises(ValueError, match="Unsupported format"): await converter.convert_text("content", format="invalid") + @pytest.mark.asyncio + async def test_plain_format(self): + """Test that format='plain' creates DoclingDocument directly.""" + config = AppConfig() + converter = DoclingLocalConverter(config) + + plain_text = ( + "MZ Wallace is an American company which designs, manufactures " + "and markets handbags and fashion accessories." + ) + doc = await converter.convert_text(plain_text, format="plain") + assert doc is not None + exported = doc.export_to_markdown() + assert "MZ Wallace" in exported + + @pytest.mark.asyncio + async def test_plain_text_without_markdown_syntax_fallback(self): + """Test that plain text without markdown syntax falls back gracefully. + + Docling's format detection fails for plain text that doesn't contain + markdown syntax (headers, lists, etc.). The converter should fall back + to creating a simple DoclingDocument directly. + """ + config = AppConfig() + converter = DoclingLocalConverter(config) + + # Plain text without any markdown syntax + plain_text = ( + "MZ Wallace is an American company which designs, manufactures " + "and markets handbags and fashion accessories. The company was " + "founded in 1999 by Monica Zwirner and Lucy Wallace Eustice." + ) + doc = await converter.convert_text(plain_text, format="md") + assert doc is not None + exported = doc.export_to_markdown() + assert "MZ Wallace" in exported + class TestDoclingLocalConverter: """Tests for DoclingLocalConverter."""