Merge pull request #194 from ggozad/fix/text-handling-format

Add 'plain' format and fallback for text conversion

CHANGELOG.md

@@ -7,6 +7,10 @@
   - Extracts unique documents from validation set context paragraphs
   - Uses MAP for retrieval evaluation (multiple supporting documents per question)
   - Run with `evaluations hotpotqa`
+- **Plain Text Format**: Added `format="plain"` for text conversion
+  - Use when content is plain text without markdown/HTML structure
+  - Falls back gracefully when docling cannot detect markdown format in content
+  - Supported in `create_document()`, `convert()`, and all converter classes
 
 ### Changed
 

docs/python.md

@@ -50,6 +50,7 @@ The `format` parameter controls how text content is parsed:
 
 - `"md"` (default) - Parse as Markdown
 - `"html"` - Parse as HTML, preserving semantic structure (headings, lists, tables)
+- `"plain"` - Plain text, no parsing (creates a simple text document)
 
 !!! note
     The document's `content` field stores the markdown export of the parsed document for consistent display. The original input is preserved in the `docling_document_json` field.

haiku_rag_slim/haiku/rag/client.py

@@ -115,7 +115,8 @@ class HaikuRAG:
                 - Path: Local file path to convert
                 - str (URL): HTTP/HTTPS URL to download and convert
                 - str (text): Raw text content to convert
-            format: The format of text content ("md" or "html"). Defaults to "md".
+            format: The format of text content ("md", "html", or "plain").
+                Defaults to "md". Use "plain" for plain text without parsing.
                 Only used when source is raw text (not a file path or URL).
                 Files and URLs determine format from extension/content-type.
 
@@ -350,8 +351,8 @@ class HaikuRAG:
             uri: Optional URI identifier for the document.
             title: Optional title for the document.
             metadata: Optional metadata dictionary.
-            format: The format of the content ("md" or "html"). Defaults to "md".
+            format: The format of the content ("md", "html", or "plain").
-                This determines which parser is used to interpret the content structure.
+                Defaults to "md". Use "plain" for plain text without parsing.
 
         Returns:
             The created Document instance.

haiku_rag_slim/haiku/rag/converters/base.py

@@ -40,7 +40,7 @@ class DocumentConverter(ABC):
         """
         pass
 
-    SUPPORTED_FORMATS = ("md", "html")
+    SUPPORTED_FORMATS = ("md", "html", "plain")
 
     @abstractmethod
     async def convert_text(
@@ -51,8 +51,8 @@ class DocumentConverter(ABC):
         Args:
             text: The text content to convert.
             name: The name to use for the document (defaults to "content.md").
-            format: The format of the text content ("md" or "html"). Defaults to "md".
+            format: The format of the text content ("md", "html", or "plain").
-                This determines which parser docling uses to interpret the content.
+                Defaults to "md". Use "plain" for plain text without parsing.
 
         Returns:
             DoclingDocument representation of the text.

haiku_rag_slim/haiku/rag/converters/docling_local.py

@@ -145,8 +145,8 @@ class DoclingLocalConverter(DocumentConverter):
         Args:
             text: The text content to convert.
             name: The name to use for the document (defaults to "content.md").
-            format: The format of the text content ("md" or "html"). Defaults to "md".
+            format: The format of the text content ("md", "html", or "plain").
-                This determines which parser docling uses to interpret the content.
+                Defaults to "md". Use "plain" for plain text without parsing.
 
         Returns:
             DoclingDocument representation of the text.

haiku_rag_slim/haiku/rag/converters/docling_serve.py

@@ -177,7 +177,7 @@ class DoclingServeConverter(DocumentConverter):
         files = {"files": (path.name, file_content, "application/octet-stream")}
         return await self._make_request(files, path.name)
 
-    SUPPORTED_FORMATS = ("md", "html")
+    SUPPORTED_FORMATS = ("md", "html", "plain")
 
     async def convert_text(
         self, text: str, name: str = "content.md", format: str = "md"
@@ -189,8 +189,8 @@ class DoclingServeConverter(DocumentConverter):
         Args:
             text: The text content to convert.
             name: The name to use for the document (defaults to "content.md").
-            format: The format of the text content ("md" or "html"). Defaults to "md".
+            format: The format of the text content ("md", "html", or "plain").
-                This determines which parser docling uses to interpret the content.
+                Defaults to "md". Use "plain" for plain text without parsing.
 
         Returns:
             DoclingDocument representation of the text.
@@ -198,6 +198,8 @@ class DoclingServeConverter(DocumentConverter):
         Raises:
             ValueError: If the text cannot be converted or format is unsupported.
         """
+        from haiku.rag.converters.text_utils import TextFileHandler
+
         if format not in self.SUPPORTED_FORMATS:
             raise ValueError(
                 f"Unsupported format: {format}. "
@@ -206,6 +208,11 @@ class DoclingServeConverter(DocumentConverter):
 
         # Derive document name from format to tell docling which parser to use
         doc_name = f"content.{format}" if name == "content.md" else name
+
+        # Plain text doesn't need remote parsing - create document directly
+        if format == "plain":
+            return TextFileHandler._create_simple_docling_document(text, doc_name)
+
         mime_type = "text/html" if format == "html" else "text/markdown"
 
         text_bytes = text.encode("utf-8")

haiku_rag_slim/haiku/rag/converters/text_utils.py

@@ -89,7 +89,22 @@ class TextFileHandler:
             return f"```{language}\n{content}\n```"
         return content
 
-    SUPPORTED_FORMATS = ("md", "html")
+    SUPPORTED_FORMATS = ("md", "html", "plain")
+
+    @staticmethod
+    def _create_simple_docling_document(text: str, name: str) -> "DoclingDocument":
+        """Create a simple DoclingDocument directly from text.
+
+        Used as fallback when docling's format detection fails for plain text
+        that doesn't contain markdown syntax.
+        """
+        from docling_core.types.doc.document import DoclingDocument
+        from docling_core.types.doc.labels import DocItemLabel
+
+        doc_name = name.rsplit(".", 1)[0] if "." in name else name
+        doc = DoclingDocument(name=doc_name)
+        doc.add_text(label=DocItemLabel.TEXT, text=text)
+        return doc
 
     @staticmethod
     def _sync_text_to_docling_document(
@@ -97,6 +112,7 @@ class TextFileHandler:
     ) -> "DoclingDocument":
         """Synchronous implementation of text to DoclingDocument conversion."""
         from docling.document_converter import DocumentConverter as DoclingDocConverter
+        from docling.exceptions import ConversionError
         from docling_core.types.io import DocumentStream
 
         if format not in TextFileHandler.SUPPORTED_FORMATS:
@@ -108,11 +124,20 @@ class TextFileHandler:
         # Derive document name from format to tell docling which parser to use
         doc_name = f"content.{format}" if name == "content.md" else name
 
+        # Plain text doesn't need parsing - create document directly
+        if format == "plain":
+            return TextFileHandler._create_simple_docling_document(text, doc_name)
+
         bytes_io = BytesIO(text.encode("utf-8"))
         doc_stream = DocumentStream(name=doc_name, stream=bytes_io)
         converter = DoclingDocConverter()
-        result = converter.convert(doc_stream)
+        try:
-        return result.document
+            result = converter.convert(doc_stream)
+            return result.document
+        except ConversionError:
+            # Docling's format detection fails for plain text without markdown syntax.
+            # Fall back to creating a simple document directly.
+            return TextFileHandler._create_simple_docling_document(text, doc_name)
 
     @staticmethod
     async def text_to_docling_document(
@@ -123,8 +148,8 @@ class TextFileHandler:
         Args:
             text: The text content to convert.
             name: The name to use for the document.
-            format: The format of the text content ("md" or "html"). Defaults to "md".
+            format: The format of the text content ("md", "html", or "plain").
-                This determines which parser docling uses to interpret the content.
+                Defaults to "md". Use "plain" for plain text without parsing.
 
         Returns:
             DoclingDocument representation of the text.

tests/test_converters.py

@@ -182,6 +182,43 @@ class TestTextToDoclingWithFormat:
         with pytest.raises(ValueError, match="Unsupported format"):
             await converter.convert_text("content", format="invalid")
 
+    @pytest.mark.asyncio
+    async def test_plain_format(self):
+        """Test that format='plain' creates DoclingDocument directly."""
+        config = AppConfig()
+        converter = DoclingLocalConverter(config)
+
+        plain_text = (
+            "MZ Wallace is an American company which designs, manufactures "
+            "and markets handbags and fashion accessories."
+        )
+        doc = await converter.convert_text(plain_text, format="plain")
+        assert doc is not None
+        exported = doc.export_to_markdown()
+        assert "MZ Wallace" in exported
+
+    @pytest.mark.asyncio
+    async def test_plain_text_without_markdown_syntax_fallback(self):
+        """Test that plain text without markdown syntax falls back gracefully.
+
+        Docling's format detection fails for plain text that doesn't contain
+        markdown syntax (headers, lists, etc.). The converter should fall back
+        to creating a simple DoclingDocument directly.
+        """
+        config = AppConfig()
+        converter = DoclingLocalConverter(config)
+
+        # Plain text without any markdown syntax
+        plain_text = (
+            "MZ Wallace is an American company which designs, manufactures "
+            "and markets handbags and fashion accessories. The company was "
+            "founded in 1999 by Monica Zwirner and Lucy Wallace Eustice."
+        )
+        doc = await converter.convert_text(plain_text, format="md")
+        assert doc is not None
+        exported = doc.export_to_markdown()
+        assert "MZ Wallace" in exported
+
 
 class TestDoclingLocalConverter:
     """Tests for DoclingLocalConverter."""