Merge pull request #26 from ggozad/feat/docling

Use docling for parsing documents

README.md

@@ -13,7 +13,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 - **Reranking**: Default search result reranking with MixedBread AI or Cohere
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
-- **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
+- **40+ file formats**: PDF, DOCX, HTML, Markdown, code files, URLs
 - **MCP server**: Expose as tools for AI assistants
 - **CLI & Python API**: Use from command line or Python
 

docs/index.md

@@ -10,7 +10,7 @@
 - **Reranking**: Optional result reranking with MixedBread AI or Cohere
 - **Question Answering**: Built-in QA agents using Ollama, OpenAI, or Anthropic.
 - **File monitoring**: Automatically index files when run as a server
-- **Extended file format support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a URL!
+- **Extended file format support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, code files and more. Or add a URL!
 - **MCP server**: Exposes functionality as MCP tools
 - **CLI commands**: Access all functionality from your terminal
 - **Python client**: Call `haiku.rag` from your own python applications

docs/server.md

@@ -35,7 +35,8 @@ The server can parse 40+ file formats including:
 - Microsoft Office (DOCX, XLSX, PPTX)
 - HTML and Markdown
 - Plain text files
-- Audio files
+- Code files (Python, JavaScript, etc.)
+- Images (processed via OCR)
 - And more...
 
 URLs are also supported for web content.

pyproject.toml

@@ -22,9 +22,9 @@ classifiers = [
 ]
 
 dependencies = [
+    "docling>=2.15.0",
     "fastmcp>=2.8.1",
     "httpx>=0.28.1",
-    "markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2",
     "mxbai-rerank>=0.1.6",
     "ollama>=0.5.1",
     "pydantic>=2.11.7",

src/haiku/rag/reader.py

@@ -1,32 +1,45 @@
 from pathlib import Path
 from typing import ClassVar
 
-from markitdown import MarkItDown
+from docling.document_converter import DocumentConverter
 
 
 class FileReader:
-    extensions: ClassVar[list[str]] = [
+    # Extensions supported by docling
+    docling_extensions: ClassVar[list[str]] = [
+        ".asciidoc",
+        ".bmp",
+        ".csv",
+        ".docx",
+        ".html",
+        ".xhtml",
+        ".jpeg",
+        ".jpg",
+        ".md",
+        ".pdf.png",
+        ".pptx",
+        ".tiff",
+        ".xlsx",
+        ".xml",
+        ".webp",
+    ]
+
+    # Plain text extensions that we'll read directly
+    text_extensions: ClassVar[list[str]] = [
         ".astro",
         ".c",
         ".cpp",
         ".css",
-        ".csv",
-        ".docx",
         ".go",
         ".h",
         ".hpp",
-        ".html",
         ".java",
         ".js",
         ".json",
         ".kt",
-        ".md",
         ".mdx",
         ".mjs",
-        ".mp3",
-        ".pdf",
         ".php",
-        ".pptx",
         ".py",
         ".rb",
         ".rs",
@@ -36,17 +49,61 @@ class FileReader:
         ".tsx",
         ".txt",
         ".vue",
-        ".wav",
-        ".xml",
-        ".xlsx",
         ".yaml",
         ".yml",
     ]
 
+    # Code file extensions with their markdown language identifiers for syntax highlighting
+    code_markdown_identifier: ClassVar[dict[str, str]] = {
+        ".astro": "astro",
+        ".c": "c",
+        ".cpp": "cpp",
+        ".css": "css",
+        ".go": "go",
+        ".h": "c",
+        ".hpp": "cpp",
+        ".java": "java",
+        ".js": "javascript",
+        ".json": "json",
+        ".kt": "kotlin",
+        ".mjs": "javascript",
+        ".php": "php",
+        ".py": "python",
+        ".rb": "ruby",
+        ".rs": "rust",
+        ".svelte": "svelte",
+        ".swift": "swift",
+        ".ts": "typescript",
+        ".tsx": "tsx",
+        ".vue": "vue",
+        ".yaml": "yaml",
+        ".yml": "yaml",
+    }
+
+    extensions: ClassVar[list[str]] = docling_extensions + text_extensions
+
     @staticmethod
     def parse_file(path: Path) -> str:
         try:
-            reader = MarkItDown()
-            return reader.convert(path).text_content
+            file_extension = path.suffix.lower()
+
+            if file_extension in FileReader.docling_extensions:
+                # Use docling for complex document formats
+                converter = DocumentConverter()
+                result = converter.convert(path)
+                return result.document.export_to_markdown()
+            elif file_extension in FileReader.text_extensions:
+                # Read plain text files directly
+                content = path.read_text(encoding="utf-8")
+
+                # Wrap code files (but not plain txt) in markdown code blocks for better presentation
+                if file_extension in FileReader.code_markdown_identifier:
+                    language = FileReader.code_markdown_identifier[file_extension]
+                    return f"```{language}\n{content}\n```"
+
+                return content
+            else:
+                # Fallback: try to read as text
+                return path.read_text(encoding="utf-8")
         except Exception:
             raise ValueError(f"Failed to parse file: {path}")