Merge pull request #144 from ggozad/feat/additional-conversion-options

Additional options for document conversion with docling

CHANGELOG.md

@@ -1,10 +1,20 @@
 # Changelog
 ## [Unreleased]
 
+### Added
+
+- **Conversion Options**: Fine-grained control over document conversion for both local and remote converters
+  - New `conversion_options` config section in `ProcessingConfig`
+  - OCR settings: `do_ocr`, `force_ocr`, `ocr_lang` for controlling OCR behavior
+  - Table extraction: `do_table_structure`, `table_mode` (fast/accurate), `table_cell_matching`
+  - Image settings: `images_scale` to control image resolution
+  - Options work identically with both `docling-local` and `docling-serve` converters
+
 ### Changed
 
 - Increase reranking candidate retrieval multiplier from 3x to 10x for improved result quality
 - **Docker Images**: Main `haiku.rag` image no longer automatically built and published
+- **Conversion Options**: Removed the legacy `pdf_backend` setting; docling now chooses the optimal backend automatically
 
 ## [0.17.0] - 2025-11-17
 

docs/configuration.md

@@ -104,6 +104,14 @@ processing:
   chunking_merge_peers: true
   chunking_use_markdown_tables: false
   markdown_preprocessor: ""
+  conversion_options:
+    do_ocr: true
+    force_ocr: false
+    ocr_lang: []
+    do_table_structure: true
+    table_mode: accurate
+    table_cell_matching: true
+    images_scale: 2.0
 
 providers:
   ollama:
@@ -236,8 +244,64 @@ processing:
   chunking_tokenizer: "Qwen/Qwen3-Embedding-0.6B"  # HuggingFace model for tokenization
   chunking_merge_peers: true                 # Merge undersized successive chunks
   chunking_use_markdown_tables: false        # Use markdown tables vs narrative format
+
+  # Conversion options (works with both local and remote converters)
+  conversion_options:
+    # OCR settings
+    do_ocr: true                             # Enable OCR for bitmap content
+    force_ocr: false                         # Replace existing text with OCR
+    ocr_lang: []                             # OCR languages (e.g., ["en", "fr", "de"])
+
+    # Table extraction
+    do_table_structure: true                 # Extract table structure
+    table_mode: accurate                     # fast or accurate
+    table_cell_matching: true                # Match table cells back to PDF cells
+
+    # Image settings
+    images_scale: 2.0                        # Image scale factor
 ```
 
+### Conversion Options
+
+The `conversion_options` section allows fine-grained control over document conversion. These options work with both `docling-local` and `docling-serve` converters.
+
+#### OCR Settings
+
+```yaml
+conversion_options:
+  do_ocr: true          # Enable OCR for bitmap/scanned content
+  force_ocr: false      # Replace all text with OCR output
+  ocr_lang: []          # List of OCR languages, e.g., ["en", "fr", "de"]
+```
+
+- **do_ocr**: When `true`, applies OCR to images and scanned pages. Disable for faster processing if documents contain only native text.
+- **force_ocr**: When `true`, replaces existing text layers with OCR output. Useful for documents with poor text extraction.
+- **ocr_lang**: List of language codes for OCR. Empty list uses default language detection. Examples: `["en"]`, `["en", "fr", "de"]`.
+
+#### Table Extraction
+
+```yaml
+conversion_options:
+  do_table_structure: true    # Extract structured table data
+  table_mode: accurate        # fast or accurate
+  table_cell_matching: true   # Match cells back to PDF
+```
+
+- **do_table_structure**: When `true`, extracts table structure. Disable for faster processing if tables aren't important.
+- **table_mode**:
+  - `accurate`: Better table structure recognition (slower)
+  - `fast`: Faster processing with simpler table detection
+- **table_cell_matching**: When `true`, matches detected table cells back to PDF cells. Disable if tables have merged cells across columns.
+
+#### Image Settings
+
+```yaml
+conversion_options:
+  images_scale: 2.0  # Image resolution scale factor
+```
+
+- **images_scale**: Scale factor for extracted images. Higher values = better quality but larger size. Typical range: 1.0-3.0.
+
 ### Local vs Remote Processing
 
 **Local processing** (default):
@@ -266,6 +330,8 @@ providers:
     timeout: 300              # Request timeout in seconds
 ```
 
+Conversion options work identically for both local and remote processing.
+
 ### Chunking Strategies
 
 **Hybrid chunking** (default):

haiku_rag_slim/haiku/rag/config/__init__.py

@@ -8,6 +8,7 @@ from haiku.rag.config.loader import (
 from haiku.rag.config.models import (
     AGUIConfig,
     AppConfig,
+    ConversionOptions,
     EmbeddingsConfig,
     LanceDBConfig,
     MonitorConfig,
@@ -25,6 +26,7 @@ __all__ = [
     "Config",
     "AGUIConfig",
     "AppConfig",
+    "ConversionOptions",
     "StorageConfig",
     "MonitorConfig",
     "LanceDBConfig",

haiku_rag_slim/haiku/rag/config/models.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
@@ -50,6 +51,23 @@ class ResearchConfig(BaseModel):
     max_concurrency: int = 1
 
 
+class ConversionOptions(BaseModel):
+    """Options for document conversion."""
+
+    # OCR options
+    do_ocr: bool = True
+    force_ocr: bool = False
+    ocr_lang: list[str] = []
+
+    # Table options
+    do_table_structure: bool = True
+    table_mode: Literal["fast", "accurate"] = "accurate"
+    table_cell_matching: bool = True
+
+    # Image options
+    images_scale: float = 2.0
+
+
 class ProcessingConfig(BaseModel):
     chunk_size: int = 256
     context_chunk_radius: int = 0
@@ -60,6 +78,7 @@ class ProcessingConfig(BaseModel):
     chunking_tokenizer: str = "Qwen/Qwen3-Embedding-0.6B"
     chunking_merge_peers: bool = True
     chunking_use_markdown_tables: bool = False
+    conversion_options: ConversionOptions = Field(default_factory=ConversionOptions)
 
 
 class OllamaConfig(BaseModel):

haiku_rag_slim/haiku/rag/converters/__init__.py

@@ -21,7 +21,7 @@ def get_converter(config: AppConfig = Config) -> DocumentConverter:
     if config.processing.converter == "docling-local":
         from haiku.rag.converters.docling_local import DoclingLocalConverter
 
-        return DoclingLocalConverter()
+        return DoclingLocalConverter(config)
 
     if config.processing.converter == "docling-serve":
         from haiku.rag.converters.docling_serve import DoclingServeConverter

haiku_rag_slim/haiku/rag/converters/docling_local.py

@@ -1,8 +1,9 @@
 """Local docling converter implementation."""
 
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, cast
 
+from haiku.rag.config import AppConfig
 from haiku.rag.converters.base import DocumentConverter
 from haiku.rag.converters.text_utils import TextFileHandler
 
@@ -39,6 +40,14 @@ class DoclingLocalConverter(DocumentConverter):
         ".webp",
     ]
 
+    def __init__(self, config: AppConfig):
+        """Initialize the converter with configuration.
+
+        Args:
+            config: Application configuration containing conversion options.
+        """
+        self.config = config
+
     @property
     def supported_extensions(self) -> list[str]:
         """Return list of file extensions supported by this converter."""
@@ -56,14 +65,61 @@ class DoclingLocalConverter(DocumentConverter):
         Raises:
             ValueError: If the file cannot be converted.
         """
-        from docling.document_converter import DocumentConverter as DoclingDocConverter
+        from docling.backend.docling_parse_backend import DoclingParseDocumentBackend
+        from docling.datamodel.base_models import InputFormat
+        from docling.datamodel.pipeline_options import (
+            OcrOptions,
+            PdfPipelineOptions,
+            TableFormerMode,
+            TableStructureOptions,
+        )
+        from docling.document_converter import (
+            DocumentConverter as DoclingDocConverter,
+        )
+        from docling.document_converter import (
+            FormatOption,
+            PdfFormatOption,
+        )
 
         try:
             file_extension = path.suffix.lower()
 
             if file_extension in self.docling_extensions:
+                # Get conversion options from config
+                opts = self.config.processing.conversion_options
+
+                # Build pipeline options for PDF conversion
+                pipeline_options = PdfPipelineOptions(
+                    do_ocr=opts.do_ocr,
+                    do_table_structure=opts.do_table_structure,
+                    images_scale=opts.images_scale,
+                    table_structure_options=TableStructureOptions(
+                        do_cell_matching=opts.table_cell_matching,
+                        mode=(
+                            TableFormerMode.FAST
+                            if opts.table_mode == "fast"
+                            else TableFormerMode.ACCURATE
+                        ),
+                    ),
+                    ocr_options=OcrOptions(
+                        force_full_page_ocr=opts.force_ocr,
+                        lang=opts.ocr_lang if opts.ocr_lang else [],
+                    ),
+                )
+
+                # Create format options for PDF
+                format_options = cast(
+                    dict[InputFormat, FormatOption],
+                    {
+                        InputFormat.PDF: PdfFormatOption(
+                            pipeline_options=pipeline_options,
+                            backend=DoclingParseDocumentBackend,
+                        )
+                    },
+                )
+
                 # Use docling for complex document formats
-                converter = DoclingDocConverter()
+                converter = DoclingDocConverter(format_options=format_options)
                 result = converter.convert(path)
                 return result.document
             elif file_extension in TextFileHandler.text_extensions:

haiku_rag_slim/haiku/rag/converters/docling_serve.py

@@ -78,9 +78,27 @@ class DoclingServeConverter(DocumentConverter):
 
         try:
             url = f"{self.base_url}/v1/convert/file"
-            data = {"to_formats": ["json"]}
+            opts = self.config.processing.conversion_options
-            headers = {}
 
+            # Build data dict with conversion options
+            data = {
+                "to_formats": ["json"],
+                # OCR options
+                "do_ocr": opts.do_ocr,
+                "force_ocr": opts.force_ocr,
+                # Table options
+                "do_table_structure": opts.do_table_structure,
+                "table_mode": opts.table_mode,
+                "table_cell_matching": opts.table_cell_matching,
+                # Image options
+                "images_scale": opts.images_scale,
+            }
+
+            # Add OCR language if specified
+            if opts.ocr_lang:
+                data["ocr_lang"] = opts.ocr_lang
+
+            headers = {}
             if self.api_key:
                 headers["X-Api-Key"] = self.api_key
 

tests/test_converters.py

@@ -118,26 +118,33 @@ class TestConverterFactory:
 class TestDoclingLocalConverter:
     """Tests for DoclingLocalConverter."""
 
-    def test_supported_extensions(self):
+    @pytest.fixture
+    def config(self):
+        """Create test configuration."""
+        return AppConfig()
+
+    @pytest.fixture
+    def converter(self, config):
+        """Create DoclingLocalConverter instance."""
+        return DoclingLocalConverter(config)
+
+    def test_supported_extensions(self, converter):
         """Test that converter reports correct supported extensions."""
-        converter = DoclingLocalConverter()
         extensions = converter.supported_extensions
         assert ".pdf" in extensions
         assert ".docx" in extensions
         assert ".py" in extensions
         assert ".txt" in extensions
 
-    def test_convert_text(self):
+    def test_convert_text(self, converter):
         """Test converting text to DoclingDocument."""
-        converter = DoclingLocalConverter()
         doc = converter.convert_text("# Test\n\nContent here", name="test.md")
         assert isinstance(doc, DoclingDocument)
         assert doc.name == "test"
 
-    def test_convert_code_file(self):
+    def test_convert_code_file(self, converter):
         """Test that code files are wrapped in code blocks."""
         python_code = "def hello():\n    print('Hello')"
-        converter = DoclingLocalConverter()
 
         with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as f:
             f.write(python_code)
@@ -149,6 +156,17 @@ class TestDoclingLocalConverter:
             assert "```" in result
             assert "def hello():" in result
 
+    def test_conversion_options_applied_to_local_converter(self, config):
+        """Test that conversion options are applied to local docling converter."""
+        config.processing.conversion_options.do_ocr = False
+        config.processing.conversion_options.table_mode = "fast"
+        config.processing.conversion_options.images_scale = 3.0
+        converter = DoclingLocalConverter(config)
+
+        assert converter.config.processing.conversion_options.do_ocr is False
+        assert converter.config.processing.conversion_options.table_mode == "fast"
+        assert converter.config.processing.conversion_options.images_scale == 3.0
+
 
 class TestDoclingServeConverter:
     """Tests for DoclingServeConverter (mocked)."""
@@ -216,6 +234,40 @@ class TestDoclingServeConverter:
         assert "headers" in call_kwargs
         assert call_kwargs["headers"]["X-Api-Key"] == "test-key"
 
+    @patch("haiku.rag.converters.docling_serve.requests.post")
+    def test_conversion_options_passed_to_api(self, mock_post, config):
+        """Test that conversion options are passed to docling-serve API."""
+        config.processing.conversion_options.do_ocr = False
+        config.processing.conversion_options.force_ocr = True
+        config.processing.conversion_options.ocr_lang = ["en", "fr"]
+        config.processing.conversion_options.table_mode = "fast"
+        config.processing.conversion_options.table_cell_matching = False
+        config.processing.conversion_options.do_table_structure = False
+        config.processing.conversion_options.images_scale = 3.0
+        converter = DoclingServeConverter(config)
+
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "status": "success",
+            "document": {"json_content": create_mock_docling_document_json("test")},
+        }
+        mock_post.return_value = mock_response
+
+        converter.convert_text("# Test")
+
+        call_kwargs = mock_post.call_args.kwargs
+        assert "data" in call_kwargs
+        data = call_kwargs["data"]
+        assert data["do_ocr"] is False
+        assert data["force_ocr"] is True
+        assert data["ocr_lang"] == ["en", "fr"]
+        assert "pdf_backend" not in data
+        assert data["table_mode"] == "fast"
+        assert data["table_cell_matching"] is False
+        assert data["do_table_structure"] is False
+        assert data["images_scale"] == 3.0
+
     @patch("haiku.rag.converters.docling_serve.requests.post")
     def test_convert_text_connection_error(self, mock_post, converter):
         """Test handling of connection errors."""