diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a19b905..5f656b02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,20 @@ # Changelog ## [Unreleased] +### Added + +- **Conversion Options**: Fine-grained control over document conversion for both local and remote converters + - New `conversion_options` config section in `ProcessingConfig` + - OCR settings: `do_ocr`, `force_ocr`, `ocr_lang` for controlling OCR behavior + - Table extraction: `do_table_structure`, `table_mode` (fast/accurate), `table_cell_matching` + - Image settings: `images_scale` to control image resolution + - Options work identically with both `docling-local` and `docling-serve` converters + ### Changed - Increase reranking candidate retrieval multiplier from 3x to 10x for improved result quality - **Docker Images**: Main `haiku.rag` image no longer automatically built and published +- **Conversion Options**: Removed the legacy `pdf_backend` setting; docling now chooses the optimal backend automatically ## [0.17.0] - 2025-11-17 diff --git a/docs/configuration.md b/docs/configuration.md index 31952632..9e50ee4d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -104,6 +104,14 @@ processing: chunking_merge_peers: true chunking_use_markdown_tables: false markdown_preprocessor: "" + conversion_options: + do_ocr: true + force_ocr: false + ocr_lang: [] + do_table_structure: true + table_mode: accurate + table_cell_matching: true + images_scale: 2.0 providers: ollama: @@ -236,8 +244,64 @@ processing: chunking_tokenizer: "Qwen/Qwen3-Embedding-0.6B" # HuggingFace model for tokenization chunking_merge_peers: true # Merge undersized successive chunks chunking_use_markdown_tables: false # Use markdown tables vs narrative format + + # Conversion options (works with both local and remote converters) + conversion_options: + # OCR settings + do_ocr: true # Enable OCR for bitmap content + force_ocr: false # Replace existing text with OCR + ocr_lang: [] # OCR languages (e.g., ["en", "fr", "de"]) + + # Table extraction + do_table_structure: true # Extract table structure + table_mode: accurate # fast or accurate + table_cell_matching: true # Match table cells back to PDF cells + + # Image settings + images_scale: 2.0 # Image scale factor ``` +### Conversion Options + +The `conversion_options` section allows fine-grained control over document conversion. These options work with both `docling-local` and `docling-serve` converters. + +#### OCR Settings + +```yaml +conversion_options: + do_ocr: true # Enable OCR for bitmap/scanned content + force_ocr: false # Replace all text with OCR output + ocr_lang: [] # List of OCR languages, e.g., ["en", "fr", "de"] +``` + +- **do_ocr**: When `true`, applies OCR to images and scanned pages. Disable for faster processing if documents contain only native text. +- **force_ocr**: When `true`, replaces existing text layers with OCR output. Useful for documents with poor text extraction. +- **ocr_lang**: List of language codes for OCR. Empty list uses default language detection. Examples: `["en"]`, `["en", "fr", "de"]`. + +#### Table Extraction + +```yaml +conversion_options: + do_table_structure: true # Extract structured table data + table_mode: accurate # fast or accurate + table_cell_matching: true # Match cells back to PDF +``` + +- **do_table_structure**: When `true`, extracts table structure. Disable for faster processing if tables aren't important. +- **table_mode**: + - `accurate`: Better table structure recognition (slower) + - `fast`: Faster processing with simpler table detection +- **table_cell_matching**: When `true`, matches detected table cells back to PDF cells. Disable if tables have merged cells across columns. + +#### Image Settings + +```yaml +conversion_options: + images_scale: 2.0 # Image resolution scale factor +``` + +- **images_scale**: Scale factor for extracted images. Higher values = better quality but larger size. Typical range: 1.0-3.0. + ### Local vs Remote Processing **Local processing** (default): @@ -266,6 +330,8 @@ providers: timeout: 300 # Request timeout in seconds ``` +Conversion options work identically for both local and remote processing. + ### Chunking Strategies **Hybrid chunking** (default): diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index b6325451..40c18ee0 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -8,6 +8,7 @@ from haiku.rag.config.loader import ( from haiku.rag.config.models import ( AGUIConfig, AppConfig, + ConversionOptions, EmbeddingsConfig, LanceDBConfig, MonitorConfig, @@ -25,6 +26,7 @@ __all__ = [ "Config", "AGUIConfig", "AppConfig", + "ConversionOptions", "StorageConfig", "MonitorConfig", "LanceDBConfig", diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 47a0e599..f71fc03b 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Literal from pydantic import BaseModel, Field @@ -50,6 +51,23 @@ class ResearchConfig(BaseModel): max_concurrency: int = 1 +class ConversionOptions(BaseModel): + """Options for document conversion.""" + + # OCR options + do_ocr: bool = True + force_ocr: bool = False + ocr_lang: list[str] = [] + + # Table options + do_table_structure: bool = True + table_mode: Literal["fast", "accurate"] = "accurate" + table_cell_matching: bool = True + + # Image options + images_scale: float = 2.0 + + class ProcessingConfig(BaseModel): chunk_size: int = 256 context_chunk_radius: int = 0 @@ -60,6 +78,7 @@ class ProcessingConfig(BaseModel): chunking_tokenizer: str = "Qwen/Qwen3-Embedding-0.6B" chunking_merge_peers: bool = True chunking_use_markdown_tables: bool = False + conversion_options: ConversionOptions = Field(default_factory=ConversionOptions) class OllamaConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/converters/__init__.py b/haiku_rag_slim/haiku/rag/converters/__init__.py index f2911a88..1ec4c4d3 100644 --- a/haiku_rag_slim/haiku/rag/converters/__init__.py +++ b/haiku_rag_slim/haiku/rag/converters/__init__.py @@ -21,7 +21,7 @@ def get_converter(config: AppConfig = Config) -> DocumentConverter: if config.processing.converter == "docling-local": from haiku.rag.converters.docling_local import DoclingLocalConverter - return DoclingLocalConverter() + return DoclingLocalConverter(config) if config.processing.converter == "docling-serve": from haiku.rag.converters.docling_serve import DoclingServeConverter diff --git a/haiku_rag_slim/haiku/rag/converters/docling_local.py b/haiku_rag_slim/haiku/rag/converters/docling_local.py index e3e5fad9..fc0dc6f4 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_local.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_local.py @@ -1,8 +1,9 @@ """Local docling converter implementation.""" from pathlib import Path -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, ClassVar, cast +from haiku.rag.config import AppConfig from haiku.rag.converters.base import DocumentConverter from haiku.rag.converters.text_utils import TextFileHandler @@ -39,6 +40,14 @@ class DoclingLocalConverter(DocumentConverter): ".webp", ] + def __init__(self, config: AppConfig): + """Initialize the converter with configuration. + + Args: + config: Application configuration containing conversion options. + """ + self.config = config + @property def supported_extensions(self) -> list[str]: """Return list of file extensions supported by this converter.""" @@ -56,14 +65,61 @@ class DoclingLocalConverter(DocumentConverter): Raises: ValueError: If the file cannot be converted. """ - from docling.document_converter import DocumentConverter as DoclingDocConverter + from docling.backend.docling_parse_backend import DoclingParseDocumentBackend + from docling.datamodel.base_models import InputFormat + from docling.datamodel.pipeline_options import ( + OcrOptions, + PdfPipelineOptions, + TableFormerMode, + TableStructureOptions, + ) + from docling.document_converter import ( + DocumentConverter as DoclingDocConverter, + ) + from docling.document_converter import ( + FormatOption, + PdfFormatOption, + ) try: file_extension = path.suffix.lower() if file_extension in self.docling_extensions: + # Get conversion options from config + opts = self.config.processing.conversion_options + + # Build pipeline options for PDF conversion + pipeline_options = PdfPipelineOptions( + do_ocr=opts.do_ocr, + do_table_structure=opts.do_table_structure, + images_scale=opts.images_scale, + table_structure_options=TableStructureOptions( + do_cell_matching=opts.table_cell_matching, + mode=( + TableFormerMode.FAST + if opts.table_mode == "fast" + else TableFormerMode.ACCURATE + ), + ), + ocr_options=OcrOptions( + force_full_page_ocr=opts.force_ocr, + lang=opts.ocr_lang if opts.ocr_lang else [], + ), + ) + + # Create format options for PDF + format_options = cast( + dict[InputFormat, FormatOption], + { + InputFormat.PDF: PdfFormatOption( + pipeline_options=pipeline_options, + backend=DoclingParseDocumentBackend, + ) + }, + ) + # Use docling for complex document formats - converter = DoclingDocConverter() + converter = DoclingDocConverter(format_options=format_options) result = converter.convert(path) return result.document elif file_extension in TextFileHandler.text_extensions: diff --git a/haiku_rag_slim/haiku/rag/converters/docling_serve.py b/haiku_rag_slim/haiku/rag/converters/docling_serve.py index 4815dbe3..d5b9af50 100644 --- a/haiku_rag_slim/haiku/rag/converters/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/converters/docling_serve.py @@ -78,9 +78,27 @@ class DoclingServeConverter(DocumentConverter): try: url = f"{self.base_url}/v1/convert/file" - data = {"to_formats": ["json"]} - headers = {} + opts = self.config.processing.conversion_options + # Build data dict with conversion options + data = { + "to_formats": ["json"], + # OCR options + "do_ocr": opts.do_ocr, + "force_ocr": opts.force_ocr, + # Table options + "do_table_structure": opts.do_table_structure, + "table_mode": opts.table_mode, + "table_cell_matching": opts.table_cell_matching, + # Image options + "images_scale": opts.images_scale, + } + + # Add OCR language if specified + if opts.ocr_lang: + data["ocr_lang"] = opts.ocr_lang + + headers = {} if self.api_key: headers["X-Api-Key"] = self.api_key diff --git a/tests/test_converters.py b/tests/test_converters.py index b059071b..d32902ec 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -118,26 +118,33 @@ class TestConverterFactory: class TestDoclingLocalConverter: """Tests for DoclingLocalConverter.""" - def test_supported_extensions(self): + @pytest.fixture + def config(self): + """Create test configuration.""" + return AppConfig() + + @pytest.fixture + def converter(self, config): + """Create DoclingLocalConverter instance.""" + return DoclingLocalConverter(config) + + def test_supported_extensions(self, converter): """Test that converter reports correct supported extensions.""" - converter = DoclingLocalConverter() extensions = converter.supported_extensions assert ".pdf" in extensions assert ".docx" in extensions assert ".py" in extensions assert ".txt" in extensions - def test_convert_text(self): + def test_convert_text(self, converter): """Test converting text to DoclingDocument.""" - converter = DoclingLocalConverter() doc = converter.convert_text("# Test\n\nContent here", name="test.md") assert isinstance(doc, DoclingDocument) assert doc.name == "test" - def test_convert_code_file(self): + def test_convert_code_file(self, converter): """Test that code files are wrapped in code blocks.""" python_code = "def hello():\n print('Hello')" - converter = DoclingLocalConverter() with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as f: f.write(python_code) @@ -149,6 +156,17 @@ class TestDoclingLocalConverter: assert "```" in result assert "def hello():" in result + def test_conversion_options_applied_to_local_converter(self, config): + """Test that conversion options are applied to local docling converter.""" + config.processing.conversion_options.do_ocr = False + config.processing.conversion_options.table_mode = "fast" + config.processing.conversion_options.images_scale = 3.0 + converter = DoclingLocalConverter(config) + + assert converter.config.processing.conversion_options.do_ocr is False + assert converter.config.processing.conversion_options.table_mode == "fast" + assert converter.config.processing.conversion_options.images_scale == 3.0 + class TestDoclingServeConverter: """Tests for DoclingServeConverter (mocked).""" @@ -216,6 +234,40 @@ class TestDoclingServeConverter: assert "headers" in call_kwargs assert call_kwargs["headers"]["X-Api-Key"] == "test-key" + @patch("haiku.rag.converters.docling_serve.requests.post") + def test_conversion_options_passed_to_api(self, mock_post, config): + """Test that conversion options are passed to docling-serve API.""" + config.processing.conversion_options.do_ocr = False + config.processing.conversion_options.force_ocr = True + config.processing.conversion_options.ocr_lang = ["en", "fr"] + config.processing.conversion_options.table_mode = "fast" + config.processing.conversion_options.table_cell_matching = False + config.processing.conversion_options.do_table_structure = False + config.processing.conversion_options.images_scale = 3.0 + converter = DoclingServeConverter(config) + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "status": "success", + "document": {"json_content": create_mock_docling_document_json("test")}, + } + mock_post.return_value = mock_response + + converter.convert_text("# Test") + + call_kwargs = mock_post.call_args.kwargs + assert "data" in call_kwargs + data = call_kwargs["data"] + assert data["do_ocr"] is False + assert data["force_ocr"] is True + assert data["ocr_lang"] == ["en", "fr"] + assert "pdf_backend" not in data + assert data["table_mode"] == "fast" + assert data["table_cell_matching"] is False + assert data["do_table_structure"] is False + assert data["images_scale"] == 3.0 + @patch("haiku.rag.converters.docling_serve.requests.post") def test_convert_text_connection_error(self, mock_post, converter): """Test handling of connection errors."""