diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc2512b..d7bce307 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,10 @@ - HTML format preserves document structure (headings, lists, sections) in DoclingDocument - Enables proper parsing of HTML content that was previously treated as plain text - **Inspector Context Modal**: Press `c` in the inspector to view expanded context for the selected chunk +- **Auto-Vacuum Configuration**: New `storage.auto_vacuum` setting to control automatic vacuuming behavior + - When `true` (default), vacuum runs automatically after document create/update operations and rebuilds + - When `false`, vacuum only runs via explicit `haiku-rag vacuum` command + - Disabling can help avoid potential crashes in high-concurrency scenarios due to LanceDB race conditions ### Changed diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index 54bb6d67..42d76dcc 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -7,11 +7,16 @@ By default, `haiku.rag` uses a local LanceDB database: ```yaml storage: data_dir: /path/to/data # Empty = use default platform location + auto_vacuum: true # Enable automatic vacuuming after operations vacuum_retention_seconds: 86400 # Cleanup threshold in seconds ``` - **data_dir**: Directory for local database storage. When empty, uses platform-specific default locations -- **vacuum_retention_seconds**: When documents are added/updated, old table versions older than this are removed. Default: 86400 seconds (1 day, safe for concurrent connections). Set to 0 for aggressive cleanup (removes all old versions immediately) +- **auto_vacuum**: When enabled (default), automatically runs vacuum after document create/update operations and database rebuilds. Set to `false` to disable automatic vacuuming and rely on manual `haiku-rag vacuum` commands only. Disabling can help avoid potential crashes in high-concurrency scenarios +- **vacuum_retention_seconds**: When vacuum runs, old table versions older than this threshold are removed. Default: 86400 seconds (1 day). Set to 0 for aggressive cleanup (removes all old versions immediately) + +!!! warning "Vacuum Retention Threshold" + The `vacuum_retention_seconds` value should be larger than the typical time it takes to process and write a document. If a concurrent operation is in progress while vacuum runs, setting this value too low can cause race conditions where vacuum removes table versions that an in-flight operation still needs. The default of 86400 seconds (1 day) is conservative and safe for most use cases. ## Remote Storage diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index 33101e14..afdc87af 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -271,8 +271,9 @@ class HaikuRAG: # Batch create all chunks in a single operation await self.chunk_repository.create(chunks) - # Vacuum old versions in background (non-blocking) - asyncio.create_task(self.store.vacuum()) + # Vacuum old versions in background (non-blocking) if auto_vacuum enabled + if self._config.storage.auto_vacuum: + asyncio.create_task(self.store.vacuum()) return created_doc except Exception: @@ -322,8 +323,9 @@ class HaikuRAG: # Batch create all chunks in a single operation await self.chunk_repository.create(chunks) - # Vacuum old versions in background (non-blocking) - asyncio.create_task(self.store.vacuum()) + # Vacuum old versions in background (non-blocking) if auto_vacuum enabled + if self._config.storage.auto_vacuum: + asyncio.create_task(self.store.vacuum()) return updated_doc except Exception: @@ -1365,11 +1367,12 @@ class HaikuRAG: async for doc_id in self._rebuild_full(documents): yield doc_id - # Final maintenance - try: - await self.store.vacuum() - except Exception: - pass + # Final maintenance if auto_vacuum enabled + if self._config.storage.auto_vacuum: + try: + await self.store.vacuum() + except Exception: + pass async def _rebuild_embed_only( self, documents: list[Document] diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 36b58d11..7048de89 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -41,6 +41,7 @@ class EmbeddingModelConfig(BaseModel): class StorageConfig(BaseModel): data_dir: Path = Field(default_factory=get_default_data_dir) + auto_vacuum: bool = True vacuum_retention_seconds: int = 86400 diff --git a/tests/test_versioning.py b/tests/test_versioning.py index 6537d767..bc404a0a 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -174,3 +174,54 @@ async def test_vacuum_completes_before_context_exit(temp_db_path, monkeypatch): ) assert final_versions >= 1, "Should have at least one version remaining" store.close() + + +@pytest.mark.asyncio +async def test_auto_vacuum_disabled_skips_vacuum(temp_db_path, monkeypatch): + """Test that auto_vacuum=False prevents automatic vacuum after operations.""" + from haiku.rag.config import Config + + # Disable auto-vacuum + monkeypatch.setattr(Config.storage, "auto_vacuum", False) + + async with HaikuRAG(db_path=temp_db_path, create=True) as client: + # Create multiple documents + for i in range(3): + await client.create_document(content=f"Test document {i}") + + # Count versions - should accumulate without vacuum + doc_versions = len(list(client.store.documents_table.list_versions())) + chunk_versions = len(list(client.store.chunks_table.list_versions())) + + # Without auto-vacuum, versions should accumulate (more than 3 from creates) + assert doc_versions >= 3, ( + f"Without auto-vacuum, should have accumulated versions, got {doc_versions}" + ) + assert chunk_versions >= 3, ( + f"Without auto-vacuum, should have accumulated versions, got {chunk_versions}" + ) + + +@pytest.mark.asyncio +async def test_auto_vacuum_enabled_triggers_vacuum(temp_db_path, monkeypatch): + """Test that auto_vacuum=True (default) triggers vacuum after operations.""" + from haiku.rag.config import Config + + # Enable auto-vacuum with aggressive retention + monkeypatch.setattr(Config.storage, "auto_vacuum", True) + monkeypatch.setattr(Config.storage, "vacuum_retention_seconds", 0) + + async with HaikuRAG(db_path=temp_db_path, create=True) as client: + # Create multiple documents + for i in range(3): + await client.create_document(content=f"Test document {i}") + + # After context exit, vacuum should have cleaned up + store = Store(temp_db_path, create=True) + final_versions = len(list(store.documents_table.list_versions())) + + # With auto_vacuum=True and retention=0, should have minimal versions + assert final_versions <= 2, ( + f"With auto-vacuum enabled, should have minimal versions, got {final_versions}" + ) + store.close()