Add auto_vacuum config option to control automatic vacuuming

CHANGELOG.md

@@ -40,6 +40,10 @@
   - HTML format preserves document structure (headings, lists, sections) in DoclingDocument
   - Enables proper parsing of HTML content that was previously treated as plain text
 - **Inspector Context Modal**: Press `c` in the inspector to view expanded context for the selected chunk
+- **Auto-Vacuum Configuration**: New `storage.auto_vacuum` setting to control automatic vacuuming behavior
+  - When `true` (default), vacuum runs automatically after document create/update operations and rebuilds
+  - When `false`, vacuum only runs via explicit `haiku-rag vacuum` command
+  - Disabling can help avoid potential crashes in high-concurrency scenarios due to LanceDB race conditions
 
 ### Changed
 

docs/configuration/storage.md

@@ -7,11 +7,16 @@ By default, `haiku.rag` uses a local LanceDB database:
 ```yaml
 storage:
   data_dir: /path/to/data  # Empty = use default platform location
+  auto_vacuum: true  # Enable automatic vacuuming after operations
   vacuum_retention_seconds: 86400  # Cleanup threshold in seconds
 ```
 
 - **data_dir**: Directory for local database storage. When empty, uses platform-specific default locations
-- **vacuum_retention_seconds**: When documents are added/updated, old table versions older than this are removed. Default: 86400 seconds (1 day, safe for concurrent connections). Set to 0 for aggressive cleanup (removes all old versions immediately)
+- **auto_vacuum**: When enabled (default), automatically runs vacuum after document create/update operations and database rebuilds. Set to `false` to disable automatic vacuuming and rely on manual `haiku-rag vacuum` commands only. Disabling can help avoid potential crashes in high-concurrency scenarios
+- **vacuum_retention_seconds**: When vacuum runs, old table versions older than this threshold are removed. Default: 86400 seconds (1 day). Set to 0 for aggressive cleanup (removes all old versions immediately)
+
+!!! warning "Vacuum Retention Threshold"
+    The `vacuum_retention_seconds` value should be larger than the typical time it takes to process and write a document. If a concurrent operation is in progress while vacuum runs, setting this value too low can cause race conditions where vacuum removes table versions that an in-flight operation still needs. The default of 86400 seconds (1 day) is conservative and safe for most use cases.
 
 ## Remote Storage
 

haiku_rag_slim/haiku/rag/client.py

@@ -271,8 +271,9 @@ class HaikuRAG:
             # Batch create all chunks in a single operation
             await self.chunk_repository.create(chunks)
 
-            # Vacuum old versions in background (non-blocking)
-            asyncio.create_task(self.store.vacuum())
+            # Vacuum old versions in background (non-blocking) if auto_vacuum enabled
+            if self._config.storage.auto_vacuum:
+                asyncio.create_task(self.store.vacuum())
 
             return created_doc
         except Exception:
@@ -322,8 +323,9 @@ class HaikuRAG:
             # Batch create all chunks in a single operation
             await self.chunk_repository.create(chunks)
 
-            # Vacuum old versions in background (non-blocking)
-            asyncio.create_task(self.store.vacuum())
+            # Vacuum old versions in background (non-blocking) if auto_vacuum enabled
+            if self._config.storage.auto_vacuum:
+                asyncio.create_task(self.store.vacuum())
 
             return updated_doc
         except Exception:
@@ -1365,11 +1367,12 @@ class HaikuRAG:
             async for doc_id in self._rebuild_full(documents):
                 yield doc_id
 
-        # Final maintenance
-        try:
-            await self.store.vacuum()
-        except Exception:
-            pass
+        # Final maintenance if auto_vacuum enabled
+        if self._config.storage.auto_vacuum:
+            try:
+                await self.store.vacuum()
+            except Exception:
+                pass
 
     async def _rebuild_embed_only(
         self, documents: list[Document]

haiku_rag_slim/haiku/rag/config/models.py

@@ -41,6 +41,7 @@ class EmbeddingModelConfig(BaseModel):
 
 class StorageConfig(BaseModel):
     data_dir: Path = Field(default_factory=get_default_data_dir)
+    auto_vacuum: bool = True
     vacuum_retention_seconds: int = 86400
 
 

tests/test_versioning.py

@@ -174,3 +174,54 @@ async def test_vacuum_completes_before_context_exit(temp_db_path, monkeypatch):
     )
     assert final_versions >= 1, "Should have at least one version remaining"
     store.close()
+
+
+@pytest.mark.asyncio
+async def test_auto_vacuum_disabled_skips_vacuum(temp_db_path, monkeypatch):
+    """Test that auto_vacuum=False prevents automatic vacuum after operations."""
+    from haiku.rag.config import Config
+
+    # Disable auto-vacuum
+    monkeypatch.setattr(Config.storage, "auto_vacuum", False)
+
+    async with HaikuRAG(db_path=temp_db_path, create=True) as client:
+        # Create multiple documents
+        for i in range(3):
+            await client.create_document(content=f"Test document {i}")
+
+        # Count versions - should accumulate without vacuum
+        doc_versions = len(list(client.store.documents_table.list_versions()))
+        chunk_versions = len(list(client.store.chunks_table.list_versions()))
+
+        # Without auto-vacuum, versions should accumulate (more than 3 from creates)
+        assert doc_versions >= 3, (
+            f"Without auto-vacuum, should have accumulated versions, got {doc_versions}"
+        )
+        assert chunk_versions >= 3, (
+            f"Without auto-vacuum, should have accumulated versions, got {chunk_versions}"
+        )
+
+
+@pytest.mark.asyncio
+async def test_auto_vacuum_enabled_triggers_vacuum(temp_db_path, monkeypatch):
+    """Test that auto_vacuum=True (default) triggers vacuum after operations."""
+    from haiku.rag.config import Config
+
+    # Enable auto-vacuum with aggressive retention
+    monkeypatch.setattr(Config.storage, "auto_vacuum", True)
+    monkeypatch.setattr(Config.storage, "vacuum_retention_seconds", 0)
+
+    async with HaikuRAG(db_path=temp_db_path, create=True) as client:
+        # Create multiple documents
+        for i in range(3):
+            await client.create_document(content=f"Test document {i}")
+
+    # After context exit, vacuum should have cleaned up
+    store = Store(temp_db_path, create=True)
+    final_versions = len(list(store.documents_table.list_versions()))
+
+    # With auto_vacuum=True and retention=0, should have minimal versions
+    assert final_versions <= 2, (
+        f"With auto-vacuum enabled, should have minimal versions, got {final_versions}"
+    )
+    store.close()