Add vacuum retention setting

src/haiku/rag/config.py

@@ -57,6 +57,11 @@ class AppConfig(BaseModel):
     # and error out when the database does not already exist.
     DISABLE_DB_AUTOCREATE: bool = False
 
+    # Vacuum retention threshold in seconds. Only versions older than this
+    # threshold will be removed during vacuum operations. Default is 60 seconds
+    # to allow concurrent connections to safely use recent versions.
+    VACUUM_RETENTION_SECONDS: int = 60
+
     @field_validator("MONITOR_DIRECTORIES", mode="before")
     @classmethod
     def parse_monitor_directories(cls, v):

src/haiku/rag/store/engine.py

@@ -78,14 +78,20 @@ class Store:
         if not skip_validation:
             self._validate_configuration()
 
-    def vacuum(self) -> None:
-        """Optimize and clean up old versions across all tables to reduce disk usage."""
+    def vacuum(self, retention_seconds: int = Config.VACUUM_RETENTION_SECONDS) -> None:
+        """Optimize and clean up old versions across all tables to reduce disk usage.
+
+        Args:
+            retention_seconds: Retention threshold in seconds. Only versions older
+                              than this will be removed. Defaults to Config.VACUUM_RETENTION_SECONDS.
+        """
         if self._has_cloud_config() and str(Config.LANCEDB_URI).startswith("db://"):
             return
 
-        # Perform maintenance per table using optimize() with cleanup_older_than 0
+        # Perform maintenance per table using optimize() with configurable retention
+        retention = timedelta(seconds=retention_seconds)
         for table in [self.documents_table, self.chunks_table, self.settings_table]:
-            table.optimize(cleanup_older_than=timedelta(0))
+            table.optimize(cleanup_older_than=retention)
 
     def _connect_to_lancedb(self, db_path: Path):
         """Establish connection to LanceDB (local, cloud, or object storage)."""

tests/test_versioning.py

@@ -122,3 +122,75 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path):
     Store(temp_db_path)
 
     assert called["value"]
+
+
+@pytest.mark.asyncio
+async def test_vacuum_with_retention_threshold(temp_db_path):
+    store = Store(temp_db_path)
+    repo = DocumentRepository(store)
+
+    # Stub embeddings to avoid network
+    dim = repo.chunk_repository.embedder._vector_dim
+
+    async def fake_embed(x):  # type: ignore[no-redef]
+        if isinstance(x, list):
+            return [[0.0] * dim for _ in x]
+        return [0.0] * dim
+
+    repo.chunk_repository.embedder.embed = fake_embed  # type: ignore[assignment]
+
+    # Create first document
+    doc1 = Document(content="First document")
+    dl_doc1 = text_to_docling_document("First document", name="doc1.md")
+    await repo._create_with_docling(doc1, dl_doc1)
+
+    # Create second document
+    doc2 = Document(content="Second document")
+    dl_doc2 = text_to_docling_document("Second document", name="doc2.md")
+    await repo._create_with_docling(doc2, dl_doc2)
+
+    # Get initial version counts (should have multiple versions from creates)
+    initial_doc_versions = len(list(store.documents_table.list_versions()))
+    initial_chunk_versions = len(list(store.chunks_table.list_versions()))
+
+    assert initial_doc_versions > 1, "Should have multiple document table versions"
+    assert initial_chunk_versions > 1, "Should have multiple chunk table versions"
+
+    # Vacuum with default threshold (60 seconds) - should keep recent versions
+    # Note: vacuum may create new versions even when not cleaning up old ones
+    store.vacuum()
+
+    after_default_doc_versions = len(list(store.documents_table.list_versions()))
+    after_default_chunk_versions = len(list(store.chunks_table.list_versions()))
+
+    # After vacuum with retention, version count should stay the same or increase
+    # (optimize may create new versions) but not decrease
+    assert after_default_doc_versions >= initial_doc_versions, (
+        "Default vacuum should not remove recent versions"
+    )
+    assert after_default_chunk_versions >= initial_chunk_versions, (
+        "Default vacuum should not remove recent versions"
+    )
+
+    # Vacuum with 0 threshold - should significantly reduce versions
+    store.vacuum(retention_seconds=0)
+
+    after_zero_doc_versions = len(list(store.documents_table.list_versions()))
+    after_zero_chunk_versions = len(list(store.chunks_table.list_versions()))
+
+    # After aggressive vacuum, should have minimal versions (1-2)
+    # Note: optimize operation may create a version after cleanup
+    assert after_zero_doc_versions <= 2, (
+        f"Should have minimal document versions after vacuum(0), got {after_zero_doc_versions}"
+    )
+    assert after_zero_chunk_versions <= 2, (
+        f"Should have minimal chunk versions after vacuum(0), got {after_zero_chunk_versions}"
+    )
+
+    # And it should be significantly fewer than before
+    assert after_zero_doc_versions < initial_doc_versions, (
+        "Should have fewer versions after vacuum(0)"
+    )
+    assert after_zero_chunk_versions < initial_chunk_versions, (
+        "Should have fewer versions after vacuum(0)"
+    )