diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index 940e9429..00000000 Binary files a/docs/.DS_Store and /dev/null differ diff --git a/docs/img/.DS_Store b/docs/img/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/docs/img/.DS_Store and /dev/null differ diff --git a/src/haiku/rag/config.py b/src/haiku/rag/config.py index 98a8b93d..40fcc8cd 100644 --- a/src/haiku/rag/config.py +++ b/src/haiku/rag/config.py @@ -57,6 +57,11 @@ class AppConfig(BaseModel): # and error out when the database does not already exist. DISABLE_DB_AUTOCREATE: bool = False + # Vacuum retention threshold in seconds. Only versions older than this + # threshold will be removed during vacuum operations. Default is 60 seconds + # to allow concurrent connections to safely use recent versions. + VACUUM_RETENTION_SECONDS: int = 60 + @field_validator("MONITOR_DIRECTORIES", mode="before") @classmethod def parse_monitor_directories(cls, v): diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py index 4ef55106..5c4da84c 100644 --- a/src/haiku/rag/store/engine.py +++ b/src/haiku/rag/store/engine.py @@ -78,14 +78,20 @@ class Store: if not skip_validation: self._validate_configuration() - def vacuum(self) -> None: - """Optimize and clean up old versions across all tables to reduce disk usage.""" + def vacuum(self, retention_seconds: int = Config.VACUUM_RETENTION_SECONDS) -> None: + """Optimize and clean up old versions across all tables to reduce disk usage. + + Args: + retention_seconds: Retention threshold in seconds. Only versions older + than this will be removed. Defaults to Config.VACUUM_RETENTION_SECONDS. + """ if self._has_cloud_config() and str(Config.LANCEDB_URI).startswith("db://"): return - # Perform maintenance per table using optimize() with cleanup_older_than 0 + # Perform maintenance per table using optimize() with configurable retention + retention = timedelta(seconds=retention_seconds) for table in [self.documents_table, self.chunks_table, self.settings_table]: - table.optimize(cleanup_older_than=timedelta(0)) + table.optimize(cleanup_older_than=retention) def _connect_to_lancedb(self, db_path: Path): """Establish connection to LanceDB (local, cloud, or object storage).""" diff --git a/tests/test_versioning.py b/tests/test_versioning.py index 3d68c7d8..1a35336e 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -122,3 +122,75 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): Store(temp_db_path) assert called["value"] + + +@pytest.mark.asyncio +async def test_vacuum_with_retention_threshold(temp_db_path): + store = Store(temp_db_path) + repo = DocumentRepository(store) + + # Stub embeddings to avoid network + dim = repo.chunk_repository.embedder._vector_dim + + async def fake_embed(x): # type: ignore[no-redef] + if isinstance(x, list): + return [[0.0] * dim for _ in x] + return [0.0] * dim + + repo.chunk_repository.embedder.embed = fake_embed # type: ignore[assignment] + + # Create first document + doc1 = Document(content="First document") + dl_doc1 = text_to_docling_document("First document", name="doc1.md") + await repo._create_with_docling(doc1, dl_doc1) + + # Create second document + doc2 = Document(content="Second document") + dl_doc2 = text_to_docling_document("Second document", name="doc2.md") + await repo._create_with_docling(doc2, dl_doc2) + + # Get initial version counts (should have multiple versions from creates) + initial_doc_versions = len(list(store.documents_table.list_versions())) + initial_chunk_versions = len(list(store.chunks_table.list_versions())) + + assert initial_doc_versions > 1, "Should have multiple document table versions" + assert initial_chunk_versions > 1, "Should have multiple chunk table versions" + + # Vacuum with default threshold (60 seconds) - should keep recent versions + # Note: vacuum may create new versions even when not cleaning up old ones + store.vacuum() + + after_default_doc_versions = len(list(store.documents_table.list_versions())) + after_default_chunk_versions = len(list(store.chunks_table.list_versions())) + + # After vacuum with retention, version count should stay the same or increase + # (optimize may create new versions) but not decrease + assert after_default_doc_versions >= initial_doc_versions, ( + "Default vacuum should not remove recent versions" + ) + assert after_default_chunk_versions >= initial_chunk_versions, ( + "Default vacuum should not remove recent versions" + ) + + # Vacuum with 0 threshold - should significantly reduce versions + store.vacuum(retention_seconds=0) + + after_zero_doc_versions = len(list(store.documents_table.list_versions())) + after_zero_chunk_versions = len(list(store.chunks_table.list_versions())) + + # After aggressive vacuum, should have minimal versions (1-2) + # Note: optimize operation may create a version after cleanup + assert after_zero_doc_versions <= 2, ( + f"Should have minimal document versions after vacuum(0), got {after_zero_doc_versions}" + ) + assert after_zero_chunk_versions <= 2, ( + f"Should have minimal chunk versions after vacuum(0), got {after_zero_chunk_versions}" + ) + + # And it should be significantly fewer than before + assert after_zero_doc_versions < initial_doc_versions, ( + "Should have fewer versions after vacuum(0)" + ) + assert after_zero_chunk_versions < initial_chunk_versions, ( + "Should have fewer versions after vacuum(0)" + )