From c6eaf9d5cd3584ba10e21ebf0a638eb9a0f41b69 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 7 Oct 2025 10:47:28 +0300 Subject: [PATCH] Add vacuum retention setting --- docs/.DS_Store | Bin 6148 -> 0 bytes docs/img/.DS_Store | Bin 6148 -> 0 bytes src/haiku/rag/config.py | 5 +++ src/haiku/rag/store/engine.py | 14 +++++-- tests/test_versioning.py | 72 ++++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 4 deletions(-) delete mode 100644 docs/.DS_Store delete mode 100644 docs/img/.DS_Store diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index 940e942902c4c95c53cc11d73aa9ce43019c31f2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~J!%6%427R!7lt%jx}3%b$PET#pTHMLVK4#Pfk0Bv(ew1vWSu%J;R&QS(yZ9s zuh>}uu>I%x1(*PA=&sm#n3*wO;SD!jzD^(a>-+t}idTWBh?%i6VYXk}5)lvq5fA|p z5P<~|$Wt7f=LJ2J9z_I1U>OAb`_SmFy>z6;r-LCz0P33MFs@^kpf)d1d+A7Jg=RH9 zShZS=AzqJmYOCvd=}66XSPdUmcQ&75XqN4;#)M`)L_q{ZU`Ak-`Q+#Sk^bBKKWkAc z0wVCw2-x~?I_&vUb+$gdp4VTi>gz$L#^nq@egc^IQM{#xaliS3+Dk_&D>VHG1O^2W H_)`MkK;;pJ diff --git a/docs/img/.DS_Store b/docs/img/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 None: - """Optimize and clean up old versions across all tables to reduce disk usage.""" + def vacuum(self, retention_seconds: int = Config.VACUUM_RETENTION_SECONDS) -> None: + """Optimize and clean up old versions across all tables to reduce disk usage. + + Args: + retention_seconds: Retention threshold in seconds. Only versions older + than this will be removed. Defaults to Config.VACUUM_RETENTION_SECONDS. + """ if self._has_cloud_config() and str(Config.LANCEDB_URI).startswith("db://"): return - # Perform maintenance per table using optimize() with cleanup_older_than 0 + # Perform maintenance per table using optimize() with configurable retention + retention = timedelta(seconds=retention_seconds) for table in [self.documents_table, self.chunks_table, self.settings_table]: - table.optimize(cleanup_older_than=timedelta(0)) + table.optimize(cleanup_older_than=retention) def _connect_to_lancedb(self, db_path: Path): """Establish connection to LanceDB (local, cloud, or object storage).""" diff --git a/tests/test_versioning.py b/tests/test_versioning.py index 3d68c7d8..1a35336e 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -122,3 +122,75 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): Store(temp_db_path) assert called["value"] + + +@pytest.mark.asyncio +async def test_vacuum_with_retention_threshold(temp_db_path): + store = Store(temp_db_path) + repo = DocumentRepository(store) + + # Stub embeddings to avoid network + dim = repo.chunk_repository.embedder._vector_dim + + async def fake_embed(x): # type: ignore[no-redef] + if isinstance(x, list): + return [[0.0] * dim for _ in x] + return [0.0] * dim + + repo.chunk_repository.embedder.embed = fake_embed # type: ignore[assignment] + + # Create first document + doc1 = Document(content="First document") + dl_doc1 = text_to_docling_document("First document", name="doc1.md") + await repo._create_with_docling(doc1, dl_doc1) + + # Create second document + doc2 = Document(content="Second document") + dl_doc2 = text_to_docling_document("Second document", name="doc2.md") + await repo._create_with_docling(doc2, dl_doc2) + + # Get initial version counts (should have multiple versions from creates) + initial_doc_versions = len(list(store.documents_table.list_versions())) + initial_chunk_versions = len(list(store.chunks_table.list_versions())) + + assert initial_doc_versions > 1, "Should have multiple document table versions" + assert initial_chunk_versions > 1, "Should have multiple chunk table versions" + + # Vacuum with default threshold (60 seconds) - should keep recent versions + # Note: vacuum may create new versions even when not cleaning up old ones + store.vacuum() + + after_default_doc_versions = len(list(store.documents_table.list_versions())) + after_default_chunk_versions = len(list(store.chunks_table.list_versions())) + + # After vacuum with retention, version count should stay the same or increase + # (optimize may create new versions) but not decrease + assert after_default_doc_versions >= initial_doc_versions, ( + "Default vacuum should not remove recent versions" + ) + assert after_default_chunk_versions >= initial_chunk_versions, ( + "Default vacuum should not remove recent versions" + ) + + # Vacuum with 0 threshold - should significantly reduce versions + store.vacuum(retention_seconds=0) + + after_zero_doc_versions = len(list(store.documents_table.list_versions())) + after_zero_chunk_versions = len(list(store.chunks_table.list_versions())) + + # After aggressive vacuum, should have minimal versions (1-2) + # Note: optimize operation may create a version after cleanup + assert after_zero_doc_versions <= 2, ( + f"Should have minimal document versions after vacuum(0), got {after_zero_doc_versions}" + ) + assert after_zero_chunk_versions <= 2, ( + f"Should have minimal chunk versions after vacuum(0), got {after_zero_chunk_versions}" + ) + + # And it should be significantly fewer than before + assert after_zero_doc_versions < initial_doc_versions, ( + "Should have fewer versions after vacuum(0)" + ) + assert after_zero_chunk_versions < initial_chunk_versions, ( + "Should have fewer versions after vacuum(0)" + )