Merge pull request #193 from ggozad/fix/vacuum-evaluations

Evaluations: use periodic vacuum to prevent disk exhaustion with large datasets

CHANGELOG.md

@@ -10,6 +10,10 @@
 
 ### Changed
 
+- **Evaluations Vacuum Strategy**: `populate_db` now uses periodic vacuum to prevent disk exhaustion with large datasets
+  - Disables auto_vacuum during population, vacuums every N documents with retention=0
+  - New `--vacuum-interval` CLI option (default: 100) to control vacuum frequency
+  - Prevents disk space issues when building databases with thousands of documents (e.g., HotpotQA)
 - **Benchmarks Documentation**: Restructured benchmarks.md for clarity
   - Added dedicated Methodology section explaining MRR, MAP, and QA Accuracy metrics
   - Organized results by dataset with retrieval and QA subsections

evaluations/evaluations/benchmark.py

@@ -67,7 +67,10 @@ def build_experiment_metadata(
 
 
 async def populate_db(
-    spec: DatasetSpec, config: AppConfig, db_path: Path | None = None
+    spec: DatasetSpec,
+    config: AppConfig,
+    db_path: Path | None = None,
+    vacuum_interval: int = 100,
 ) -> None:
     db = spec.db_path(db_path)
     db.parent.mkdir(parents=True, exist_ok=True)
@@ -75,9 +78,13 @@ async def populate_db(
     if spec.document_limit is not None:
         corpus = corpus.select(range(min(spec.document_limit, len(corpus))))
 
+    # Disable auto_vacuum - we'll vacuum periodically instead to prevent disk exhaustion
+    config.storage.auto_vacuum = False
+
     with Progress() as progress:
         task = progress.add_task("[green]Populating database...", total=len(corpus))
         async with HaikuRAG(db, config=config) as rag:
+            docs_since_vacuum = 0
             for doc in corpus:
                 doc_mapping = cast(Mapping[str, Any], doc)
                 payload = spec.document_mapper(doc_mapping)
@@ -101,8 +108,17 @@ async def populate_db(
                     metadata=payload.metadata,
                     format=payload.format,
                 )
+                docs_since_vacuum += 1
                 progress.advance(task)
 
+                # Periodic vacuum to prevent disk exhaustion
+                if docs_since_vacuum >= vacuum_interval:
+                    await rag.store.vacuum(retention_seconds=0)
+                    docs_since_vacuum = 0
+
+            # Final vacuum
+            await rag.store.vacuum(retention_seconds=0)
+
 
 async def run_retrieval_benchmark(
     spec: DatasetSpec,
@@ -309,10 +325,13 @@ async def evaluate_dataset(
     limit: int | None,
     name: str | None,
     db_path: Path | None,
+    vacuum_interval: int = 100,
 ) -> None:
     if not skip_db:
         console.print(f"Using dataset: {spec.key}", style="bold magenta")
-        await populate_db(spec, config, db_path=db_path)
+        await populate_db(
+            spec, config, db_path=db_path, vacuum_interval=vacuum_interval
+        )
 
     if not skip_retrieval:
         console.print("Running retrieval benchmarks...", style="bold blue")
@@ -346,6 +365,9 @@ def run(
         None, "--limit", help="Limit number of test cases for both retrieval and QA."
     ),
     name: str | None = typer.Option(None, "--name", help="Override evaluation name."),
+    vacuum_interval: int = typer.Option(
+        100, "--vacuum-interval", help="Vacuum every N documents during DB population."
+    ),
 ) -> None:
     spec = DATASETS.get(dataset.lower())
     if spec is None:
@@ -382,6 +404,7 @@ def run(
             limit=limit,
             name=name,
             db_path=db,
+            vacuum_interval=vacuum_interval,
         )
     )