From 056b9ad09028f2dc2633aa771e09319a337aed20 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 12 Dec 2025 15:53:23 +0200 Subject: [PATCH 1/2] Use periodic vacuum to prevent disk exhaustion with large datasets --- CHANGELOG.md | 3 +++ evaluations/evaluations/benchmark.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fee7f76b..456e0a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ ### Changed +- **Evaluations Vacuum Strategy**: `populate_db` now uses periodic vacuum to prevent disk exhaustion with large datasets + - Disables auto_vacuum during population, vacuums every 100 documents with retention=0 + - Prevents disk space issues when building databases with thousands of documents (e.g., HotpotQA) - **Benchmarks Documentation**: Restructured benchmarks.md for clarity - Added dedicated Methodology section explaining MRR, MAP, and QA Accuracy metrics - Organized results by dataset with retrieval and QA subsections diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index b6d260b8..cb689e55 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -75,9 +75,14 @@ async def populate_db( if spec.document_limit is not None: corpus = corpus.select(range(min(spec.document_limit, len(corpus)))) + # Disable auto_vacuum - we'll vacuum periodically instead to prevent disk exhaustion + config.storage.auto_vacuum = False + vacuum_interval = 100 + with Progress() as progress: task = progress.add_task("[green]Populating database...", total=len(corpus)) async with HaikuRAG(db, config=config) as rag: + docs_since_vacuum = 0 for doc in corpus: doc_mapping = cast(Mapping[str, Any], doc) payload = spec.document_mapper(doc_mapping) @@ -101,8 +106,17 @@ async def populate_db( metadata=payload.metadata, format=payload.format, ) + docs_since_vacuum += 1 progress.advance(task) + # Periodic vacuum to prevent disk exhaustion + if docs_since_vacuum >= vacuum_interval: + await rag.store.vacuum(retention_seconds=0) + docs_since_vacuum = 0 + + # Final vacuum + await rag.store.vacuum(retention_seconds=0) + async def run_retrieval_benchmark( spec: DatasetSpec, From 40e0ea090da2b7e6c7d731f65273f0b423071418 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 12 Dec 2025 16:04:06 +0200 Subject: [PATCH 2/2] Add option to override vacuum interval --- CHANGELOG.md | 3 ++- evaluations/evaluations/benchmark.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 456e0a9c..0df7d0b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ ### Changed - **Evaluations Vacuum Strategy**: `populate_db` now uses periodic vacuum to prevent disk exhaustion with large datasets - - Disables auto_vacuum during population, vacuums every 100 documents with retention=0 + - Disables auto_vacuum during population, vacuums every N documents with retention=0 + - New `--vacuum-interval` CLI option (default: 100) to control vacuum frequency - Prevents disk space issues when building databases with thousands of documents (e.g., HotpotQA) - **Benchmarks Documentation**: Restructured benchmarks.md for clarity - Added dedicated Methodology section explaining MRR, MAP, and QA Accuracy metrics diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index cb689e55..c2e94cd4 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -67,7 +67,10 @@ def build_experiment_metadata( async def populate_db( - spec: DatasetSpec, config: AppConfig, db_path: Path | None = None + spec: DatasetSpec, + config: AppConfig, + db_path: Path | None = None, + vacuum_interval: int = 100, ) -> None: db = spec.db_path(db_path) db.parent.mkdir(parents=True, exist_ok=True) @@ -77,7 +80,6 @@ async def populate_db( # Disable auto_vacuum - we'll vacuum periodically instead to prevent disk exhaustion config.storage.auto_vacuum = False - vacuum_interval = 100 with Progress() as progress: task = progress.add_task("[green]Populating database...", total=len(corpus)) @@ -323,10 +325,13 @@ async def evaluate_dataset( limit: int | None, name: str | None, db_path: Path | None, + vacuum_interval: int = 100, ) -> None: if not skip_db: console.print(f"Using dataset: {spec.key}", style="bold magenta") - await populate_db(spec, config, db_path=db_path) + await populate_db( + spec, config, db_path=db_path, vacuum_interval=vacuum_interval + ) if not skip_retrieval: console.print("Running retrieval benchmarks...", style="bold blue") @@ -360,6 +365,9 @@ def run( None, "--limit", help="Limit number of test cases for both retrieval and QA." ), name: str | None = typer.Option(None, "--name", help="Override evaluation name."), + vacuum_interval: int = typer.Option( + 100, "--vacuum-interval", help="Vacuum every N documents during DB population." + ), ) -> None: spec = DATASETS.get(dataset.lower()) if spec is None: @@ -396,6 +404,7 @@ def run( limit=limit, name=name, db_path=db, + vacuum_interval=vacuum_interval, ) )