diff --git a/CHANGELOG.md b/CHANGELOG.md index fee7f76b..456e0a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ ### Changed +- **Evaluations Vacuum Strategy**: `populate_db` now uses periodic vacuum to prevent disk exhaustion with large datasets + - Disables auto_vacuum during population, vacuums every 100 documents with retention=0 + - Prevents disk space issues when building databases with thousands of documents (e.g., HotpotQA) - **Benchmarks Documentation**: Restructured benchmarks.md for clarity - Added dedicated Methodology section explaining MRR, MAP, and QA Accuracy metrics - Organized results by dataset with retrieval and QA subsections diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index b6d260b8..cb689e55 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -75,9 +75,14 @@ async def populate_db( if spec.document_limit is not None: corpus = corpus.select(range(min(spec.document_limit, len(corpus)))) + # Disable auto_vacuum - we'll vacuum periodically instead to prevent disk exhaustion + config.storage.auto_vacuum = False + vacuum_interval = 100 + with Progress() as progress: task = progress.add_task("[green]Populating database...", total=len(corpus)) async with HaikuRAG(db, config=config) as rag: + docs_since_vacuum = 0 for doc in corpus: doc_mapping = cast(Mapping[str, Any], doc) payload = spec.document_mapper(doc_mapping) @@ -101,8 +106,17 @@ async def populate_db( metadata=payload.metadata, format=payload.format, ) + docs_since_vacuum += 1 progress.advance(task) + # Periodic vacuum to prevent disk exhaustion + if docs_since_vacuum >= vacuum_interval: + await rag.store.vacuum(retention_seconds=0) + docs_since_vacuum = 0 + + # Final vacuum + await rag.store.vacuum(retention_seconds=0) + async def run_retrieval_benchmark( spec: DatasetSpec,