diff --git a/evaluations/README.md b/evaluations/README.md index 06acebd5..7c881a22 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -89,3 +89,21 @@ By default, evaluation databases are stored in the haiku.rag data directory: - **Windows**: `C:/Users//AppData/Roaming/haiku.rag/evaluations/dbs/` You can override this with the `--db` option. + +### Evaluating over several databases + +With [`lancedb.databases`](https://ggozad.github.io/haiku.rag/configuration/storage/#several-databases) +configured, `evaluations run --skip-db` benchmarks the configured set: +retrieval, QA and live conversations all search every database in it, and each +result and citation names the one it came from. A mapping of one follows the same +path and keeps its configured name. + +Population is not part of that. It writes one database, so it needs a path: + +```bash +evaluations run hotpotqa --db /path/to/one.lancedb # populate, then benchmark +evaluations run hotpotqa --skip-db # benchmark the configured set +``` + +A `--db` path names one database and overrides the configured set for the whole +run, population and benchmarks alike. diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index ac056341..182d5977 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -50,12 +50,12 @@ async def evaluate_dataset( console.print(f"Document filter: {document_filter}", style="dim") if not skip_db: - if spec.covers_a_set(config, db_path): + if spec.uses_configured_databases(config, db_path): raise ValueError( - "lancedb.databases names several databases and population writes " - "to one, so it would ingest into a database the run does not " - "read. Pass --skip-db to evaluate the configured set, or --db " - "PATH to populate and evaluate one database." + "lancedb.databases places the databases this run reads, and " + "population writes to one, so it would ingest into a database " + "the run does not read. Pass --skip-db to evaluate the " + "configured set, or --db PATH to populate and evaluate one." ) console.print(f"Using dataset: {spec.key}", style="bold magenta") await populate_db( diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 30c8956a..96c56232 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -83,12 +83,14 @@ class DatasetSpec: compaction: bool = False experiment_metadata: dict[str, Any] | None = None - def covers_a_set(self, config, override_path: Path | None = None) -> bool: - """Whether `lancedb.databases` names the databases to evaluate over. + def uses_configured_databases( + self, config, override_path: Path | None = None + ) -> bool: + """Whether `lancedb.databases` places the databases to evaluate over. - A path names one database and wins over the configured set, both when it - comes from `--db` and when the client resolves it, so a run over a set is - one where the configuration names several and nobody named a path. + A path names one database and wins over the configuration, both when it + comes from `--db` and when the client resolves it. True for a mapping of + one, which is a configured database like any other and keeps its name. """ return bool(config.lancedb.databases) and override_path is None diff --git a/evaluations/evaluations/qa.py b/evaluations/evaluations/qa.py index 75816355..fdf8d0dd 100644 --- a/evaluations/evaluations/qa.py +++ b/evaluations/evaluations/qa.py @@ -252,7 +252,9 @@ def _prepare_qa_run( return _QARun( cases=cases, - db=None if spec.covers_a_set(config, db_path) else spec.db_path(db_path), + db=None + if spec.uses_configured_databases(config, db_path) + else spec.db_path(db_path), judge_config=judge_config, eval_name=eval_name, experiment_metadata=experiment_metadata, diff --git a/evaluations/evaluations/retrieval.py b/evaluations/evaluations/retrieval.py index ad6d84c3..9768c587 100644 --- a/evaluations/evaluations/retrieval.py +++ b/evaluations/evaluations/retrieval.py @@ -72,7 +72,11 @@ async def run_retrieval_benchmark( evaluators=list(spec.retrieval_evaluators), ) - db = None if spec.covers_a_set(config, db_path) else spec.db_path(db_path) + db = ( + None + if spec.uses_configured_databases(config, db_path) + else spec.db_path(db_path) + ) async with HaikuRAG(db, config=config, read_only=True) as rag: async def retrieval_target(question: str) -> list[str]: diff --git a/evaluations/tests/test_config.py b/evaluations/tests/test_config.py index 6cafc5c4..a3123e89 100644 --- a/evaluations/tests/test_config.py +++ b/evaluations/tests/test_config.py @@ -161,7 +161,7 @@ class TestCoversASet: lancedb=LanceDBConfig(databases={"a": "/a.lancedb", "b": "/b.lancedb"}) ) - assert spec.covers_a_set(config) is True + assert spec.uses_configured_databases(config) is True def test_a_named_path_overrides_the_set(self): """`--db` is documented as an override, so it names the one database to @@ -177,7 +177,7 @@ class TestCoversASet: lancedb=LanceDBConfig(databases={"a": "/a.lancedb", "b": "/b.lancedb"}) ) - assert spec.covers_a_set(config, _Path("/chosen.lancedb")) is False + assert spec.uses_configured_databases(config, _Path("/chosen.lancedb")) is False def test_one_database_is_not_a_set(self): from haiku.rag.config.models import AppConfig @@ -186,4 +186,4 @@ class TestCoversASet: spec = next(iter(DATASETS.values())) - assert spec.covers_a_set(AppConfig()) is False + assert spec.uses_configured_databases(AppConfig()) is False