Name the evaluations set check for what it answers

`covers_a_set` is true for a mapping of one, which is a configured database
like any other; `uses_configured_databases` says that. Its population guard
said "several" for the same reason. Document evaluating the configured set
with `--skip-db`, against population, which writes one database and needs a
path.

evaluations/README.md

@@ -89,3 +89,21 @@ By default, evaluation databases are stored in the haiku.rag data directory:
 - **Windows**: `C:/Users/<USER>/AppData/Roaming/haiku.rag/evaluations/dbs/`
 
 You can override this with the `--db` option.
+
+### Evaluating over several databases
+
+With [`lancedb.databases`](https://ggozad.github.io/haiku.rag/configuration/storage/#several-databases)
+configured, `evaluations run <dataset> --skip-db` benchmarks the configured set:
+retrieval, QA and live conversations all search every database in it, and each
+result and citation names the one it came from. A mapping of one follows the same
+path and keeps its configured name.
+
+Population is not part of that. It writes one database, so it needs a path:
+
+```bash
+evaluations run hotpotqa --db /path/to/one.lancedb   # populate, then benchmark
+evaluations run hotpotqa --skip-db                   # benchmark the configured set
+```
+
+A `--db` path names one database and overrides the configured set for the whole
+run, population and benchmarks alike.

evaluations/evaluations/benchmark.py

@@ -50,12 +50,12 @@ async def evaluate_dataset(
         console.print(f"Document filter: {document_filter}", style="dim")
 
     if not skip_db:
-        if spec.covers_a_set(config, db_path):
+        if spec.uses_configured_databases(config, db_path):
             raise ValueError(
-                "lancedb.databases names several databases and population writes "
-                "to one, so it would ingest into a database the run does not "
-                "read. Pass --skip-db to evaluate the configured set, or --db "
-                "PATH to populate and evaluate one database."
+                "lancedb.databases places the databases this run reads, and "
+                "population writes to one, so it would ingest into a database "
+                "the run does not read. Pass --skip-db to evaluate the "
+                "configured set, or --db PATH to populate and evaluate one."
             )
         console.print(f"Using dataset: {spec.key}", style="bold magenta")
         await populate_db(

evaluations/evaluations/config.py

@@ -83,12 +83,14 @@ class DatasetSpec:
     compaction: bool = False
     experiment_metadata: dict[str, Any] | None = None
 
-    def covers_a_set(self, config, override_path: Path | None = None) -> bool:
-        """Whether `lancedb.databases` names the databases to evaluate over.
+    def uses_configured_databases(
+        self, config, override_path: Path | None = None
+    ) -> bool:
+        """Whether `lancedb.databases` places the databases to evaluate over.
 
-        A path names one database and wins over the configured set, both when it
-        comes from `--db` and when the client resolves it, so a run over a set is
-        one where the configuration names several and nobody named a path.
+        A path names one database and wins over the configuration, both when it
+        comes from `--db` and when the client resolves it. True for a mapping of
+        one, which is a configured database like any other and keeps its name.
         """
         return bool(config.lancedb.databases) and override_path is None
 

evaluations/evaluations/qa.py

@@ -252,7 +252,9 @@ def _prepare_qa_run(
 
     return _QARun(
         cases=cases,
-        db=None if spec.covers_a_set(config, db_path) else spec.db_path(db_path),
+        db=None
+        if spec.uses_configured_databases(config, db_path)
+        else spec.db_path(db_path),
         judge_config=judge_config,
         eval_name=eval_name,
         experiment_metadata=experiment_metadata,

evaluations/evaluations/retrieval.py

@@ -72,7 +72,11 @@ async def run_retrieval_benchmark(
         evaluators=list(spec.retrieval_evaluators),
     )
 
-    db = None if spec.covers_a_set(config, db_path) else spec.db_path(db_path)
+    db = (
+        None
+        if spec.uses_configured_databases(config, db_path)
+        else spec.db_path(db_path)
+    )
     async with HaikuRAG(db, config=config, read_only=True) as rag:
 
         async def retrieval_target(question: str) -> list[str]:

evaluations/tests/test_config.py

@@ -161,7 +161,7 @@ class TestCoversASet:
             lancedb=LanceDBConfig(databases={"a": "/a.lancedb", "b": "/b.lancedb"})
         )
 
-        assert spec.covers_a_set(config) is True
+        assert spec.uses_configured_databases(config) is True
 
     def test_a_named_path_overrides_the_set(self):
         """`--db` is documented as an override, so it names the one database to
@@ -177,7 +177,7 @@ class TestCoversASet:
             lancedb=LanceDBConfig(databases={"a": "/a.lancedb", "b": "/b.lancedb"})
         )
 
-        assert spec.covers_a_set(config, _Path("/chosen.lancedb")) is False
+        assert spec.uses_configured_databases(config, _Path("/chosen.lancedb")) is False
 
     def test_one_database_is_not_a_set(self):
         from haiku.rag.config.models import AppConfig
@@ -186,4 +186,4 @@ class TestCoversASet:
 
         spec = next(iter(DATASETS.values()))
 
-        assert spec.covers_a_set(AppConfig()) is False
+        assert spec.uses_configured_databases(AppConfig()) is False