Evaluate over a configured set of databases

The runner passed the dataset's path to every arm, which opens one database
and is what makes `--db` meaningful. A run over `lancedb.databases` has to
pass none instead, so the client resolves the set, and `DatasetSpec.covers_a_set`
is the one place that decides which of the two a run is.

evaluations/evaluations/capability_runner.py

@@ -120,7 +120,7 @@ class _EvalDeps:
 
 def _prepare_agent(
     capability_factory: CapabilityFactory,
-    db_path: Path,
+    db_path: Path | None,
     config: AppConfig,
     capability_model: str | Model,
     document_filter: str | None,
@@ -158,7 +158,7 @@ def _state_after_run(
 
 async def run_capability_question(
     capability_factory: CapabilityFactory,
-    db_path: Path,
+    db_path: Path | None,
     config: AppConfig,
     question: str,
     capability_model: str | Model,
@@ -195,7 +195,7 @@ async def run_capability_question(
 
 async def run_capability_conversation(
     capability_factory: CapabilityFactory,
-    db_path: Path,
+    db_path: Path | None,
     config: AppConfig,
     questions: list[str],
     capability_model: str | Model,

evaluations/evaluations/config.py

@@ -83,6 +83,14 @@ class DatasetSpec:
     compaction: bool = False
     experiment_metadata: dict[str, Any] | None = None
 
+    def covers_a_set(self, config) -> bool:
+        """Whether `lancedb.databases` names the databases to evaluate over.
+
+        A path names one database and wins over the configured set, so a run over
+        a set has to pass none — the client resolves it.
+        """
+        return bool(config.lancedb.databases)
+
     def db_path(self, override_path: Path | None = None) -> Path:
         """Get the database path.
 

evaluations/evaluations/qa.py

@@ -195,7 +195,7 @@ def _filter_qa_corpus(corpus, case_ids: set[str] | None):
 
 class _QARun(NamedTuple):
     cases: list[Case[Any, Any, dict[str, Any]]]
-    db: Path
+    db: Path | None
     judge_config: ModelConfig
     eval_name: str
     experiment_metadata: dict[str, Any]
@@ -243,7 +243,7 @@ def _prepare_qa_run(
 
     return _QARun(
         cases=cases,
-        db=spec.db_path(db_path),
+        db=None if spec.covers_a_set(config) else spec.db_path(db_path),
         judge_config=judge_config,
         eval_name=eval_name,
         experiment_metadata=experiment_metadata,

evaluations/evaluations/retrieval.py

@@ -72,7 +72,7 @@ async def run_retrieval_benchmark(
         evaluators=list(spec.retrieval_evaluators),
     )
 
-    db = spec.db_path(db_path)
+    db = None if spec.covers_a_set(config) else spec.db_path(db_path)
     async with HaikuRAG(db, config=config, read_only=True) as rag:
 
         async def retrieval_target(question: str) -> list[str]:

evaluations/tests/test_config.py

@@ -145,3 +145,29 @@ class TestRetrievalSample:
         )
         assert sample.skip is True
         assert sample.source_type == "image"
+
+
+class TestCoversASet:
+    """A run over `lancedb.databases` must pass no path, since a path names one
+    database and wins over the configured set."""
+
+    def test_a_configured_set_is_covered(self):
+        from haiku.rag.config.models import AppConfig, LanceDBConfig
+
+        from evaluations.datasets import DATASETS
+
+        spec = next(iter(DATASETS.values()))
+        config = AppConfig(
+            lancedb=LanceDBConfig(databases={"a": "/a.lancedb", "b": "/b.lancedb"})
+        )
+
+        assert spec.covers_a_set(config) is True
+
+    def test_one_database_is_not_a_set(self):
+        from haiku.rag.config.models import AppConfig
+
+        from evaluations.datasets import DATASETS
+
+        spec = next(iter(DATASETS.values()))
+
+        assert spec.covers_a_set(AppConfig()) is False