Wait for drained jobs instead of sleeping in the breaker isolation test

test_breaker_isolates_sources snapshotted job status after a fixed 0.2s
sleep and failed on a loaded runner while two good jobs were still in
flight. Poll for the drained jobs under a 5s deadline.

Assert the paused source's jobs are unattempted: the queued-uri assertion
alone passes with source isolation disabled, since a retried job returns
to QUEUED.

.agents/plugins/marketplace.json

@@ -0,0 +1,20 @@
+{
+  "name": "haiku-rag",
+  "interface": {
+    "displayName": "haiku.rag"
+  },
+  "plugins": [
+    {
+      "name": "haiku-rag",
+      "source": {
+        "source": "local",
+        "path": "./plugins/haiku-rag"
+      },
+      "policy": {
+        "installation": "AVAILABLE",
+        "authentication": "ON_INSTALL"
+      },
+      "category": "Productivity"
+    }
+  ]
+}

.claude-plugin/marketplace.json

@@ -0,0 +1,14 @@
+{
+  "name": "haiku-rag",
+  "description": "The haiku.rag knowledge base as Claude Code tools and a skill.",
+  "owner": {
+    "name": "Yiorgis Gozadinos"
+  },
+  "plugins": [
+    {
+      "name": "haiku-rag",
+      "source": "./plugins/haiku-rag",
+      "description": "Search, read and analyze your haiku.rag knowledge base from Claude Code."
+    }
+  ]
+}

CHANGELOG.md

@@ -2,6 +2,148 @@
 
 ## [Unreleased]
 
+### Added
+
+- Claude Code and Codex plugin under `plugins/haiku-rag/`: two client manifests
+  sharing the server configuration and the `haiku-rag` Agent Skill.
+- MCP tool `execute_code(code, filter, sources)`: runs a program in the
+  analysis sandbox over the selected documents and returns what it printed;
+  one sandbox per call.
+- In the analysis sandbox, `search()` results carry `chunk_meta`,
+  `list_documents()` rows and `metadata.json` carry the document `metadata`,
+  and `/documents/{id}/chunks.jsonl` lists chunk ids with their metadata.
+  `recovery_hint` in `haiku.rag.sandbox`.
+- MCP tools `get_document_outline` (heading tree with page numbers) and
+  `get_document_section` (one section's text, subsections included), built
+  on `document_items`. `build_toc` in `haiku.rag.context`.
+- MCP server `instructions`, `version`, and read-only `ToolAnnotations` on
+  every tool; every parameter carries a description. `filter` on
+  `search_documents` and `search_documents_by_image`. `DocumentInfo.metadata`.
+
+### Changed
+
+- `pydantic-monty>=0.0.23`. The analysis sandbox gains `collections`,
+  `itertools`, `functools`, `dataclasses`, function decorators and
+  `str.format`.
+- `fastmcp>=4.0.2,<5.0.0`, on MCP Python SDK 2. The MCP server answers both the
+  session-based and the sessionless (2026-07-28) protocol.
+- Default models are `ollama:qwen3.8`: `ModelConfig`, `qa.model`,
+  `processing.title_model` (was `ollama:gpt-oss`) and
+  `processing.conversion_options.picture_description.model` (was
+  `ollama:ministral-3`). Run `ollama pull qwen3.8`.
+- `qa.model.vision` defaults to `true`, matching `qwen3.8`. Set it `false` when
+  pointing `qa.model` at a text-only model.
+- `enable_thinking` on `provider: ollama` maps to `reasoning_effort` for every
+  model, not only `gpt-oss`: `false` sends `none`, `true` sends `high`.
+  `gpt-oss` keeps `low` for `false`.
+- `processing.conversion_options.picture_description.model` defaults to
+  `enable_thinking: false`, and the field now reaches the VLM: docling's
+  picture-description request carries `reasoning_effort` in `params`.
+- MCP `search_documents` and `search_documents_by_image` expand results to
+  their section (`HaikuRAG.expand_context`) and return the agent rendering
+  as text (rank, `Document ID`, `Collection` over several databases, title,
+  headings, the matched chunk's metadata, passage) and pictures as
+  `ImageContent` blocks, with no structured content.
+  `SearchResult.format_for_agent(include_document_id=, include_chunk_meta=)`;
+  `collect_pictures` in `haiku.rag.tools.search`.
+- MCP tools raise on failure, with the error's message; an empty result no
+  longer doubles as an error.
+- `haiku-rag mcp` covers the configured `lancedb.databases` set. `sources` on
+  `search_documents`, `search_documents_by_image` and `execute_code`; `source`
+  on `get_document`; an unknown name is a tool error. `DocumentInfo.source`.
+
+### Fixed
+
+- `toc.json` `item_range` in the analysis sandbox is a line slice into
+  `items.jsonl`, as documented; it held item positions.
+- Past `analysis.code_timeout` a sandbox program starts no further host call.
+  Files served from memory and in-code `search()` / `list_documents()` were
+  not checked against the deadline.
+
+### Removed
+
+- MCP tools `ask_question` and `analyze`.
+- `format_citations` in `haiku.rag.utils`; `format_citations_rich` stays.
+- MCP write tools `add_document_from_file`, `add_document_from_url`,
+  `add_document_from_text` and `delete_document`. The server opens the
+  database read-only; ingest with `haiku-rag add`, `add-src`, `delete` or
+  `haiku-ingester`. `create_mcp_server` loses `read_only`.
+
+## [0.82.1] - 2026-09-03
+
+### Fixed
+
+- Document timestamps include timezone information and satisfy the MCP
+  date-time output schema. Stored timestamps without an offset are read as
+  host-local time; a database written under a different host timezone shifts
+  by that offset.
+
+## [0.82.0] - 2026-09-03
+
+### Removed
+
+- `lancedb.uri`. Write `lancedb.databases: {NAME: <location>}`; a config carrying
+  `uri` fails to load with that message. Configurations generated by
+  `init-config` through 0.81 carry `uri: ""` and must drop the key.
+- `HAIKU_RAG_DB`. Capabilities cover the databases the configuration places, or
+  the `db_path` argument.
+- `DB_PATH` in the `app/` backend and `examples/custom_agent_agui.py`. Both load
+  the configuration as the CLI does (`HAIKU_RAG_CONFIG_PATH`, `./haiku.rag.yaml`,
+  the platform directory); the compose files set
+  `HAIKU_RAG_CONFIG_PATH=/app/haiku.rag.yaml` and mount `DB_VOLUME` at `/data`.
+
+### Changed
+
+- `qa.max_searches` counts search units: searches a model emits in one
+  response share a unit, up to 3 per unit; sequential searches pay one unit
+  each.
+- Searches in one model response deduplicate their results: evidence a sibling
+  search already showed collapses to a reference line, and a picture attaches
+  once per response.
+- Every database has a name: the key in `lancedb.databases`, or the path's stem
+  for `--db PATH`, `db_path=` and the default database, which is the entry
+  `haiku.rag` under `storage.data_dir` and selectable by that name.
+  `SearchResult.source`, `Document.source` and `Citation.source` carry it on
+  every value a database produces.
+- `db_path=` beside a configured `lancedb.databases` raises
+  `AmbiguousDatabaseError` (`HaikuRAG`, `create_capability`, `create_mcp_server`,
+  `Sandbox`). `haiku-rag --db PATH` and `haiku-ingester --db PATH` open that path
+  whatever is configured.
+- `DatabaseRef(name, location, given)` replaces `DatabaseRef(name, uri, db_path)`;
+  `DatabaseScope.at(path)` added; `locate_database` returns `Path | str`.
+  `IngesterApp(config, scope)` takes a resolved scope in place of `db_path`.
+  `SingleDatabaseSession(ref, config)` replaces `SingleDatabaseSession(db_path,
+  config, source=)`. `Store.db_path` is `None` for a database behind a URI.
+- Opening a configured or default database that does not exist raises
+  `SourceUnavailableError` naming the database and the remedy (`haiku-rag init`
+  or `create=True`), where the default database raised `FileNotFoundError` with
+  its path. A database given as a path still raises `FileNotFoundError`.
+- `Store(location, config)`, `connect_lancedb(location, config)`,
+  `gather_database_info(location, config)` and `run_doctor(config, location, ...)`
+  take the database location, a path or a URI. `ConnectionMode.of(location)`
+  replaces `ConnectionMode.from_config`. `DatabaseRef.connection()` and
+  `default_db_path` removed.
+
+## [0.81.0] - 2026-09-01
+
+### Added
+
+- `search.vector_nprobes` (default 20) sets the IVF partitions each vector query searches.
+
+### Documentation
+
+- `docs/configuration/storage.md` "Vector Indexing" carries the measured with/without IVF_PQ retrieval comparison; `docs/benchmarks.md` states the published numbers are measured without a vector index.
+- Re-indexing note corrected: `optimize()` adds new chunks to an existing vector index, a rebuild retrains centroids.
+
+### Fixed
+
+- Cross-database fusion without a reranker orders the union by cosine
+  similarity to the query, with within-database rank breaking ties, instead
+  of round-robin by database declaration order. Full-text-only searches order
+  by retrieval score. Fused results carry the ordering score.
+
+## [0.80.0] - 2026-08-31
+
 ### Changed
 
 - lancedb 0.37.1.
@@ -13,9 +155,6 @@
   text untruncated. `format_citations_rich` takes a `full` argument.
 - `doctor` fails when the chunks FTS index covers no rows.
 - FTS and hybrid searches log a warning when the FTS index covers no rows.
-- `evaluations run --retrieval-limit N`: candidates each database fetches during the retrieval benchmark, overriding the dataset's `retrieval_limit`.
-- `mtrag_pooled` evaluation dataset and its reference config `evaluations/configs/mtrag_pooled.yaml`: all four MTRAG domains pooled and partitioned across `n` collections, `--alpha` interpolating between one domain per collection and a uniform shard.
-- `mtrag_federated` evaluation dataset and its reference config `evaluations/configs/mtrag_federated.yaml`: MTRAG ClapNQ partitioned by article title into `n` collections, scored on retrieval only with Recall@5/@10, nDCG@5 and MAP. `python -m evaluations.datasets.mtrag_federated --config REF --n N --out PATH` builds the partition and emits the config that searches it.
 
 ### Removed
 
@@ -24,8 +163,6 @@
 
 ### Fixed
 
-- Batched evaluation ingest converts inline content as text instead of letting `HaikuRAG.convert` disambiguate it, so a passage beginning with a URL is stored rather than fetched over HTTP. 187 MTRAG cloud and fiqa passages start with one; no clapnq passage does, so no existing dataset's numbers change.
-- `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results.
 - FTS and hybrid search on a database whose FTS index covers no rows. Chunk
   writes now build the index and rebuild it if it covers none; `haiku-rag
   vacuum` also repairs it.
@@ -2296,7 +2433,11 @@ Existing documents without DoclingDocument data will work but won't have provena
 
 - Initial version tracking
 
-[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.79.0...HEAD
+[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.82.1...HEAD
+[0.82.1]: https://github.com/ggozad/haiku.rag/compare/0.82.0...0.82.1
+[0.82.0]: https://github.com/ggozad/haiku.rag/compare/0.81.0...0.82.0
+[0.81.0]: https://github.com/ggozad/haiku.rag/compare/0.80.0...0.81.0
+[0.80.0]: https://github.com/ggozad/haiku.rag/compare/0.79.0...0.80.0
 [0.79.0]: https://github.com/ggozad/haiku.rag/compare/0.78.0...0.79.0
 [0.78.0]: https://github.com/ggozad/haiku.rag/compare/0.77.0...0.78.0
 [0.77.0]: https://github.com/ggozad/haiku.rag/compare/0.77.0...0.77.0

README.md

@@ -16,7 +16,7 @@ Built on [LanceDB](https://lancedb.com/), [Pydantic AI](https://ai.pydantic.dev/
 - **Hybrid search** — Vector + full-text with Reciprocal Rank Fusion
 - **Multimodal & cross-modal search** — Multimodal embedders (vLLM, VoyageAI, Cohere) put picture vectors in the same space as text; supports text-as-query → figure hits and image-as-query
 - **Question answering** — RAG capability with citations (page numbers, section headings)
-- **Vision QA** — Vision-capable models receive figure bytes alongside chunk text; attach your own images to questions in `ask`, `analyze`, MCP, and the chat TUI
+- **Vision QA** — Vision-capable models receive figure bytes alongside chunk text; attach your own images to questions in `ask`, `analyze` and the chat TUI
 - **Reranking** — local cross-encoders, Cohere, Zero Entropy, or vLLM
 - **Analysis capability** — Complex analytical tasks via sandboxed Python code execution (aggregation, computation, multi-document analysis)
 - **Evidence compaction** — Optional capability that replaces earlier questions' search results on the request with the evidence they cited, so long conversations stop resending everything they retrieved
@@ -110,12 +110,26 @@ For direct agent composition, see the [capabilities documentation](https://ggoza
 
 ## MCP Server
 
-Use with AI assistants like Claude Desktop:
+Use with AI assistants like Claude Code, Codex, and Claude Desktop:
 
 ```bash
 haiku-rag mcp --stdio
 ```
 
+In Claude Code, install the plugin, which registers the server and a skill:
+
+```bash
+claude plugin marketplace add ggozad/haiku.rag
+claude plugin install haiku-rag
+```
+
+In Codex, install the same plugin from its marketplace:
+
+```bash
+codex plugin marketplace add ggozad/haiku.rag
+codex plugin add haiku-rag@haiku-rag
+```
+
 Add to your Claude Desktop configuration:
 
 ```json
@@ -129,7 +143,7 @@ Add to your Claude Desktop configuration:
 }
 ```
 
-Provides tools for document management, search, QA, and analysis directly in your AI assistant.
+Provides search, document reading, and analysis tools directly in your AI assistant.
 
 ## Examples
 

app/.env.example

@@ -2,8 +2,9 @@
 ANTHROPIC_API_KEY=your-anthropic-key
 OPENAI_API_KEY=your-openai-key
 
-# Database path
-DB_PATH=/path/to/your/haiku.rag.lancedb
+# Host path of the LanceDB database, mounted at /data where haiku.rag.yaml
+# places it
+DB_VOLUME=./data/haiku.rag.lancedb
 
 # Optional: Ollama base URL (if using local models)
 # Use host.docker.internal to reach Ollama running on the host machine

app/README.md

@@ -40,7 +40,8 @@ A conversational RAG interface built with [CopilotKit](https://copilotkit.ai/) a
 
 | Variable | Description | Required |
 |----------|-------------|----------|
-| `DB_PATH` | Path to your haiku.rag LanceDB database | Yes |
+| `DB_VOLUME` | Host path of the LanceDB database the compose files mount at `/data`, where `haiku.rag.yaml` places it (default `./data/haiku.rag.lancedb`) | No |
+| `HAIKU_RAG_CONFIG_PATH` | The configuration file; the compose files set it to the mounted `/app/haiku.rag.yaml` | No |
 | `ANTHROPIC_API_KEY` | Anthropic API key | One LLM key required |
 | `OPENAI_API_KEY` | OpenAI API key | One LLM key required |
 | `OLLAMA_BASE_URL` | Ollama server URL (default: `http://host.docker.internal:11434`) | For local models |

app/backend/main.py

@@ -1,9 +1,7 @@
 import asyncio
 import logging
-import os
 from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
-from pathlib import Path
 from typing import Any
 
 from ag_ui.core import EventType, StateSnapshotEvent
@@ -26,8 +24,8 @@ from haiku.rag.capabilities.policy import (
 )
 from haiku.rag.capabilities.rag import AGENT_PREAMBLE, RAGState, create_capability
 from haiku.rag.client import HaikuRAG
-from haiku.rag.config import load_yaml_config
-from haiku.rag.config.models import AppConfig
+from haiku.rag.client.scope import DatabaseScope
+from haiku.rag.config import get_config
 from haiku.rag.telemetry import configure as configure_telemetry
 from haiku.rag.utils import get_model
 
@@ -40,19 +38,23 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-# Load config
-config_path = Path("/app/haiku.rag.yaml")
-if config_path.exists():
-    yaml_data = load_yaml_config(config_path)
-    config = AppConfig.model_validate(yaml_data)
-else:
-    config = AppConfig()
+# The configuration places the database. This app serves one.
+config = get_config()
+scope = DatabaseScope.resolve(config)
+if scope.covers_multiple:
+    raise SystemExit(
+        f"lancedb.databases names {', '.join(scope.names)}; this app serves one "
+        "database: configure exactly one entry"
+    )
+[database] = scope.databases
 
-# Get DB path from environment
-db_path_str = os.getenv("DB_PATH", "haiku_rag.lancedb")
-db_path = Path(db_path_str)
 
-logger.info(f"Database path: {db_path}")
+def _database_exists() -> bool:
+    """A database behind a URI has no path to check."""
+    return database.db_path is None or database.db_path.exists()
+
+
+logger.info(f"Database: {database.name} at {database.location}")
 logger.info(f"QA Provider: {config.qa.model.provider}, Model: {config.qa.model.name}")
 
 # Only HaikuRAG client is a singleton (expensive to create)
@@ -71,7 +73,7 @@ async def get_client() -> HaikuRAG:
     if _client is None:
         async with _client_lock:
             if _client is None:
-                client = HaikuRAG(db_path=db_path, config=config, create=True)
+                client = HaikuRAG(config=config, create=True)
                 await client.__aenter__()
                 _client = client
     return _client
@@ -82,7 +84,7 @@ class AppDeps:
     state: dict[str, Any] = field(default_factory=dict)
 
 
-capability = create_capability(db_path=db_path, config=config, defer_loading=False)
+capability = create_capability(config=config, defer_loading=False)
 
 agent = Agent(
     get_model(config.qa.model, config),
@@ -138,15 +140,15 @@ async def health_check(_: Request) -> JSONResponse:
             "status": "healthy",
             "qa_provider": config.qa.model.provider,
             "qa_model": config.qa.model.name,
-            "db_path": str(db_path),
-            "db_exists": db_path.exists(),
+            "db_path": str(database.location),
+            "db_exists": _database_exists(),
         }
     )
 
 
 async def list_documents(_: Request) -> JSONResponse:
     """List all documents in the database."""
-    if not db_path.exists():
+    if not _database_exists():
         return JSONResponse({"documents": [], "error": "Database not found"})
 
     client = await get_client()
@@ -162,11 +164,11 @@ async def list_documents(_: Request) -> JSONResponse:
 
 async def db_info(_: Request) -> JSONResponse:
     """Get database info and statistics."""
-    if not db_path.exists():
+    if not _database_exists():
         return JSONResponse(
             {
                 "exists": False,
-                "path": str(db_path),
+                "path": str(database.location),
                 "documents": 0,
                 "chunks": 0,
             }
@@ -180,7 +182,7 @@ async def db_info(_: Request) -> JSONResponse:
     return JSONResponse(
         {
             "exists": True,
-            "path": str(db_path),
+            "path": str(database.location),
             "documents": stats["documents"].get("num_rows", 0),
             "chunks": stats["chunks"].get("num_rows", 0),
             "documents_bytes": stats["documents"].get("total_bytes", 0),
@@ -214,7 +216,7 @@ async def visualize_chunk(request: Request) -> JSONResponse:
         if isinstance(parsed, list):
             refs = [str(x) for x in parsed]
 
-    if not db_path.exists():
+    if not _database_exists():
         return JSONResponse({"error": "Database not found"}, status_code=404)
 
     client = await get_client()

app/backend/pyproject.toml

@@ -8,7 +8,7 @@ dependencies = [
     "uvicorn[standard]>=0.40.0",
     "pydantic-ai-slim[ag-ui,anthropic,openai]>=2.18.0,<3.0.0",
     "python-dotenv>=1.2.1",
-    "haiku.rag-slim>=0.79.0",
+    "haiku.rag-slim>=0.82.1",
     "logfire[pydantic-ai]>=3.17.0",
 ]
 

app/docker-compose.dev.yml

@@ -11,13 +11,14 @@ services:
     ports:
       - "127.0.0.1:8001:8000"
     environment:
-      - DB_PATH=/data
+      - HAIKU_RAG_CONFIG_PATH=/app/haiku.rag.yaml
       - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
       - OPENAI_API_KEY=${OPENAI_API_KEY:-}
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434}
       - LOGFIRE_TOKEN=${LOGFIRE_TOKEN:-}
     volumes:
-      - ${DB_PATH:-./data/haiku.rag.lancedb}:/data
+      # haiku.rag.yaml places the database at /data.
+      - ${DB_VOLUME:-./data/haiku.rag.lancedb}:/data
       - ./backend:/app/src:ro
       - ./haiku.rag.yaml:/app/haiku.rag.yaml:ro
     extra_hosts:

app/docker-compose.yml

@@ -7,13 +7,14 @@ services:
     ports:
       - "127.0.0.1:8001:8000"
     environment:
-      - DB_PATH=/data
+      - HAIKU_RAG_CONFIG_PATH=/app/haiku.rag.yaml
       - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
       - OPENAI_API_KEY=${OPENAI_API_KEY:-}
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434}
       - LOGFIRE_TOKEN=${LOGFIRE_TOKEN:-}
     volumes:
-      - ${DB_PATH:-./data/haiku.rag.lancedb}:/data
+      # haiku.rag.yaml places the database at /data.
+      - ${DB_VOLUME:-./data/haiku.rag.lancedb}:/data
       - ./haiku.rag.yaml:/app/haiku.rag.yaml:ro
     extra_hosts:
       - "host.docker.internal:host-gateway"

app/haiku.rag.yaml.example

@@ -1,6 +1,12 @@
 # haiku.rag configuration for the chat app
 # Copy to haiku.rag.yaml and customize as needed
 
+# The database. The compose files mount DB_VOLUME (default
+# ./data/haiku.rag.lancedb) at /data.
+lancedb:
+  databases:
+    haiku.rag: /data
+
 # QA model configuration
 qa:
   model:

docker/Dockerfile

@@ -40,4 +40,4 @@ EXPOSE 8001 8765
 
 # Default command: read-only MCP server. The companion ingester service is
 # launched via docker-compose against the same image.
-CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "--read-only", "mcp", "--port", "8001"]
+CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "mcp", "--port", "8001"]

docker/Dockerfile.slim

@@ -39,4 +39,4 @@ EXPOSE 8001 8765
 
 # Default command: read-only MCP server. The companion ingester service is
 # launched via docker-compose against the same image.
-CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "--read-only", "mcp", "--port", "8001"]
+CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "mcp", "--port", "8001"]

docs/apps.md

@@ -40,8 +40,8 @@ Create a `.env` file in the `app/` directory:
 ANTHROPIC_API_KEY=your-anthropic-key
 OPENAI_API_KEY=your-openai-key
 
-# Database path
-DB_PATH=/path/to/your/haiku.rag.lancedb
+# Host path of the LanceDB database, mounted at /data
+DB_VOLUME=/path/to/your/haiku.rag.lancedb
 
 # Optional: Ollama base URL (if using local models)
 OLLAMA_BASE_URL=http://localhost:11434
@@ -50,16 +50,22 @@ OLLAMA_BASE_URL=http://localhost:11434
 LOGFIRE_TOKEN=your-logfire-token
 ```
 
-For full configuration, mount a `haiku.rag.yaml` file:
+The mounted `haiku.rag.yaml` places the database at `/data` and configures the models; the compose files point `HAIKU_RAG_CONFIG_PATH` at it:
 
 ```yaml
 # app/haiku.rag.yaml
+lancedb:
+  databases:
+    haiku.rag: /data
+
 qa:
   model:
     provider: anthropic
     name: claude-sonnet-4-20250514
 ```
 
+Outside compose, the backend loads its configuration like the CLI: `HAIKU_RAG_CONFIG_PATH`, then `./haiku.rag.yaml`, then the platform directory.
+
 ## API endpoints
 
 | Endpoint | Method | Description |

docs/benchmarks.md

@@ -6,6 +6,8 @@ We evaluate `haiku.rag` on a small set of datasets that exercise different parts
 
 Numbers below were measured on a recent `haiku.rag` version. Most rows were judged by `Qwen3.6-35B-A3B-NVFP4`; the `Qwen3.8-27B` rows were judged by the currently pinned `qwen3.8`, as their footnote states. Rows are not re-judged when the pinned judge changes, so compare rows judged by the same judge and treat cross-judge differences as unmeasured.
 
+No benchmark database carries a vector index, so every number below reflects exact brute-force kNN rather than approximate search. A vector index is never built automatically. `haiku-rag create-index` builds one, and `haiku-rag doctor` reports whether a database has it. For the measured effect of indexing on retrieval, see [Vector Indexing](configuration/storage.md#vector-indexing).
+
 ### OpenRAG Bench (ORB)
 
 [OpenRAG Bench](https://huggingface.co/datasets/vectara/open_ragbench) contains ArXiv research papers with multimodal question-answering pairs. Queries include both text-based and image-based questions, testing retrieval and reasoning over visual content like figures, charts, and diagrams. Each query maps to one relevant document.

docs/capabilities/analysis.md

@@ -16,7 +16,7 @@ When `qa.max_searches` or `analysis.max_executions` runs out, the exhausted tool
 | `analysis_execute_code(code)` | Run Python against the virtual document filesystem. |
 | `analysis_cite(chunk_ids)` | Register retrieved or filesystem-derived chunk IDs. |
 
-The sandbox exposes documents under `/documents/{document_id}/` with `metadata.json`, `content.txt`, `items.jsonl`, and `toc.json`.
+The sandbox exposes documents under `/documents/{document_id}/` with `metadata.json`, `content.txt`, `items.jsonl`, `chunks.jsonl` (chunk ids with their metadata) and `toc.json`. In code, `await search()` results carry `chunk_meta` and `await list_documents()` rows carry `metadata`. The interpreter's limits and the per-call budgets are listed under [MCP, Code](../mcp.md#code).
 
 ## Compose an agent
 

docs/capabilities/index.md

@@ -145,12 +145,6 @@ Applications serving AG-UI should adapt the agent with Pydantic AI's `AGUIAdapte
 
 ## Database Selection
 
-RAG and analysis capabilities select databases in this order:
-
-1. The `db_path` argument.
-2. `HAIKU_RAG_DB`.
-3. [`lancedb.databases`](../configuration/storage.md#multiple-databases), which selects the full configured set.
-4. [`lancedb.uri`](../configuration/storage.md#changing-the-default-database-path), which selects one database.
-5. `config.storage.data_dir / "haiku.rag.lancedb"`.
+RAG and analysis capabilities cover the databases the configuration places: [`lancedb.databases`](../configuration/storage.md#multiple-databases), or with nothing configured the default database `haiku.rag` under `storage.data_dir`. The `db_path` argument places one database where the configuration places none; beside `lancedb.databases` it raises `AmbiguousDatabaseError`.
 
 Passing a client through `rag=` bypasses this selection. The capability uses the databases covered by that client and does not close it.

docs/cli.md

@@ -12,7 +12,7 @@ The `haiku-rag` CLI provides complete document management functionality.
 
     Per-command options:
 
-    - `--db` - Specify custom database path
+    - `--db` - Open the database at this path, named by its stem, whatever the configuration places
     - `-h` - Show help for specific command
 
     Example:
@@ -24,7 +24,7 @@ The `haiku-rag` CLI provides complete document management functionality.
     haiku-rag add -h
     ```
 
-    With `lancedb.databases` configured, `search`, `ask`, `analyze`, and `chat` use the full set by default. Select one database for other commands with `--db-name` or `--db`. `settings`, `init-config`, and `download-models` do not open a database. See [Multiple Databases](configuration/storage.md#multiple-databases).
+    With `lancedb.databases` configured, `search`, `ask`, `analyze`, `chat`, and `mcp` use the full set by default. Select one database for other commands with `--db-name` or `--db`. `settings`, `init-config`, and `download-models` do not open a database. See [Multiple Databases](configuration/storage.md#multiple-databases).
 
 ## Document Management
 
@@ -402,15 +402,16 @@ haiku-rag create-index [--db /path/to/your.lancedb]
 - Creates an IVF_PQ index using the configured `search.vector_index_metric` (cosine/l2)
 
 **When to use:**
-- After ingesting documents (indexes are not created automatically)
-- After adding significant new data to rebuild the index
-- Use `haiku-rag info` to check index status and see how many chunks are indexed/unindexed
+- On a collection over 100,000 chunks (below that, brute-force kNN is exact and fast enough)
+- After substantial corpus growth, to retrain the centroids
+- Use `haiku-rag info` to check index status and see how many chunks are indexed/unindexed, or `haiku-rag doctor` for the same as a health check
+
+See [Vector Indexing](configuration/storage.md#vector-indexing) for the measured accuracy and build cost.
 
 **Search behavior:**
 - Without index: Brute-force kNN search (exact nearest neighbors, slower for large datasets)
-- With index: Fast ANN (approximate nearest neighbors) using IVF_PQ
-- With stale index: LanceDB combines indexed results (fast ANN) + brute-force kNN on unindexed rows
-- Performance degrades as more unindexed data accumulates
+- With index: ANN (approximate nearest neighbors) using IVF_PQ, tuned by `search.vector_nprobes`
+- Between a write and the next `optimize()`: LanceDB combines ANN over indexed rows with brute-force kNN over the remainder
 
 ### Rebuild Database
 
@@ -476,9 +477,6 @@ haiku-rag mcp --port 9000
 
 # Bind to all interfaces (containers, trusted LAN)
 haiku-rag mcp --host 0.0.0.0
-
-# Read-only mode (no write tools)
-haiku-rag --read-only mcp
 ```
 
 See [MCP](mcp.md) for details. For continuous document ingestion

docs/configuration/index.md

@@ -61,7 +61,7 @@ embeddings:
 qa:
   model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     enable_thinking: true
 ```
 
@@ -85,10 +85,9 @@ ingester:
       delete_orphans: true
 
 lancedb:
-  uri: ""  # Empty for local, or db://, s3://, az://, gs://
-  api_key: ""
+  databases: {}  # Name-to-location map; empty places haiku.rag under data_dir
+  api_key: ""  # LanceDB Cloud (db://) credentials
   region: ""
-  databases: {}  # Name-to-location map to search multiple at once; excludes uri
 
 embeddings:
   model:
@@ -106,7 +105,7 @@ reranking:
 qa:
   model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     enable_thinking: true
     temperature: 0.3
   max_searches: 5
@@ -136,7 +135,7 @@ processing:
   auto_title: false              # Auto-generate titles on ingestion
   title_model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     enable_thinking: false
     temperature: 0.3
     max_tokens: 100

docs/configuration/processing.md

@@ -30,7 +30,7 @@ processing:
   auto_title: false                          # Auto-generate titles on ingestion
   title_model:                               # LLM for title generation (fallback)
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     enable_thinking: false
 
   # Conversion options (works with both local and remote converters)
@@ -54,7 +54,7 @@ processing:
     picture_description:
       model:
         provider: ollama
-        name: ministral-3
+        name: qwen3.8
   pictures: image                            # none | description | image
 ```
 
@@ -270,7 +270,7 @@ processing:
     picture_description:          # only consulted when pictures == "description"
       model:
         provider: ollama          # any OpenAI-compatible /v1/chat/completions provider
-        name: ministral-3
+        name: qwen3.8
       timeout: 90
       max_tokens: 200
 ```
@@ -294,7 +294,7 @@ Three independent settings drive ingest, retrieval, and QA:
 |---|---|---|
 | `processing.pictures` | Generate and/or describe pictures at ingest? | `none` / `description` / `image` (default) |
 | `embeddings.model.multimodal` | Can the embedder index image content? | `false` (default, text-only) / `true` (supported on `vllm`, `voyageai`, `cohere`) |
-| `qa.model.vision` | Can the QA model interpret images? | `false` (default) / `true` |
+| `qa.model.vision` | Can the QA model interpret images? | `false` / `true` (default) |
 
 The Embedder column below is driven by `embeddings.model.multimodal`, not the provider name — a vision-capable model under a text-only configuration still indexes no images, and an image-only document then produces zero chunks. See [Multimodal embedders](providers.md#multimodal-embedders).
 
@@ -313,7 +313,7 @@ The Embedder column below is driven by `embeddings.model.multimodal`, not the pr
 - `qa.model.vision: false` — text chunks only (descriptions, when present, answer figure questions in prose).
 - `qa.model.vision: true` — text chunks + raw picture bytes via `BinaryContent`. The model reads figures directly. Requires `pictures != none` so the bytes exist.
 
-`qa.model.vision` is independent of ingestion. Flipping it never requires reingesting. Setting `vision: true` against a text-only model causes silent acceptance and confabulation on Ollama and a 400 on OpenAI. Default `false` is the safe choice.
+`qa.model.vision` is independent of ingestion. Flipping it never requires reingesting. It declares what the model can read: the default `qwen3.8` is vision-capable, so the default is `true`. Set it `false` when pointing `qa.model` at a text-only model, where `true` causes silent acceptance and confabulation on Ollama and a 400 on OpenAI.
 
 **Recommended combinations:**
 
@@ -368,7 +368,7 @@ processing:
   auto_title: true
   title_model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     enable_thinking: false
 ```
 

docs/configuration/providers.md

@@ -15,7 +15,7 @@ Configure model behavior for the `qa` and `analysis` capabilities. These setting
 qa:
   model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     temperature: 0.3
     max_tokens: 500
 ```
@@ -79,7 +79,7 @@ See the [Pydantic AI thinking documentation](https://ai.pydantic.dev/thinking/)
 - **Google**: Gemini models with thinking support
 - **Groq**: Models with reasoning capabilities
 - **Bedrock**: Claude, Qwen, and `gpt-oss` models. Bedrock Converse does not serve the proprietary OpenAI models, so configuring one raises an error. Reach those through `provider: bedrock-mantle`.
-- **Ollama**: Models supporting reasoning (gpt-oss, etc.)
+- **Ollama**: Any model with a thinking capability. `enable_thinking` maps to `reasoning_effort`: `false` sends `none` (`low` for `gpt-oss`, whose template has no `none` level), `true` sends `high`.
 - **vLLM**: Models with a pydantic-ai reasoning profile (gpt-oss). Qwen3, Gemma, and similar templates ignore the OpenAI `reasoning_effort` that `enable_thinking` translates to — use [`extra_body`](#raw-provider-pass-through) to drive them.
 - **LM Studio**: Models supporting reasoning (gpt-oss, etc.)
 
@@ -311,7 +311,7 @@ Configure which LLM provider to use for question answering. Any provider and mod
 qa:
   model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
 ```
 
 The Ollama base URL can be configured via the `OLLAMA_BASE_URL` environment variable, config file, or defaults to `http://localhost:11434`:

docs/configuration/qa.md

@@ -20,22 +20,22 @@ Context expansion is automatic and section-aware. For structured documents (with
 
 ## Question Answering Configuration
 
-Configure the RAG capability (used by `client.ask`, `haiku-rag ask`, and the MCP `ask_question` tool):
+Configure the RAG capability (used by `client.ask` and `haiku-rag ask`):
 
 ```yaml
 qa:
   model:
     provider: ollama
-    name: gpt-oss
+    name: qwen3.8
     enable_thinking: true
     temperature: 0.3          # Default: 0.3
-    vision: false             # Set true for vision-capable models
-  max_searches: 5       # Maximum search tool calls per question
+    vision: true              # Set false for text-only models
+  max_searches: 5       # Maximum search units per question
 ```
 
 - **model**: LLM configuration (see [Providers](providers.md#model-settings))
 - **model.vision**: Set to `true` for vision-capable models (`qwen2.5vl`, `qwen3.6`, `gpt-4o`, `claude-sonnet`, …). The capability's `search` tool only attaches picture bytes (`BinaryContent`) to its `ToolReturn` when this is `true`, otherwise picture bytes are withheld. See [Pictures × embedder × QA model](processing.md#pictures-embedder-qa-model-how-the-pieces-compose) for the full matrix.
-- **max_searches**: Maximum number of search tool calls a capability can make per question (default: 5). Shared by the RAG and analysis capabilities.
+- **max_searches**: Maximum number of search units a capability can spend per question (default: 5). Up to three searches emitted in the same model response share one unit, so a model that rephrases its query in one response spends one unit. A search in a later response starts a new unit, as does each further group of three within one response. Shared by the RAG and analysis capabilities. Searches in one response also deduplicate their returns: evidence a sibling search already showed collapses to a reference line, and each picture attaches once per response.
 
 !!! note "Thinking on vLLM"
     `enable_thinking` only applies to models with a pydantic-ai reasoning profile (o-series, gpt-5, gpt-oss). For other vLLM-served models such as Qwen3 or the Gemma family, the field is a silent no-op — set the chat template switch via [`extra_body`](providers.md#raw-provider-pass-through) instead.
@@ -50,13 +50,13 @@ analysis:
     provider: anthropic
     name: claude-sonnet-4-20250514
     temperature: 0.0        # Default: 0.0 (deterministic for code generation)
-  code_timeout: 60.0      # Max seconds a call may spend reading documents
+  code_timeout: 60.0      # Per call: compute stops, no read or search starts past it
   max_output_chars: 50000 # Truncate output after this many chars
   max_executions: 15      # Max execute_code calls per question
 ```
 
 - **model**: LLM configuration (see [Providers](providers.md#model-settings)). When unset, falls back to `qa.model`.
-- **code_timeout**: Seconds a single `execute_code` call may spend reading documents (default: 60). The sandbox refuses further reads past this point. Code that computes without reading is killed by the worker watchdog at the same limit. `code_timeout * max_executions` is the cumulative ceiling across all calls in one question.
+- **code_timeout**: Seconds a single `execute_code` call has (default: 60). Past it the sandbox starts no further host call, a document read or an in-code `search()` / `list_documents()`; one already running finishes. Code that computes without host calls is killed by the worker watchdog at the same limit. `code_timeout * max_executions` is the cumulative ceiling across all calls in one question.
 - **max_output_chars**: Truncate code output after this many characters (default: 50000)
 - **max_executions**: Maximum `execute_code` calls per question before the capability is told to answer from what it has (default: 15)
 

docs/configuration/storage.md

@@ -68,18 +68,19 @@ Vacuum also folds new rows into the full-text index. Search stays correct withou
 
 This is an upstream limitation rather than a `haiku.rag` setting. Compaction bounds itself by row count instead of bytes, and LanceDB's async API exposes no batch size or fragment target to override it. Tracked at [lancedb/lancedb#2325](https://github.com/lancedb/lancedb/issues/2325). The requirement above will drop once compaction batches by bytes.
 
-### Changing the Default Database Path
+### Placing the Database
 
-`storage.data_dir` holds the default database, always called `haiku.rag.lancedb`. To put the database somewhere else for every command, give `lancedb.uri` a local path:
+`lancedb.databases` maps a name to a location, a local path or a URI, and is the one way to place databases. With nothing configured, the database is the entry `haiku.rag` at `<storage.data_dir>/haiku.rag.lancedb`. To put one database somewhere else, name it:
 
 ```yaml
 lancedb:
-  uri: /data/notes.lancedb
+  databases:
+    notes: /data/notes.lancedb
 ```
 
-An explicit `--db PATH` overrides `lancedb.uri` for that invocation.
+The name is what `source` carries in search results, citations and documents, and what `--db-name` and `sources` select. The default database answers to `haiku.rag`.
 
-This places one database without naming it. Its `source` is `None` in search results, citations and documents, since only [`lancedb.databases`](#multiple-databases) assigns the names that carry provenance. A path here changes where the database lives, not what it is called.
+An explicit `--db PATH` on the command line opens that database instead, named by the path's stem, whatever is configured. From Python, `db_path` places the database only where the configuration places none: beside `lancedb.databases` it raises `AmbiguousDatabaseError`.
 
 A value with no scheme is a local path wherever it is configured, so `haiku-rag init` creates it and every command that opens an existing database requires it to exist. A mistyped path fails rather than becoming a new empty database.
 
@@ -109,28 +110,31 @@ async with HaikuRAG(create=True) as client:
 
 The [default location](index.md#configuration-file-locations) is platform-specific (e.g., `~/Library/Application Support/haiku.rag/` on macOS).
 
-Opening a nonexistent unnamed local database raises `FileNotFoundError`, naming its path. This prevents accidental database creation from typos or misconfigured paths. A database named in `lancedb.databases` raises `SourceUnavailableError` instead, naming the database and not its location.
+Opening a nonexistent local database given as a path raises `FileNotFoundError`, naming the path. This prevents accidental database creation from typos or misconfigured paths. A configured or default database raises `SourceUnavailableError` instead, naming the database and not its location.
 
 ## Remote Storage
 
-For remote storage, use the `lancedb` settings with various backends:
+For remote storage, give the database a URI as its location. Credentials and storage options are connection settings, shared by every database in the configuration:
 
 ```yaml
 # LanceDB Cloud
 lancedb:
-  uri: db://your-database-name
+  databases:
+    papers: db://your-database-name
   api_key: your-api-key
-  region: us-west-2  # optional
+  region: us-west-2
 
 # Amazon S3
 lancedb:
-  uri: s3://my-bucket/my-table
+  databases:
+    papers: s3://my-bucket/my-table
   storage_options:
     region: us-east-1
 
 # Amazon S3 with explicit credentials
 lancedb:
-  uri: s3://my-bucket/my-table
+  databases:
+    papers: s3://my-bucket/my-table
   storage_options:
     aws_access_key_id: YOUR_ACCESS_KEY
     aws_secret_access_key: YOUR_SECRET_KEY
@@ -138,7 +142,8 @@ lancedb:
 
 # S3-compatible (SeaweedFS, Tigris, etc.)
 lancedb:
-  uri: s3://my-bucket/my-table
+  databases:
+    papers: s3://my-bucket/my-table
   storage_options:
     endpoint: http://localhost:8333
     aws_access_key_id: YOUR_ACCESS_KEY
@@ -148,21 +153,24 @@ lancedb:
 
 # Azure Blob Storage
 lancedb:
-  uri: az://my-container/my-table
+  databases:
+    papers: az://my-container/my-table
 
 # Google Cloud Storage
 lancedb:
-  uri: gs://my-bucket/my-table
+  databases:
+    papers: gs://my-bucket/my-table
 
 # HDFS
 lancedb:
-  uri: hdfs://namenode:port/path/to/table
+  databases:
+    papers: hdfs://namenode:port/path/to/table
 ```
 
 - **LanceDB Cloud** (`db://`): Requires `api_key` and `region`. Table optimization and indexing are managed server-side.
 - **Object storage** (`s3://`, `gs://`, `az://`, `hdfs://`): Uses `storage_options` for credentials and endpoint configuration. Authentication can also be provided via environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc.) or cloud provider SDK defaults (AWS CLI, Azure CLI, gcloud).
 - **S3-compatible stores** (MinIO, Tigris, etc.): Set `endpoint` in `storage_options`. When using `http://` endpoints, also set `allow_http: "true"`.
-- **Local path** (no scheme): `uri` also takes a local path, which is how the default database is pointed elsewhere. See [Changing the Default Database Path](#changing-the-default-database-path).
+- **Local path** (no scheme): a location without a scheme is a local path. See [Placing the Database](#placing-the-database).
 
 The `storage_options` keys are case-insensitive and passed directly to the underlying object store library. Available keys depend on the backend. See the [LanceDB storage docs](https://lancedb.com/docs/storage/) for details.
 
@@ -188,11 +196,11 @@ writing process per database URI, any number of read-only consumers.
 The recommended layout for production is "different buckets, same account, separate IAM roles per process":
 
 - **Ingestion process** — IAM role with `s3:Get/List` on the documents bucket and `s3:Get/Put/Delete` on the LanceDB bucket. Runs `haiku-ingester serve` (with `ingester.sources[type=s3]` pointing at the documents bucket). Exactly one such process per LanceDB URI.
-- **Consumer processes** (1..N) — IAM role with `s3:Get/List` on the LanceDB bucket only. Run `haiku-rag --read-only mcp`, the chat TUI, etc. They never see the documents bucket.
+- **Consumer processes** (1..N) — IAM role with `s3:Get/List` on the LanceDB bucket only. Run `haiku-rag mcp`, the chat TUI, etc. They never see the documents bucket.
 
 Each process picks up its own credentials from the AWS default chain (env vars, IAM instance role, AWS profile), so no credentials are hard-coded in the configuration files.
 
-`haiku-ingester` writes the database the configuration places, so a `lancedb.uri` needs no further option. `--db PATH` overrides it. When `lancedb.databases` contains more than one database the ingester has no way to name which it writes, and refuses to start with `AmbiguousDatabaseError`: give each database its own ingester process, each with a configuration naming a single database, or select one with `--db PATH`. A one-entry mapping is unambiguous and is accepted.
+`haiku-ingester` writes the database the configuration places, so a one-entry `lancedb.databases` needs no further option. `--db PATH` overrides it. When `lancedb.databases` contains more than one database the ingester has no way to name which it writes, and refuses to start with `AmbiguousDatabaseError`: give each database its own ingester process, each with a configuration naming a single database, or select one with `--db PATH`.
 
 ## Multiple Databases
 
@@ -206,7 +214,7 @@ lancedb:
     notes: /data/notes.lancedb
 ```
 
-A location can be a URI or local path. `databases` and `uri` are mutually exclusive.
+A location can be a URI or local path.
 
 Results, documents, and citations use the configured name as `source`. An unavailable configured database raises `SourceUnavailableError`, which names the database and not its location, so a location never travels in an error a consumer might render or log. A migration, configuration or read-only failure keeps its own type, with the database named in the message. Commands that report on a database, such as `info`, still show where it is.
 
@@ -227,7 +235,7 @@ results = await client.search("query")                     # every database
 results = await client.search("query", sources=["papers"])   # one of them
 ```
 
-Candidates are combined into one ranked list with the configured reranker, or with reciprocal rank fusion when reranking is disabled. `SearchResult.source`, `Citation.source`, and `Document.source` contain the configured database name. The name is retained when a client covers only one named database. Databases configured through `lancedb.uri` are unnamed, so their `source` is `None`.
+Candidates are combined into one ranked list with the configured reranker, or by cosine similarity to the query when reranking is disabled, with within-database rank breaking ties (full-text-only searches order by retrieval score). `SearchResult.source`, `Citation.source`, and `Document.source` carry the database name, for a set and for one database alike.
 
 The CLI labels results and citations only when the operation spans multiple databases. A command already narrowed with `--db-name` does not repeat the name on every result.
 
@@ -245,17 +253,13 @@ The chat document filter selects by document and database: the search is narrowe
 
 #### Ranking
 
-Reciprocal rank fusion compares positions rather than scores, so each database contributes top-ranked results even when another database has stronger matches. A reranker scores the combined candidate set directly, which has been measured to help aggregate retrieval and to hurt attribution between near-identical documents.
+Without a reranker, the fused list is ordered by cosine similarity between the query vector and each candidate. The databases in a selection share an embedder, so similarity in that one space is comparable across databases, where retrieval scores are each database's own arithmetic. Ties resolve by the candidate's rank within its own database, and configured order decides only when both tie. Similarities rarely tie exactly, so declaration order decides almost nothing: on MTRAG retrieval benchmarks, reversing it left recall unchanged in every cell. Full-text-only searches have no query vector and order by retrieval score instead.
 
-Aggregate retrieval is stronger with a reranker. In a 3,045-query evaluation over a corpus split across three databases, reranking produced retrieval MAP 0.9914, compared with 0.9918 for the same corpus in one database. Without a reranker, MAP was 0.6044, compared with 0.9798 in one database. Reranking cost grows with the number of databases because each contributes candidates.
+Results are not guaranteed to spread across databases: a database with nothing relevant to a query contributes nothing, and a strong database can fill every slot. On MTRAG retrieval benchmarks over two to eight collections, cosine fusion holds recall roughly flat as collections are added, where position-based fusion lost up to half its recall at eight.
 
-A reranker scores the combined candidates with no notion of which database each came from, so on near-identical text it can pick the wrong database's chunk, where fusion keeps them apart because each database contributes its own top-ranked result. In two nine-case acceptance runs over a synthetic corpus holding one station in two databases under near-identical names, attribution was weaker with reranking: citing the right database succeeded 5 of 9 and 6 of 9 times with a reranker, against 8 of 9 and 9 of 9 without.
+A configured reranker scores the combined candidate set directly, ignoring which database each candidate came from, and remains the strongest option: roughly 6 to 8 recall points above cosine fusion on the same benchmarks. Its cost grows with the number of databases because each contributes candidates.
 
-Configure a reranker where retrieval breadth matters, and measure it where answers have to attribute between documents that read alike.
-
-Without a reranker, consider increasing `search.limit` with the number of databases. With three complete rankings and a limit of 5, a database may contribute only one or two results. A higher limit also sends more results to the caller and model.
-
-Image queries are vector-only and skip the reranker: there is no query text to score a document against, so candidates keep their vector ranking and fusion ranks by position.
+Image queries are vector-only and skip the reranker: the reranker interface takes a text query, and multimodal reranking applies to pictures on the candidate side, not to image queries. Their fused list is ordered by cosine similarity like any other vector search.
 
 If a selected database is unavailable, the operation fails with `SourceUnavailableError`, which names that database.
 
@@ -277,9 +281,9 @@ Conversion, chunking, and title generation do not access a database and remain a
 
 Commands use database sets as follows:
 
-- **Set-capable**: `search`, `ask`, `analyze`, and `chat` use the full configured set, or the single database selected by `--db-name`.
+- **Set-capable**: `search`, `ask`, `analyze`, `chat`, and `mcp` use the full configured set, or the single database selected by `--db-name`.
 - **Config-only**: `settings`, `init-config`, and `download-models` do not open a database.
-- **Single-database**: everything else — document writes, `rebuild`, `vacuum`, `migrate`, `init`, `info`, `history`, `tag`, `doctor`, `list`, `inspect`, `visualize`, and `mcp` — works on one database, selected with the global `--db-name` option.
+- **Single-database**: everything else — document writes, `rebuild`, `vacuum`, `migrate`, `init`, `info`, `history`, `tag`, `doctor`, `list`, `inspect`, and `visualize` — works on one database, selected with the global `--db-name` option.
 
 ```bash
 haiku-rag search "query"         # every configured database
@@ -287,7 +291,7 @@ haiku-rag --db-name papers list  # one of them
 haiku-rag --db-name papers migrate
 ```
 
-`--db-name` selects an entry from `lancedb.databases`, including remote entries. `--db` selects a local path and overrides the configured location. A single-database command requires one of these options when multiple databases are configured. A configured set of one is selected automatically.
+`--db-name` selects an entry from `lancedb.databases`, including remote entries, and `haiku.rag` when nothing is configured. `--db` opens a local path, named by its stem, whatever is configured. A single-database command requires one of these options when multiple databases are configured. A configured set of one is selected automatically.
 
 Each database is created, migrated and vacuumed on its own:
 
@@ -304,6 +308,7 @@ Configure vector search settings:
 search:
   vector_index_metric: cosine  # cosine or l2
   vector_refine_factor: 30     # Re-ranking factor for accuracy
+  vector_nprobes: 20           # IVF partitions searched per query
 ```
 
 For search behavior settings (`limit`, `max_context_chars`), see [Search and Question Answering](qa.md#search-settings).
@@ -313,10 +318,22 @@ For search behavior settings (`limit`, `max_context_chars`), see [Search and Que
   - `l2`: Euclidean distance
 - **vector_refine_factor**: Improves accuracy when using a vector index by retrieving `refine_factor * limit` candidates (using approximate search) and re-ranking them with exact distances. Higher values increase accuracy but slow down queries. Default: 30
   - **Only applies with a vector index** - has no effect on brute-force search, which already returns exact results
+- **vector_nprobes**: How many IVF partitions each query searches. Higher values increase recall and latency. A larger corpus holds more partitions, so the same value covers a smaller fraction of it. Default: 20
+  - **Only applies with a vector index** - ignored by brute-force search
 
 !!! note
     Vector indexes are only necessary for large datasets with over 100,000 chunks. For smaller datasets, LanceDB's brute-force kNN search provides exact results with good performance. Only create an index if you notice search performance degradation on large datasets.
 
+Retrieval MAP with and without an index, measured on copies of the benchmark databases with no reranker:
+
+| Dataset | Chunks | Dim | Exact | Indexed | Delta | Build | Peak RSS |
+|---------|-------:|----:|------:|--------:|------:|------:|---------:|
+| `hotpotqa` | 70,527 | 2560 | 0.6978 | 0.6979 | +0.0001 | 29.3 s | 3.19 GB |
+| `orb_multimodal_nemotron` | 121,168 | 2048 | 0.9799 | 0.9800 | +0.0001 | 25.8 s | 3.38 GB |
+| `frames` | 425,940 | 2560 | 0.5431 | 0.5387 | -0.0044 | 34.1 s | 4.02 GB |
+
+An index costs no accuracy at 70k and 121k chunks and 0.0044 MAP at 426k. A larger corpus holds more IVF partitions, so the default number of probes covers a smaller fraction of the space, and `vector_refine_factor` can only re-score what those probes returned. Raise `vector_nprobes` to trade latency for recall on a large corpus. Build cost is near-flat in row count because training samples the data rather than scanning it, and vector dimension drives it more than corpus size.
+
 **Index creation:**
 
 Vector indexes are **not created automatically** during document ingestion to avoid slowing down the process. After you've added documents (at least 256 chunks required), create the index manually:
@@ -332,12 +349,12 @@ This command:
 
 **Re-indexing:**
 
-Indexes are not automatically updated when you add new documents. After adding a significant amount of new data:
+New chunks reach the index without a rebuild. `optimize()`, which runs after writes while `auto_vacuum` is on, adds them as a delta part. Between a write and the next optimize, LanceDB serves ANN over the indexed rows and a brute-force scan over the remainder, then combines the results.
+
+A rebuild retrains the centroids, which are fitted once at build time and never recomputed. As a corpus grows past the distribution it was trained on the partitioning fits it less well, and delta parts accumulate. Rebuild after substantial growth:
 
 ```bash
-haiku-rag create-index  # Rebuilds the index with all data
+haiku-rag create-index
 ```
 
-Searches still work with stale indexes - LanceDB uses the index for old data (fast ANN) and brute-force kNN for new unindexed rows, then combines the results. However, performance degrades as more unindexed data accumulates.
-
 For datasets with fewer than 256 chunks, searches use brute-force kNN scans (exact nearest neighbors, 100% recall) which work well for small datasets but don't scale beyond a few hundred thousand vectors.

docs/mcp.md

@@ -19,15 +19,71 @@ haiku-rag mcp --host 0.0.0.0 --port 8001
 # stdio transport (for Claude Desktop)
 haiku-rag mcp --stdio
 
-# Read-only mode (excludes write tools)
-haiku-rag --read-only mcp --stdio
 ```
 
 `--host` defaults to `127.0.0.1` (loopback only). Bind to `0.0.0.0` only
 when you want the MCP server reachable from outside the local machine —
 e.g. inside a Docker container with port mapping, or on a trusted LAN.
 
-**Read-only mode:** When `--read-only` is specified, write tools (`add_document_from_file`, `add_document_from_url`, `add_document_from_text`, `delete_document`) are not registered. Only search and query tools remain available.
+The server opens the database read-only. Ingestion goes through the CLI
+(`haiku-rag add`, `add-src`, `delete`) or [`haiku-ingester`](ingester.md).
+
+## Collections
+
+With several databases in `lancedb.databases`, the server covers all of
+them, as `haiku-rag search` does. Results and documents name theirs in
+`source`. `sources` on `search_documents`, `search_documents_by_image`
+and `execute_code` restricts a call to a subset; `source` on `get_document` names the database holding the
+document. A name the server does not cover is an error.
+`haiku-rag --db-name NAME mcp` serves one. See
+[Multiple Databases](configuration/storage.md#multiple-databases).
+
+## Claude Code
+
+The repository ships a plugin that registers the server and a skill telling
+Claude when and how to use it:
+
+```bash
+claude plugin marketplace add ggozad/haiku.rag
+claude plugin install haiku-rag
+```
+
+The plugin runs `haiku-rag mcp --stdio`, so `haiku-rag` must be on the PATH
+and the configuration decides the database. The skill pre-approves every tool
+and is also invocable as `/haiku-rag`. To register the server without the
+plugin:
+
+```bash
+claude mcp add haiku-rag -- haiku-rag mcp --stdio
+```
+
+The skill works with that registration too: copy `plugins/haiku-rag/skills/haiku-rag`
+into `~/.claude/skills/` and change the tool prefix in its `allowed-tools` from
+`mcp__plugin_haiku-rag_haiku-rag__` to `mcp__haiku-rag__`.
+
+## Codex
+
+The repository's Codex plugin registers the server and installs the same Agent
+Skill:
+
+```bash
+codex plugin marketplace add ggozad/haiku.rag
+codex plugin add haiku-rag@haiku-rag
+```
+
+The plugin runs `haiku-rag mcp --stdio`, so `haiku-rag` must be on the PATH.
+Invoke the skill as `$haiku-rag`. Codex can also select it automatically from
+its description. To register the server without the plugin:
+
+```bash
+codex mcp add haiku-rag -- haiku-rag mcp --stdio
+```
+
+The skill works with that registration too: copy
+`plugins/haiku-rag/skills/haiku-rag` into `~/.agents/skills/`.
+The `allowed-tools` field supplies Claude Code's tool pre-approval and may be
+ignored by other Agent Skills clients. Codex configures MCP tool approvals
+separately in `config.toml`.
 
 ## Claude Desktop Integration
 
@@ -57,63 +113,95 @@ With a custom database path:
 }
 ```
 
-After restarting Claude Desktop, you can ask Claude to search your documents, add new content, or answer questions using your knowledge base.
+After restarting Claude Desktop, you can ask Claude to search your documents or answer questions using your knowledge base.
 
-## Available Tools
+## Tools
 
-### Document Management
+Every tool is read-only and says so in its annotations. Each parameter carries
+a description in the tool schema, so the listing below names them without
+repeating it.
 
-- **`add_document_from_file`** - Add documents from local file paths
-  - `file_path` (required): Path to the file
-  - `metadata` (optional): Key-value metadata
-  - `title` (optional): Human-readable title
+| Tool | Registered | Parameters |
+|---|---|---|
+| `search_documents` | always | `query`, `limit`, `include_images`, `filter`, `sources` |
+| `search_documents_by_image` | multimodal embedder only | `image_base64`, `limit`, `include_images`, `filter`, `sources` |
+| `get_document` | always | `document_id`, `source` |
+| `get_document_outline` | always | `document_id`, `source` |
+| `get_document_section` | always | `document_id`, `section_id`, `source` |
+| `list_documents` | always | `limit`, `offset`, `filter` |
+| `execute_code` | always | `code`, `filter`, `sources` |
 
-- **`add_document_from_url`** - Add documents from URLs
-  - `url` (required): URL to fetch
-  - `metadata` (optional): Key-value metadata
-  - `title` (optional): Human-readable title
+`search_documents` runs hybrid search, vector and full-text. Its text content
+is the rendering the in-process agents read: results best first, each with its
+rank, `Document ID`, `Collection` when the server covers several, the document
+title, section headings, the matched chunk's metadata when it has any, and the
+passage expanded to its section the way the agents get it
+(`search.max_context_chars` caps it). Pictures in the results follow as
+image blocks, one per distinct picture, each preceded by a line naming its
+result; `include_images: false` leaves them out. Search results carry no
+structured content, so every client shows the model the same text and
+images. Scores are not comparable across
+queries or search types, so rank is the signal. `search_documents_by_image`
+embeds the query image and searches by vector similarity alone.
 
-- **`add_document_from_text`** - Add documents from raw text content
-  - `content` (required): Text content
-  - `uri` (optional): URI identifier
-  - `metadata` (optional): Key-value metadata
-  - `title` (optional): Human-readable title
+`get_document` returns a document whole, in reading order. For a long one,
+`get_document_outline` returns the heading tree with page numbers and
+`get_document_section` the text of one section, subsections included; a
+node's `id` in the outline is the `section_id`. A document without headings
+has an empty outline. `list_documents` returns titles, URIs and metadata,
+which is how a client learns what a filter can match.
 
-- **`get_document`** - Retrieve a document by ID
-  - `document_id` (required): The document ID
+### Code
 
-- **`list_documents`** - List documents with pagination and filtering
-  - `limit` (optional): Maximum number to return
-  - `offset` (optional): Number to skip
-  - `filter` (optional): SQL WHERE clause for filtering
+`execute_code` runs a Python program in the sandbox of the
+[analysis capability](capabilities/analysis.md), over the documents `filter`
+and `sources` select, and returns what it printed. The program reads
+`/documents/{document_id}/` (`metadata.json`, `content.txt`, `items.jsonl`,
+`chunks.jsonl`, `toc.json`) and can `await search()` and
+`await list_documents()`; the tool description spells out the fields and the
+patterns that matter. Each call is one program: nothing carries over between
+calls, and the sandbox is created and closed per call. A failing program is a
+tool error carrying the interpreter's message and any output printed before
+it. No model runs on the server. Claude Code moves a call still running after
+about two minutes to a background task.
 
-- **`delete_document`** - Delete a document by ID
-  - `document_id` (required): The document ID
+The interpreter is [Monty](https://github.com/pydantic/monty), a Python subset.
+Useful modules include `json`, `re`, `math`, `pathlib`, `datetime`,
+`collections`, `itertools`, `functools` and `dataclasses`. Absent, and often
+reached for: `decimal` and `statistics`. No generator functions, class
+inheritance or `match` statements, and a file object cannot be iterated. Files are read-only, and
+there is no network and no filesystem
+beyond `/documents`. `analysis.code_timeout` is the call's budget: compute is
+stopped at it, and past it no further host call starts, a file read or an
+in-code search alike, though one already running finishes.
+`analysis.max_output_chars` bounds the output.
 
-### Search
+### Filters
 
-- **`search_documents`** - Search using hybrid search (vector + full-text)
-  - `query` (required): Search query
-  - `limit` (optional): Maximum results (uses config default if not specified)
-  - `include_images` (optional, default `true`): Attach base64-encoded picture bytes to picture-labeled results
+`filter` is a SQL WHERE clause over the document columns `id`, `uri`, `title`,
+`metadata`, `created_at`, `updated_at`. `metadata` is a JSON string, so match
+its keys with LIKE:
 
-- **`search_documents_by_image`** - Search using an image as the query (registered only when the configured embedder supports images)
-  - `image_base64` (required): Base64-encoded image (PNG/JPEG bytes)
-  - `limit` (optional): Maximum results
-  - `include_images` (optional, default `true`)
+```sql
+metadata LIKE '%"author": "Smith"%'
+uri LIKE '%.pdf'
+title = 'Q3 report'
+```
 
-### Question Answering
+### Errors
 
-- **`ask_question`** - Ask questions about your documents
-  - `question` (required): The question to ask
-  - `cite` (optional): Include source citations (default: false)
-  - `images_base64` (optional): Base64-encoded images attached to the question (requires a vision-capable QA model)
+A failure is an MCP error carrying its message, never an empty result: a
+document or section id that matches nothing, a collection the server does not
+cover, a filter the query engine rejects, invalid base64, a program that fails
+in `execute_code` with the error it hit, and anything unexpected with its own
+message.
 
-- **`analyze`** - Answer complex analytical questions via code execution
-  - `question` (required): The question to answer
-  - `filter` (optional): SQL WHERE clause to restrict document access
-  - `images_base64` (optional): Base64-encoded images attached to the question (requires a vision-capable analysis model)
-  - Best for aggregation, computation, and multi-document analysis
+### Instructions
+
+The server publishes `instructions` describing the knowledge base: what it
+holds, when to reach for it, the collection names when it covers several, and
+`prompts.domain_preamble` when set. Claude Code and Codex show them to the
+model. Claude Desktop does not, so every tool description stands on its own.
 
 ## Continuous ingestion
 

docs/overview.md

@@ -35,7 +35,8 @@ snapshot is only meaningful while this process is the only writer.
 ## Storage
 
 LanceDB is embedded, so there is no server. The same code runs against a local
-directory, S3, GCS, Azure or LanceDB Cloud by changing `lancedb.uri`.
+directory, S3, GCS, Azure or LanceDB Cloud by changing a database's location in
+`lancedb.databases`.
 
 Tables are versioned. Vacuum collapses old versions on a retention window, and
 [tags](cli.md) name a state across all tables so a database can be restored to

docs/python.md

@@ -27,7 +27,7 @@ async with HaikuRAG("path/to/database.lancedb", read_only=True) as client:
 `async with` is the lifecycle. A caller that owns the client some other way releases it with `await client.aclose()`, which does the same work for every client shape. `client.close()` closes the connection to one database and nothing else, since draining the background vacuum and releasing the embedder and reranker are awaitable; it refuses a client covering several.
 
 !!! note
-    Databases must be explicitly created with `create=True` or via `haiku-rag init` before use. Opening a nonexistent unnamed local database raises `FileNotFoundError`, naming its path; one named in `lancedb.databases` raises `SourceUnavailableError`, which names the database rather than its location.
+    Databases must be explicitly created with `create=True` or via `haiku-rag init` before use. Opening a nonexistent local database given as a path raises `FileNotFoundError`, naming the path; a configured or default database raises `SourceUnavailableError`, which names the database rather than its location. A path beside a configured `lancedb.databases` raises `AmbiguousDatabaseError`.
 
 !!! note
     Read-only mode is useful for safely accessing databases without risk of modification. It blocks all write operations and downgrades an embedding provider/name mismatch to a warning instead of raising `ConfigMismatchError`.
@@ -268,8 +268,8 @@ On the constructor `sources=[]` means something else. Passing `sources` alongsid
 
 ```python
 client.covers_multiple      # whether the client covers more than one database
-client.source_names         # configured names, in order
-client.source               # one configured name, or None for a set or unnamed database
+client.source_names         # database names, in order; known before the client opens
+client.source               # the one database's name, or None for a set
 
 owner = await client.reader_for("papers")      # the client reading that database
 papers, wiki = await client.clients_for(["papers", "wiki"])
@@ -277,6 +277,17 @@ papers, wiki = await client.clients_for(["papers", "wiki"])
 
 `reader_for` and `clients_for` open databases lazily and return borrowed clients. They remain valid while the covering client is open and inherit its read-only mode. The covering client owns and closes their database sessions.
 
+To learn what a configuration covers without opening anything, resolve it:
+
+```python
+from haiku.rag.client import DatabaseScope
+
+for ref in DatabaseScope.resolve(config).databases:
+    print(ref.name, ref.location)   # "haiku.rag", Path(".../haiku.rag.lancedb") when nothing is configured
+```
+
+`DatabaseScope.resolve` is pure: it reads the configuration and classifies each location as a local path or a URI.
+
 ### Filtering Search Results
 
 Filter search results to only include chunks from documents matching specific criteria:

docs/tutorial.md

@@ -12,7 +12,7 @@ You also need [Ollama](https://ollama.com/) for the default embedding and answer
 
 ```bash
 ollama pull qwen3-embedding:4b
-ollama pull gpt-oss
+ollama pull qwen3.8
 ```
 
 !!! note "Prefer OpenAI?"

evaluations/README.md

@@ -49,7 +49,7 @@ datasets and judge:
 
 ```bash
 evaluations run hotpotqa --target rag-capability
-evaluations run hotpotqa --target analysis-capability --capability-model ollama:gpt-oss
+evaluations run hotpotqa --target analysis-capability --capability-model ollama:qwen3.8
 ```
 
 `--capability-model "provider:name"` overrides the capability model independently from

evaluations/configs/mtrag_federated.yaml

@@ -1,56 +0,0 @@
-# Reference config for `mtrag_federated`: IBM MTRAG ClapNQ, partitioned by
-# article title into four collections, scored on retrieval only.
-#
-# Build the partition and emit the config that searches exactly it:
-#   uv run python -m evaluations.datasets.mtrag_federated \
-#       --config configs/mtrag_federated.yaml --n 4 --out ~/configs/fed-n4.yaml
-#   evaluations run mtrag_federated --config ~/configs/fed-n4.yaml \
-#       --skip-db --skip-qa
-#
-# The databases below are the canonical n=4 partition at seed 20260831. Sweep
-# configs for other collection counts live outside the repo, because a config
-# here must be named after a registered dataset.
-#
-# No reranking block on purpose: this eval measures the reciprocal-rank fusion
-# path, where retrieval depth per collection is `limit // n`. Adding a reranker
-# is the comparison arm, not the baseline.
-# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.
-
-environment: development
-
-storage:
-  auto_vacuum: false
-
-lancedb:
-  # Declaration order is load-bearing: fusion resolves equal ranks to this
-  # order, so permuting these four keys is an arm.
-  databases:
-    clapnq_0: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_0.lancedb
-    clapnq_1: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_1.lancedb
-    clapnq_2: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_2.lancedb
-    clapnq_3: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_3.lancedb
-
-embeddings:
-  model:
-    provider: openai
-    name: qwen3-embedding-4b
-    vector_dim: 2560
-    base_url: http://vllm:11431/v1
-
-search:
-  # Matches the spec's retrieval_limit and the product default.
-  limit: 5
-
-evaluations:
-  judge:
-    provider: openai
-    name: Inferact/Qwen3.8-27B-NVFP4
-    base_url: http://vllm:11439/v1
-    temperature: 0.6
-    max_tokens: 16384
-    extra_body:
-      top_p: 0.95
-      top_k: 20
-      min_p: 0
-      chat_template_kwargs:
-        reasoning_effort: low

evaluations/configs/mtrag_pooled.yaml

@@ -1,49 +0,0 @@
-# Reference config for `mtrag_pooled`: all four MTRAG domains (clapnq, cloud,
-# fiqa, govt) pooled and partitioned across `n` collections, scored on retrieval
-# only.
-#
-# This is the heterogeneous corpus. `mtrag_federated` partitions one domain by
-# article title, which is round-robin fusion's friendliest case: no collection is
-# ever off-topic for a query, so the guaranteed-slot waste is never exercised.
-# Here a query belongs to one domain and the others are genuinely off-topic.
-#
-# Build the partition and emit the config that searches exactly it:
-#   uv run python -m evaluations.datasets.mtrag_federated \
-#       --config configs/mtrag_pooled.yaml --pooled --n 4 --alpha 0 \
-#       --out ~/configs/pooled-n4-a0.yaml
-#   evaluations run mtrag_pooled --config ~/configs/pooled-n4-a0.yaml \
-#       --skip-db --skip-qa
-#
-# alpha 0 keeps each collection to one domain; alpha 1 shards titles across all
-# of them, which is the degenerate sharding endpoint rather than a rival design.
-# The operator emits the databases, so none are listed here.
-
-environment: development
-
-storage:
-  auto_vacuum: false
-
-embeddings:
-  model:
-    provider: openai
-    name: qwen3-embedding-4b
-    vector_dim: 2560
-    base_url: http://vllm:11431/v1
-
-search:
-  # Matches the spec's retrieval_limit and the product default.
-  limit: 5
-
-evaluations:
-  judge:
-    provider: openai
-    name: Inferact/Qwen3.8-27B-NVFP4
-    base_url: http://vllm:11439/v1
-    temperature: 0.6
-    max_tokens: 16384
-    extra_body:
-      top_p: 0.95
-      top_k: 20
-      min_p: 0
-      chat_template_kwargs:
-        reasoning_effort: low

evaluations/evaluations/benchmark.py

@@ -41,7 +41,6 @@ async def evaluate_dataset(
     vacuum_interval: int = 100,
     multimodal_only: bool = False,
     judge_model: ModelConfig | None = None,
-    retrieval_limit: int | None = None,
     target: Target = "rag-capability",
     capability_model: ModelConfig | None = None,
     case_ids: set[str] | None = None,
@@ -50,6 +49,13 @@ async def evaluate_dataset(
     if document_filter is not None:
         console.print(f"Document filter: {document_filter}", style="dim")
 
+    if db_path is not None and config.lancedb.databases:
+        raise ValueError(
+            "--db PATH places the database where the configuration places none, "
+            f"and this configuration names {', '.join(config.lancedb.databases)} "
+            "in lancedb.databases. Drop --db to evaluate the configured set."
+        )
+
     if not skip_db:
         if spec.uses_configured_databases(config, db_path):
             raise ValueError(
@@ -73,7 +79,6 @@ async def evaluate_dataset(
             db_path=db_path,
             multimodal_only=multimodal_only,
             document_filter=document_filter,
-            retrieval_limit=retrieval_limit,
         )
 
     if not skip_qa:
@@ -159,7 +164,11 @@ def run(
     config: Path | None = typer.Option(
         None, "--config", help="Path to haiku.rag YAML config file."
     ),
-    db: Path | None = typer.Option(None, "--db", help="Override the database path."),
+    db: Path | None = typer.Option(
+        None,
+        "--db",
+        help="Database path, where the configuration places no database.",
+    ),
     skip_db: bool = typer.Option(
         False, "--skip-db", help="Skip updating the evaluation db."
     ),
@@ -171,14 +180,6 @@ def run(
         None, "--limit", help="Limit number of test cases for both retrieval and QA."
     ),
     name: str | None = typer.Option(None, "--name", help="Override evaluation name."),
-    retrieval_limit: int | None = typer.Option(
-        None,
-        "--retrieval-limit",
-        help=(
-            "Candidates each database fetches, overriding the dataset's. "
-            "Sets how deep hybrid search looks before its results are scored."
-        ),
-    ),
     vacuum_interval: int = typer.Option(
         100, "--vacuum-interval", help="Vacuum every N documents during DB population."
     ),
@@ -245,7 +246,6 @@ def run(
             vacuum_interval=vacuum_interval,
             multimodal_only=multimodal_only,
             judge_model=judge_model_config,
-            retrieval_limit=retrieval_limit,
             target=target_value,
             capability_model=capability_model_config,
             case_ids=_load_case_ids(filter_ids),

evaluations/evaluations/capability_runner.py

@@ -44,7 +44,7 @@ class CapabilityRunResult:
     cited_uris: list[str] = field(default_factory=list)
     cited_chunk_ids: list[str] = field(default_factory=list)
     # The database each cited chunk came from, in the order they were cited.
-    # Empty string where the database is unnamed.
+    # Empty string for a citation built without a source.
     cited_sources: list[str] = field(default_factory=list)
     searched_uris: list[str] = field(default_factory=list)
     n_searches: int = 0

evaluations/evaluations/config.py

@@ -89,9 +89,9 @@ class DatasetSpec:
     ) -> bool:
         """Whether `lancedb.databases` places the databases to evaluate over.
 
-        A path names one database and wins over the configuration, both when it
-        comes from `--db` and when the client resolves it. True for a mapping of
-        one, which is a configured database like any other and keeps its name.
+        `--db PATH` places the database where the configuration places none;
+        `evaluate_dataset` refuses the two together. True for a mapping of one,
+        which is a configured database like any other and keeps its name.
         """
         return bool(config.lancedb.databases) and override_path is None
 

evaluations/evaluations/datasets/__init__.py

@@ -8,7 +8,6 @@ from .mtrag import (
     MTRAG_CLAPNQ_REWRITE_SPEC,
     MTRAG_CLAPNQ_SPEC,
 )
-from .mtrag_federated import MTRAG_FEDERATED_SPEC, MTRAG_POOLED_SPEC
 from .open_rag_bench import (
     ORB_MULTIMODAL_NEMOTRON_SPEC,
     ORB_MULTIMODAL_SPEC,
@@ -25,8 +24,6 @@ DATASETS: dict[str, DatasetSpec] = {
         MTRAG_CLAPNQ_REWRITE_SPEC,
         MTRAG_CLAPNQ_LIVE_SPEC,
         MTRAG_CLAPNQ_LIVE_UNCOMPACTED_SPEC,
-        MTRAG_FEDERATED_SPEC,
-        MTRAG_POOLED_SPEC,
         ORB_TEXT_SPEC,
         ORB_MULTIMODAL_SPEC,
         ORB_MULTIMODAL_NEMOTRON_SPEC,

evaluations/evaluations/datasets/mtrag_federated.py

@@ -1,598 +0,0 @@
-import argparse
-import asyncio
-import hashlib
-import json
-import random
-import zipfile
-from collections.abc import Iterable, Mapping, Sequence
-from pathlib import Path
-from typing import Any
-
-import yaml
-from datasets import Dataset
-
-from evaluations.config import DatasetSpec
-from evaluations.datasets.mtrag import (
-    _load_qrels,
-    build_mtrag_case,
-    load_clapnq_corpus,
-    load_clapnq_retrieval,
-    map_mtrag_document,
-    map_mtrag_retrieval,
-)
-from evaluations.evaluators import (
-    CitationMAPEvaluator,
-    MAPEvaluator,
-    NDCGEvaluator,
-    RecallEvaluator,
-)
-from haiku.rag.config import load_yaml_config
-from haiku.rag.config.models import AppConfig
-from haiku.rag.utils import get_default_data_dir
-
-COLLECTION_PREFIX = "clapnq"
-# The four corpora MTRAG ships, in upstream order.
-DOMAINS = ("clapnq", "cloud", "fiqa", "govt")
-DEFAULT_SEED = 20260831
-# Whole titles are kept, and the 148 titles holding a gold passage carry 10,723
-# passages between them, so that is the floor. A budget near it leaves no
-# cross-topic distractors at all and inflates recall; this default leaves about
-# 29,000, a gold-title share near a quarter.
-GOLD_TITLE_FLOOR = 10_723
-DEFAULT_BUDGET = 40_000
-INGEST_BATCH_SIZE = 512
-FTS_INDEX_NAME = "content_fts_idx"
-
-
-def _hash_int(payload: str) -> int:
-    """sha256 rather than `hash()`, which is salted per process: the partition is
-    never stored, and scoring recomputes it in a different process than the one
-    that ingested."""
-    return int.from_bytes(hashlib.sha256(payload.encode()).digest()[:8], "big")
-
-
-def _unit(payload: str) -> float:
-    """A stable value in [0, 1) for probabilistic assignment."""
-    return (_hash_int(payload) % 10**9) / 10**9
-
-
-def domain_files(domain: str, variant: str = "lastturn") -> tuple[str, str, str]:
-    """Corpus, qrels and query paths for one MTRAG domain."""
-    return (
-        f"corpora/passage_level/{domain}.jsonl.zip",
-        f"mtrag-human/retrieval_tasks/{domain}/qrels/dev.tsv",
-        f"mtrag-human/retrieval_tasks/{domain}/{domain}_{variant}.jsonl",
-    )
-
-
-def _domain_collection(title: str, domain: str, n: int, seed: int) -> int:
-    """The collection a title takes from its domain.
-
-    Domains map onto collections proportionally: with more collections than
-    domains each domain is subdivided by title, with fewer, domains are grouped.
-    """
-    index = DOMAINS.index(domain)
-    count = len(DOMAINS)
-    if n >= count:
-        per = n // count
-        return index * per + (_hash_int(f"{seed}/sub/{title}") % per)
-    return index * n // count
-
-
-def collection_of(
-    title: str,
-    n: int,
-    seed: int = DEFAULT_SEED,
-    alpha: float = 0.0,
-    domain: str | None = None,
-) -> int:
-    """Which of `n` collections holds a title's passages.
-
-    Keyed on the title, so an article's passages never split.
-
-    Without a domain the assignment is a pure hash — a topically arbitrary
-    grouping of whole articles, which is all the quota and order-bias arms need.
-    With one, `alpha` interpolates between the domain partition (0, each
-    collection one topic) and a uniform shard (1, domain ignored). Sharding is
-    the endpoint of the knob rather than a rival design.
-    """
-    if n < 1:
-        raise ValueError("a partition needs at least one collection")
-    shard = _hash_int(f"{seed}/{title}") % n
-    if domain is None or alpha >= 1.0:
-        return shard
-    if alpha > 0.0 and _unit(f"{seed}/alpha/{title}") < alpha:
-        return shard
-    return _domain_collection(title, domain, n, seed)
-
-
-def collection_names(n: int) -> tuple[str, ...]:
-    """The collection names in declaration order.
-
-    Order is load-bearing: fusion resolves equal ranks to configured order, so
-    permuting these names is an arm rather than a cosmetic change.
-    """
-    return tuple(f"{COLLECTION_PREFIX}_{index}" for index in range(n))
-
-
-def database_paths(n: int, seed: int = DEFAULT_SEED) -> dict[str, str]:
-    """Where each collection's database lives.
-
-    The partition is in the filename, so a build at one `(n, seed)` can never
-    overwrite another's databases or be searched by the wrong config.
-    """
-    root = get_default_data_dir() / "evaluations" / "dbs"
-    return {
-        name: str(root / f"mtrag_federated_s{seed}_n{n}_{index}.lancedb")
-        for index, name in enumerate(collection_names(n))
-    }
-
-
-def sample_records(
-    records: Sequence[Mapping[str, Any]],
-    gold_ids: Iterable[str],
-    budget: int = DEFAULT_BUDGET,
-    seed: int = DEFAULT_SEED,
-) -> list[Mapping[str, Any]]:
-    """A fixed sub-corpus: every gold passage, plus seeded distractor titles.
-
-    Whole titles are kept or dropped together. Gold is mandatory, so a budget
-    below the gold floor yields the gold titles alone rather than an incomplete
-    corpus that would score as missing retrievals.
-    """
-    by_title: dict[str, list[Mapping[str, Any]]] = {}
-    title_of: dict[str, str] = {}
-    for row in records:
-        by_title.setdefault(row["title"], []).append(row)
-        title_of[row["_id"]] = row["title"]
-
-    wanted = set(gold_ids)
-    missing = sorted(wanted - set(title_of))
-    if missing:
-        raise ValueError(
-            f"{len(missing)} gold passages do not resolve to the corpus, "
-            f"first few: {missing[:3]}"
-        )
-
-    gold_titles = {title_of[passage_id] for passage_id in wanted}
-    kept = {title for title in by_title if title in gold_titles}
-    total = sum(len(by_title[title]) for title in kept)
-
-    distractors = [title for title in by_title if title not in gold_titles]
-    random.Random(seed).shuffle(distractors)
-    for title in distractors:
-        size = len(by_title[title])
-        if total + size > budget:
-            continue
-        kept.add(title)
-        total += size
-
-    return [row for row in records if row["title"] in kept]
-
-
-def partition_records(
-    records: Sequence[Mapping[str, Any]],
-    n: int,
-    seed: int = DEFAULT_SEED,
-) -> dict[str, list[Mapping[str, Any]]]:
-    """Route every record to its collection, naming all `n` even when empty."""
-    names = collection_names(n)
-    grouped: dict[str, list[Mapping[str, Any]]] = {name: [] for name in names}
-    for row in records:
-        grouped[names[collection_of(row["title"], n, seed)]].append(row)
-    return grouped
-
-
-def pool_composition(
-    records: Sequence[Mapping[str, Any]], gold_ids: Iterable[str]
-) -> tuple[int, int]:
-    """Passages in gold-bearing titles, and passages in distractor titles.
-
-    A pool with no distractors scores as an easy retrieval task and says
-    nothing, so the build reports this rather than leaving it to be inferred
-    from the budget.
-    """
-    wanted = set(gold_ids)
-    gold_titles = {row["title"] for row in records if row["_id"] in wanted}
-    gold_side = sum(1 for row in records if row["title"] in gold_titles)
-    return gold_side, len(records) - gold_side
-
-
-def gold_passage_ids() -> set[str]:
-    """Every corpus id the ClapNQ qrels reference."""
-    return {passage_id for ids in _load_qrels().values() for passage_id in ids}
-
-
-def load_pool(
-    budget: int = DEFAULT_BUDGET, seed: int = DEFAULT_SEED
-) -> list[Mapping[str, Any]]:
-    records = [dict(row) for row in load_clapnq_corpus()]
-    return sample_records(records, gold_passage_ids(), budget, seed)
-
-
-POOLED_PREFIX = "dom"
-
-
-class PassageIdCollision(AssertionError):
-    """Two domains claim the same passage id, so uri-keyed gold is ambiguous."""
-
-
-def pooled_collection_names(n: int) -> tuple[str, ...]:
-    """Names for the pooled partition, positional and distinct from the
-    single-domain set so the two never share database paths."""
-    return tuple(f"{POOLED_PREFIX}_{index}" for index in range(n))
-
-
-def pooled_database_paths(
-    n: int, alpha: float, seed: int = DEFAULT_SEED
-) -> dict[str, str]:
-    root = get_default_data_dir() / "evaluations" / "dbs"
-    tag = f"s{seed}_a{alpha:g}_n{n}"
-    return {
-        name: str(root / f"mtrag_pooled_{tag}_{index}.lancedb")
-        for index, name in enumerate(pooled_collection_names(n))
-    }
-
-
-def load_pooled_records() -> list[Mapping[str, Any]]:
-    """Every passage of all four domains, each tagged with the domain it came
-    from. Raises when two domains claim one passage id, since gold is uri-keyed.
-    """
-    from evaluations.datasets.mtrag import _download
-
-    records: list[Mapping[str, Any]] = []
-    seen: dict[str, str] = {}
-    for domain in DOMAINS:
-        corpus_file, _, _ = domain_files(domain)
-        path = _download(corpus_file)
-        with zipfile.ZipFile(path) as archive:
-            with archive.open(archive.namelist()[0]) as handle:
-                for line in handle:
-                    row = json.loads(line)
-                    passage_id = row["_id"]
-                    if passage_id in seen and seen[passage_id] != domain:
-                        raise PassageIdCollision(
-                            f"{passage_id} claimed by {seen[passage_id]} and {domain}"
-                        )
-                    seen[passage_id] = domain
-                    records.append(
-                        {
-                            "_id": passage_id,
-                            "title": row["title"],
-                            "text": row["text"],
-                            "domain": domain,
-                        }
-                    )
-    return records
-
-
-def load_pooled_queries(variant: str = "lastturn") -> list[dict[str, Any]]:
-    """Retrieval queries from every domain, each with its gold passage uris."""
-    from evaluations.datasets.mtrag import _download, _parse_qrels
-
-    out: list[dict[str, Any]] = []
-    for domain in DOMAINS:
-        _, qrels_file, query_file = domain_files(domain, variant)
-        qrels = _parse_qrels(_download(qrels_file).read_text().splitlines())
-        for line in _download(query_file).read_text().splitlines():
-            if not line.strip():
-                continue
-            query = json.loads(line)
-            expected = qrels.get(query["_id"])
-            if not expected:
-                continue
-            out.append(
-                {
-                    "query_id": f"{domain}/{query['_id']}",
-                    "question": query["text"],
-                    "expected_uris": expected,
-                    "domain": domain,
-                }
-            )
-    return out
-
-
-def pooled_gold_ids(variant: str = "lastturn") -> set[str]:
-    return {
-        uri for query in load_pooled_queries(variant) for uri in query["expected_uris"]
-    }
-
-
-def load_pooled(
-    budget: int = DEFAULT_BUDGET, seed: int = DEFAULT_SEED
-) -> list[Mapping[str, Any]]:
-    return sample_pooled_records(load_pooled_records(), pooled_gold_ids(), budget, seed)
-
-
-def sample_pooled_records(
-    records: Sequence[Mapping[str, Any]],
-    gold_ids: Iterable[str],
-    budget: int = DEFAULT_BUDGET,
-    seed: int = DEFAULT_SEED,
-) -> list[Mapping[str, Any]]:
-    """A fixed sub-corpus at passage level, keeping every gold passage.
-
-    The single-domain dataset keeps whole titles, which cannot work here: `title`
-    is the empty string for every cloud and fiqa passage, so two of the four
-    domains have exactly one title covering 72,442 and 61,022 passages. Whole
-    titles put the gold floor at 146,543 passages, leaving no distractors at any
-    budget below the entire corpus.
-
-    Passage level costs nothing this comparison needs: at alpha=0 the domain
-    places a collection, so a query's gold is concentrated by construction rather
-    than by the atom.
-    """
-    wanted = set(gold_ids)
-    by_id = {row["_id"]: row for row in records}
-    missing = sorted(wanted - set(by_id))
-    if missing:
-        raise ValueError(
-            f"{len(missing)} gold passages do not resolve to the pooled corpus, "
-            f"first few: {missing[:3]}"
-        )
-    kept = set(wanted)
-    others = [row["_id"] for row in records if row["_id"] not in wanted]
-    random.Random(seed).shuffle(others)
-    for passage_id in others:
-        if len(kept) >= budget:
-            break
-        kept.add(passage_id)
-    return [row for row in records if row["_id"] in kept]
-
-
-def partition_pooled(
-    records: Sequence[Mapping[str, Any]],
-    n: int,
-    alpha: float,
-    seed: int = DEFAULT_SEED,
-) -> dict[str, list[Mapping[str, Any]]]:
-    """Route pooled records to collections, honouring each record's domain.
-
-    Keyed on the passage id rather than the title, because two of the four
-    domains have no titles. See `sample_pooled_records`.
-    """
-    names = pooled_collection_names(n)
-    grouped: dict[str, list[Mapping[str, Any]]] = {name: [] for name in names}
-    for row in records:
-        index = collection_of(row["_id"], n, seed, alpha=alpha, domain=row["domain"])
-        grouped[names[index]].append(row)
-    return grouped
-
-
-def _unused_document_loader() -> Dataset:
-    raise RuntimeError(
-        "the federated corpus is built by build_databases(); run with --skip-db"
-    )
-
-
-def emitted_config(reference: Path, n: int, seed: int = DEFAULT_SEED) -> dict[str, Any]:
-    """The reference config with this partition's databases placed in it."""
-    settings = load_yaml_config(reference)
-    lancedb = dict(settings.get("lancedb") or {})
-    lancedb.pop("uri", None)
-    lancedb["databases"] = database_paths(n, seed)
-    settings["lancedb"] = lancedb
-    return settings
-
-
-class FTSIndexNotCoveringRows(AssertionError):
-    """The chunks FTS index does not cover every row, so full-text search is
-    dead while still returning results."""
-
-
-async def assert_fts_covers_rows(table: Any, name: str) -> None:
-    rows = await table.count_rows()
-    indices = {index.name for index in await table.list_indices()}
-    if FTS_INDEX_NAME not in indices:
-        raise FTSIndexNotCoveringRows(f"{name}: no {FTS_INDEX_NAME} on {rows} rows")
-    stats = await table.index_stats(FTS_INDEX_NAME)
-    indexed = getattr(stats, "num_indexed_rows", 0) or 0
-    if indexed < rows:
-        raise FTSIndexNotCoveringRows(
-            f"{name}: {FTS_INDEX_NAME} covers {indexed} of {rows} rows; "
-            "full-text search would return near-arbitrary rows"
-        )
-
-
-async def build_databases(
-    config: AppConfig,
-    n: int,
-    seed: int = DEFAULT_SEED,
-    budget: int = DEFAULT_BUDGET,
-) -> dict[str, int]:
-    """Ingest the partition into one database per collection.
-
-    Each member is opened by configured name with a scope of one, which is what
-    makes `create=True` legal on a client whose config places several.
-    """
-    from haiku.rag.client import HaikuRAG
-
-    from evaluations.population import _ingest_batched
-
-    names = collection_names(n)
-    configured = set(config.lancedb.databases or {})
-    missing = sorted(set(names) - configured)
-    if missing:
-        raise ValueError(
-            f"lancedb.databases must place every collection; missing {missing}"
-        )
-
-    pool = load_pool(budget, seed)
-    gold_side, distractors = pool_composition(pool, gold_passage_ids())
-    print(
-        f"pool: {len(pool)} passages, {gold_side} in gold-bearing titles, "
-        f"{distractors} distractors"
-    )
-    if not distractors:
-        print(
-            "  WARNING: no distractor titles, so every passage belongs to an "
-            f"answer-bearing article; raise --budget above {GOLD_TITLE_FLOOR}"
-        )
-    grouped = partition_records(pool, n, seed)
-    written: dict[str, int] = {}
-    for name in names:
-        async with HaikuRAG(config=config, sources=[name], create=True) as client:
-            await _ingest_batched(
-                client, MTRAG_FEDERATED_SPEC, grouped[name], INGEST_BATCH_SIZE
-            )
-            # The chunks FTS index is built once when the table is created, over
-            # zero rows, and nothing folds later rows into it but an optimize.
-            # `auto_vacuum` is false here, as in every reference config, so
-            # without this the index covers nothing and full-text search returns
-            # near-arbitrary rows while still looking like it works.
-            await client.store.vacuum(retention_seconds=0)
-            await assert_fts_covers_rows(client.store.chunks_table, name)
-        written[name] = len(grouped[name])
-    return written
-
-
-async def build_pooled_databases(
-    config: AppConfig,
-    n: int,
-    alpha: float,
-    seed: int = DEFAULT_SEED,
-    budget: int = DEFAULT_BUDGET,
-) -> dict[str, int]:
-    """Ingest the four-domain pooled partition, one database per collection."""
-    from haiku.rag.client import HaikuRAG
-
-    from evaluations.population import _ingest_batched
-
-    names = pooled_collection_names(n)
-    configured = set(config.lancedb.databases or {})
-    missing = sorted(set(names) - configured)
-    if missing:
-        raise ValueError(
-            f"lancedb.databases must place every collection; missing {missing}"
-        )
-
-    pool = load_pooled(budget, seed)
-    by_domain: dict[str, int] = {}
-    for row in pool:
-        by_domain[row["domain"]] = by_domain.get(row["domain"], 0) + 1
-    # Passage level, not `pool_composition`: that counts gold-bearing titles,
-    # which is meaningless here since cloud and fiqa have one empty title each.
-    gold = pooled_gold_ids()
-    gold_kept = sum(1 for row in pool if row["_id"] in gold)
-    print(
-        f"pool: {len(pool)} passages, {gold_kept} gold, "
-        f"{len(pool) - gold_kept} distractors, by domain {by_domain}"
-    )
-    grouped = partition_pooled(pool, n, alpha, seed)
-    written: dict[str, int] = {}
-    for name in names:
-        async with HaikuRAG(config=config, sources=[name], create=True) as client:
-            await _ingest_batched(
-                client, MTRAG_POOLED_SPEC, grouped[name], INGEST_BATCH_SIZE
-            )
-            await client.store.vacuum(retention_seconds=0)
-            await assert_fts_covers_rows(client.store.chunks_table, name)
-        written[name] = len(grouped[name])
-    return written
-
-
-MTRAG_FEDERATED_SPEC = DatasetSpec(
-    key="mtrag_federated",
-    # Never read: the run searches the configured set. Present because the spec
-    # requires one, and pointed at the first collection so a stray --db is
-    # obviously wrong rather than silently plausible.
-    db_filename="mtrag_federated_unused.lancedb",
-    document_loader=_unused_document_loader,
-    document_mapper=map_mtrag_document,
-    # The QA phase is not wired yet, and the loader is what run_qa_benchmark
-    # reaches first, so a forgotten --skip-qa fails loudly there. The builder is
-    # the one the generation tasks will need when QA arrives.
-    qa_loader=_unused_document_loader,
-    qa_case_builder=build_mtrag_case,
-    retrieval_loader=lambda: load_clapnq_retrieval("lastturn"),
-    retrieval_mapper=map_mtrag_retrieval,
-    retrieval_evaluators=[
-        RecallEvaluator(5),
-        RecallEvaluator(10),
-        NDCGEvaluator(5),
-        MAPEvaluator(),
-    ],
-    citation_evaluator=CitationMAPEvaluator(),
-    # The product default, which is where the fusion depth quota bites.
-    retrieval_limit=5,
-    ingest_batch_size=INGEST_BATCH_SIZE,
-)
-
-
-MTRAG_POOLED_SPEC = DatasetSpec(
-    key="mtrag_pooled",
-    db_filename="mtrag_pooled_unused.lancedb",
-    document_loader=_unused_document_loader,
-    document_mapper=map_mtrag_document,
-    qa_loader=_unused_document_loader,
-    qa_case_builder=build_mtrag_case,
-    retrieval_loader=lambda: Dataset.from_list(load_pooled_queries("lastturn")),
-    retrieval_mapper=map_mtrag_retrieval,
-    retrieval_evaluators=[
-        RecallEvaluator(5),
-        RecallEvaluator(10),
-        NDCGEvaluator(5),
-        MAPEvaluator(),
-    ],
-    citation_evaluator=CitationMAPEvaluator(),
-    retrieval_limit=5,
-    ingest_batch_size=INGEST_BATCH_SIZE,
-)
-
-
-async def main() -> None:
-    parser = argparse.ArgumentParser(
-        description=(
-            "Build the federated ClapNQ partition and emit the config that "
-            "searches exactly it."
-        )
-    )
-    parser.add_argument("--config", type=Path, required=True, help="reference config")
-    parser.add_argument("--n", type=int, required=True, help="collection count")
-    parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
-    parser.add_argument("--budget", type=int, default=DEFAULT_BUDGET)
-    parser.add_argument(
-        "--pooled",
-        action="store_true",
-        help="build the four-domain pooled corpus instead of clapnq alone",
-    )
-    parser.add_argument(
-        "--alpha",
-        type=float,
-        default=0.0,
-        help="pooled only: 0 keeps a collection to one domain, 1 shards across all",
-    )
-    parser.add_argument(
-        "--out",
-        type=Path,
-        required=True,
-        help="where to write the emitted config for this partition",
-    )
-    args = parser.parse_args()
-
-    if args.pooled:
-        settings = load_yaml_config(args.config)
-        lancedb = dict(settings.get("lancedb") or {})
-        lancedb.pop("uri", None)
-        lancedb["databases"] = pooled_database_paths(args.n, args.alpha, args.seed)
-        settings["lancedb"] = lancedb
-    else:
-        settings = emitted_config(args.config, args.n, args.seed)
-    args.out.parent.mkdir(parents=True, exist_ok=True)
-    args.out.write_text(yaml.safe_dump(settings, sort_keys=False))
-    print(f"wrote {args.out}")
-
-    # Reload from disk, so the config that builds is the file that will search.
-    config = AppConfig.model_validate(load_yaml_config(args.out))
-    if args.pooled:
-        written = await build_pooled_databases(
-            config, args.n, args.alpha, args.seed, args.budget
-        )
-    else:
-        written = await build_databases(config, args.n, args.seed, args.budget)
-    for name, count in written.items():
-        print(f"{name}: {count} passages")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

evaluations/evaluations/population.py

@@ -39,9 +39,6 @@ async def _ingest_batched(
         row["uri"]: row["id"] for row in uri_rows if row["id"] not in chunked_ids
     }
 
-    from haiku.rag.converters import get_converter
-
-    converter = get_converter(rag._config)
     batch: list[DocumentImport] = []
     for doc in corpus:
         payload = spec.document_mapper(cast(Mapping[str, Any], doc))
@@ -51,13 +48,7 @@ async def _ingest_batched(
         if payload.uri in chunkless:
             await rag.delete_document(chunkless[payload.uri])
         assert payload.content is not None, "batched ingest requires inline content"
-        # Convert as text explicitly. `rag.convert` disambiguates a str by
-        # parsing it, and a passage beginning with a URL (187 of them across
-        # MTRAG's cloud and fiqa corpora) is then fetched over HTTP instead of
-        # stored. Batched ingest has already asserted the content is inline.
-        docling_document = await converter.convert_text(
-            payload.content, format=payload.format
-        )
+        docling_document = await rag.convert(payload.content, format=payload.format)
         chunks = await rag.chunk(docling_document)
         batch.append(
             DocumentImport(

evaluations/evaluations/retrieval.py

@@ -24,7 +24,6 @@ async def run_retrieval_benchmark(
     db_path: Path | None = None,
     multimodal_only: bool = False,
     document_filter: str | None = None,
-    retrieval_limit: int | None = None,
 ) -> dict[str, float] | None:
     if spec.retrieval_loader is None or spec.retrieval_mapper is None:
         console.print("Skipping retrieval benchmark; no retrieval config.")
@@ -73,7 +72,6 @@ async def run_retrieval_benchmark(
         evaluators=list(spec.retrieval_evaluators),
     )
 
-    fetch = retrieval_limit or spec.retrieval_limit
     db = (
         None
         if spec.uses_configured_databases(config, db_path)
@@ -84,7 +82,7 @@ async def run_retrieval_benchmark(
         async def retrieval_target(question: str) -> list[str]:
             chunks = await rag.search(
                 query=question,
-                limit=fetch,
+                limit=spec.retrieval_limit,
                 include_images=False,
                 filter=document_filter,
             )

evaluations/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag-evals"
 description = "Benchmarking and evaluation scripts for haiku.rag"
-version = "0.79.0"
+version = "0.82.1"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 requires-python = ">=3.12"

evaluations/tests/test_benchmark.py

@@ -711,6 +711,37 @@ class TestEvaluateDatasetJudgeModel:
         assert mock_qa.call_args[1]["judge_model"] is custom_judge
 
 
+@pytest.mark.asyncio
+async def test_a_db_path_beside_configured_databases_is_refused(tmp_path) -> None:
+    """`--db` places the database where the configuration places none; beside
+    `lancedb.databases` the run refuses before touching anything."""
+    from haiku.rag.config.models import LanceDBConfig
+
+    config = AppConfig(
+        lancedb=LanceDBConfig(databases={"alpha": str(tmp_path / "a.lancedb")})
+    )
+    spec = DatasetSpec(
+        key="test",
+        db_filename="test.lancedb",
+        document_loader=lambda: None,  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+        document_mapper=lambda doc: None,
+        qa_loader=lambda: [],  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+        qa_case_builder=lambda idx, doc: None,  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+    )
+
+    with pytest.raises(ValueError, match="alpha"):
+        await evaluate_dataset(
+            spec=spec,
+            config=config,
+            skip_db=True,
+            skip_retrieval=True,
+            skip_qa=True,
+            limit=None,
+            name=None,
+            db_path=tmp_path / "other.lancedb",
+        )
+
+
 class TestExperimentMetadataTargets:
     def test_default_target_is_rag_capability(self) -> None:
         result = build_experiment_metadata(
@@ -888,11 +919,7 @@ class TestBatchedIngest:
         rag.store.chunks_table = _table(
             [{"document_id": f"id-{uri}"} for uri in complete_uris]
         )
-        # Batched ingest converts text through the configured converter, the
-        # same path create_document uses, so the double needs a real config.
-        from haiku.rag.config.models import AppConfig
-
-        rag._config = AppConfig()
+        rag.convert = AsyncMock(side_effect=lambda content, **kw: f"docling:{content}")
         rag.chunk = AsyncMock(return_value=[])
         rag.import_documents = AsyncMock()
         rag.delete_document = AsyncMock()
@@ -937,11 +964,8 @@ class TestBatchedIngest:
         await _ingest_batched(rag, self._spec(), corpus, batch_size=10)
 
         (batch,), _ = rag.import_documents.call_args
-        # Conversion goes through the configured converter now, not rag.convert,
-        # so the batch contents are the assertion: exactly the incomplete uris,
-        # which is stricter than counting conversions.
         assert [imp.uri for imp in batch] == ["u1", "u3"]
-        assert len(batch) == 2
+        assert rag.convert.await_count == 2
         rag.delete_document.assert_not_awaited()
 
     @pytest.mark.asyncio

evaluations/tests/test_capability_runner.py

@@ -485,8 +485,8 @@ def test_records_the_database_each_citation_came_from():
     assert result.cited_sources == ["alpha", "beta", "alpha"]
 
 
-def test_an_unnamed_database_records_no_source():
-    """One database names nothing: the field holds an empty string."""
+def test_a_hand_built_citation_without_a_source_records_an_empty_string():
+    """A citation built without a source is recorded as an empty string."""
     from haiku.rag.capabilities._base import EvidenceState
     from haiku.rag.capabilities.ledger import CapabilityEvidenceRecord
     from haiku.rag.store.models.citation import Citation

evaluations/tests/test_mtrag_federated.py

@@ -1,480 +0,0 @@
-import os
-import subprocess
-import sys
-
-import pytest
-
-from evaluations.datasets import DATASETS
-from evaluations.datasets.mtrag_federated import (
-    DEFAULT_BUDGET,
-    DOMAINS,
-    FTSIndexNotCoveringRows,
-    GOLD_TITLE_FLOOR,
-    MTRAG_FEDERATED_SPEC,
-    assert_fts_covers_rows,
-    collection_names,
-    collection_of,
-    domain_files,
-    partition_pooled,
-    pooled_collection_names,
-    pooled_database_paths,
-    sample_pooled_records,
-    partition_records,
-    pool_composition,
-    sample_records,
-)
-
-
-def _smoke_config():
-    """A config placing two databases, so the run resolves a federated client."""
-    from haiku.rag.config.models import AppConfig
-
-    return AppConfig.model_validate(
-        {
-            "lancedb": {
-                "databases": {
-                    "clapnq_0": "/tmp/a.lancedb",
-                    "clapnq_1": "/tmp/b.lancedb",
-                }
-            }
-        }
-    )
-
-
-def record(passage_id: str, title: str) -> dict[str, str]:
-    return {"_id": passage_id, "title": title, "text": f"text of {passage_id}"}
-
-
-def corpus(titles: dict[str, int]) -> list[dict[str, str]]:
-    """One record per passage, `titles` mapping a title to its passage count."""
-    return [
-        record(f"{title}_{index}", title)
-        for title, count in titles.items()
-        for index in range(count)
-    ]
-
-
-class TestCollectionOf:
-    def test_assigns_within_range(self) -> None:
-        for n in (2, 4, 8):
-            assigned = {collection_of(f"title {i}", n) for i in range(200)}
-            assert assigned <= set(range(n))
-
-    def test_uses_every_collection(self) -> None:
-        """A partition that leaves a collection empty is not a partition."""
-        for n in (2, 4, 8):
-            assigned = {collection_of(f"title {i}", n) for i in range(200)}
-            assert assigned == set(range(n))
-
-    def test_is_stable_across_processes(self) -> None:
-        """Salted `hash()` would make a build unreproducible between runs.
-
-        The partition is never stored, so scoring recomputes it in a different
-        process than the one that ingested.
-        """
-        code = (
-            "from evaluations.datasets.mtrag_federated import collection_of;"
-            "print([collection_of(f'title {i}', 8) for i in range(12)])"
-        )
-        runs = {
-            subprocess.run(
-                [sys.executable, "-c", code],
-                capture_output=True,
-                text=True,
-                check=True,
-                env={**os.environ, "PYTHONHASHSEED": seed},
-            ).stdout.strip()
-            for seed in ("0", "1", "12345")
-        }
-        assert len(runs) == 1, f"assignment varies with PYTHONHASHSEED: {runs}"
-
-    def test_seed_changes_the_assignment(self) -> None:
-        titles = [f"title {i}" for i in range(200)]
-        one = [collection_of(t, 8, seed=1) for t in titles]
-        two = [collection_of(t, 8, seed=2) for t in titles]
-        assert one != two
-
-    def test_rejects_a_collection_count_below_one(self) -> None:
-        with pytest.raises(ValueError, match="at least one collection"):
-            collection_of("title", 0)
-
-
-class TestCollectionNames:
-    def test_names_one_per_collection(self) -> None:
-        assert collection_names(3) == (
-            "clapnq_0",
-            "clapnq_1",
-            "clapnq_2",
-        )
-
-    def test_declaration_order_is_the_name_order(self) -> None:
-        """Fusion resolves ties to configured order, so the order is load-bearing."""
-        names = collection_names(4)
-        assert list(names) == sorted(names, key=lambda name: int(name.split("_")[1]))
-
-
-class TestPartitionRecords:
-    def test_keeps_every_record(self) -> None:
-        records = corpus({"a": 3, "b": 2, "c": 4})
-        grouped = partition_records(records, 2)
-        assert sum(len(rows) for rows in grouped.values()) == len(records)
-
-    def test_never_splits_a_title(self) -> None:
-        """A title is the atom: its passages must share a collection, or a
-        query's gold spreads for reasons the partition never intended."""
-        records = corpus({f"title {i}": 5 for i in range(40)})
-        grouped = partition_records(records, 4)
-        holders: dict[str, set[str]] = {}
-        for name, rows in grouped.items():
-            for row in rows:
-                holders.setdefault(row["title"], set()).add(name)
-        split = {title: names for title, names in holders.items() if len(names) > 1}
-        assert not split, f"titles split across collections: {split}"
-
-    def test_names_every_collection_even_when_one_is_empty(self) -> None:
-        """The config declares n databases, so the build must create n."""
-        records = corpus({"only": 2})
-        grouped = partition_records(records, 4)
-        assert set(grouped) == set(collection_names(4))
-
-
-class TestSampleRecords:
-    def test_keeps_every_gold_passage(self) -> None:
-        records = corpus({f"title {i}": 10 for i in range(50)})
-        gold = {"title 3_1", "title 17_4", "title 42_9"}
-        sampled = sample_records(records, gold, budget=60)
-        assert gold <= {row["_id"] for row in sampled}
-
-    def test_keeps_whole_titles_holding_gold(self) -> None:
-        records = corpus({f"title {i}": 10 for i in range(50)})
-        sampled = sample_records(records, {"title 3_1"}, budget=0)
-        assert sorted(row["_id"] for row in sampled) == sorted(
-            f"title 3_{i}" for i in range(10)
-        )
-
-    def test_respects_the_budget(self) -> None:
-        records = corpus({f"title {i}": 10 for i in range(50)})
-        sampled = sample_records(records, {"title 3_1"}, budget=100)
-        assert len(sampled) <= 100
-
-    def test_budget_below_the_gold_floor_still_keeps_gold(self) -> None:
-        records = corpus({f"title {i}": 10 for i in range(50)})
-        gold = {f"title {i}_0" for i in range(20)}
-        sampled = sample_records(records, gold, budget=5)
-        assert len(sampled) == 200
-        assert gold <= {row["_id"] for row in sampled}
-
-    def test_is_stable_for_a_seed(self) -> None:
-        records = corpus({f"title {i}": 10 for i in range(50)})
-        first = sample_records(records, {"title 0_0"}, budget=100, seed=7)
-        second = sample_records(records, {"title 0_0"}, budget=100, seed=7)
-        assert [row["_id"] for row in first] == [row["_id"] for row in second]
-
-    def test_seed_changes_the_distractors(self) -> None:
-        records = corpus({f"title {i}": 10 for i in range(50)})
-        first = sample_records(records, {"title 0_0"}, budget=100, seed=7)
-        second = sample_records(records, {"title 0_0"}, budget=100, seed=8)
-        assert {row["_id"] for row in first} != {row["_id"] for row in second}
-
-    def test_rejects_gold_the_corpus_does_not_hold(self) -> None:
-        records = corpus({"a": 2})
-        with pytest.raises(ValueError, match="do not resolve"):
-            sample_records(records, {"missing"}, budget=10)
-
-
-class TestSpec:
-    def test_registers_under_its_key(self) -> None:
-        assert DATASETS[MTRAG_FEDERATED_SPEC.key] is MTRAG_FEDERATED_SPEC
-
-    def test_opts_out_of_the_shared_population(self) -> None:
-        """The databases are built by build_databases, not populate_db."""
-        with pytest.raises(RuntimeError, match="build_databases"):
-            MTRAG_FEDERATED_SPEC.document_loader()
-
-    def test_retrieval_limit_matches_the_product_default(self) -> None:
-        """5 is config's search.limit, the setting the depth quota bites at."""
-        assert MTRAG_FEDERATED_SPEC.retrieval_limit == 5
-
-    def test_scores_retrieval_without_a_judge(self) -> None:
-        assert MTRAG_FEDERATED_SPEC.retrieval_evaluators
-        assert MTRAG_FEDERATED_SPEC.retrieval_loader is not None
-        assert MTRAG_FEDERATED_SPEC.retrieval_mapper is not None
-
-
-class TestPoolComposition:
-    def test_separates_gold_bearing_titles_from_distractors(self) -> None:
-        records = corpus({"answers": 4, "filler": 6})
-        gold_side, distractors = pool_composition(records, {"answers_2"})
-        assert (gold_side, distractors) == (4, 6)
-
-    def test_reports_no_distractors_when_the_budget_is_at_the_floor(self) -> None:
-        """A pool of only answer-bearing articles scores as an easy task and
-        says nothing, so the build has to be able to see it."""
-        records = corpus({f"title {i}": 10 for i in range(5)})
-        gold = {f"title {i}_0" for i in range(5)}
-        sampled = sample_records(records, gold, budget=1)
-        assert pool_composition(sampled, gold) == (50, 0)
-
-    def test_the_default_budget_clears_the_gold_floor(self) -> None:
-        assert DEFAULT_BUDGET > GOLD_TITLE_FLOOR
-
-
-class TestRetrievalLimitOverride:
-    async def test_override_replaces_the_spec_value(self, monkeypatch) -> None:
-        """Fetch depth is a run knob: hybrid search degenerates below roughly 50
-        candidates, so every regime would otherwise need its own dataset."""
-        seen: list[int | None] = []
-
-        async def fake_search(self, query, limit=None, **kwargs):  # noqa: ANN001
-            seen.append(limit)
-            return []
-
-        from haiku.rag.client import HaikuRAG
-
-        monkeypatch.setattr(HaikuRAG, "search", fake_search)
-        from evaluations.retrieval import run_retrieval_benchmark
-
-        await run_retrieval_benchmark(
-            MTRAG_FEDERATED_SPEC,
-            _smoke_config(),
-            limit=1,
-            retrieval_limit=77,
-        )
-        assert seen and set(seen) == {77}
-
-    async def test_spec_value_is_the_default(self, monkeypatch) -> None:
-        seen: list[int | None] = []
-
-        async def fake_search(self, query, limit=None, **kwargs):  # noqa: ANN001
-            seen.append(limit)
-            return []
-
-        from haiku.rag.client import HaikuRAG
-
-        monkeypatch.setattr(HaikuRAG, "search", fake_search)
-        from evaluations.retrieval import run_retrieval_benchmark
-
-        await run_retrieval_benchmark(MTRAG_FEDERATED_SPEC, _smoke_config(), limit=1)
-        assert seen and set(seen) == {MTRAG_FEDERATED_SPEC.retrieval_limit}
-
-
-class TestFTSCoverageAssertion:
-    """The chunks FTS index is built once over zero rows and only an optimize
-    folds later rows in, so a build that skips it ships dead full-text search
-    that still returns results."""
-
-    class _Index:
-        def __init__(self, name: str) -> None:
-            self.name = name
-
-    class _Stats:
-        def __init__(self, indexed: int) -> None:
-            self.num_indexed_rows = indexed
-
-    class _Table:
-        def __init__(self, rows: int, indexed: int | None) -> None:
-            self._rows = rows
-            self._indexed = indexed
-
-        async def count_rows(self) -> int:
-            return self._rows
-
-        async def list_indices(self):
-            if self._indexed is None:
-                return []
-            return [TestFTSCoverageAssertion._Index("content_fts_idx")]
-
-        async def index_stats(self, name: str):
-            assert name == "content_fts_idx"
-            return TestFTSCoverageAssertion._Stats(self._indexed or 0)
-
-    async def test_passes_when_the_index_covers_every_row(self) -> None:
-        await assert_fts_covers_rows(self._Table(100, 100), "clapnq_0")
-
-    async def test_rejects_a_zero_row_index(self) -> None:
-        with pytest.raises(FTSIndexNotCoveringRows, match="covers 0 of 100"):
-            await assert_fts_covers_rows(self._Table(100, 0), "clapnq_0")
-
-    async def test_rejects_a_partially_covering_index(self) -> None:
-        with pytest.raises(FTSIndexNotCoveringRows, match="covers 60 of 100"):
-            await assert_fts_covers_rows(self._Table(100, 60), "clapnq_0")
-
-    async def test_rejects_a_missing_index(self) -> None:
-        with pytest.raises(FTSIndexNotCoveringRows, match="no content_fts_idx"):
-            await assert_fts_covers_rows(self._Table(100, None), "clapnq_0")
-
-
-class TestDomains:
-    def test_names_the_four_upstream_domains(self) -> None:
-        assert DOMAINS == ("clapnq", "cloud", "fiqa", "govt")
-
-    def test_paths_follow_the_upstream_layout(self) -> None:
-        assert domain_files("govt") == (
-            "corpora/passage_level/govt.jsonl.zip",
-            "mtrag-human/retrieval_tasks/govt/qrels/dev.tsv",
-            "mtrag-human/retrieval_tasks/govt/govt_lastturn.jsonl",
-        )
-
-
-class TestDomainPartition:
-    """With four real domains, alpha finally means something: 0 keeps a
-    collection to one topic, 1 shards titles across all of them."""
-
-    def test_alpha_zero_keeps_a_domain_together_when_n_matches(self) -> None:
-        for domain_index, domain in enumerate(DOMAINS):
-            assigned = {
-                collection_of(f"{domain} title {i}", 4, alpha=0.0, domain=domain)
-                for i in range(50)
-            }
-            assert assigned == {domain_index}
-
-    def test_alpha_zero_subdivides_within_a_domain_when_n_exceeds_it(self) -> None:
-        for domain_index, domain in enumerate(DOMAINS):
-            assigned = {
-                collection_of(f"{domain} title {i}", 8, alpha=0.0, domain=domain)
-                for i in range(200)
-            }
-            assert assigned == {domain_index * 2, domain_index * 2 + 1}
-
-    def test_alpha_zero_groups_domains_when_n_is_below_it(self) -> None:
-        assigned = {
-            (domain, collection_of(f"t{i}", 2, alpha=0.0, domain=domain))
-            for domain in DOMAINS
-            for i in range(20)
-        }
-        by_collection: dict[int, set[str]] = {}
-        for domain, collection in assigned:
-            by_collection.setdefault(collection, set()).add(domain)
-        assert set(by_collection) == {0, 1}
-        assert all(len(v) == 2 for v in by_collection.values())
-
-    def test_alpha_one_ignores_the_domain(self) -> None:
-        """The shard endpoint: a title's collection must not depend on its domain."""
-        titles = [f"title {i}" for i in range(200)]
-        as_clapnq = [collection_of(t, 8, alpha=1.0, domain="clapnq") for t in titles]
-        as_govt = [collection_of(t, 8, alpha=1.0, domain="govt") for t in titles]
-        assert as_clapnq == as_govt
-
-    def test_alpha_one_spreads_a_single_domain_across_every_collection(self) -> None:
-        assigned = {
-            collection_of(f"title {i}", 8, alpha=1.0, domain="clapnq")
-            for i in range(400)
-        }
-        assert assigned == set(range(8))
-
-    def test_intermediate_alpha_moves_some_titles_off_their_domain(self) -> None:
-        titles = [f"title {i}" for i in range(400)]
-        home = [collection_of(t, 4, alpha=0.0, domain="fiqa") for t in titles]
-        mixed = [collection_of(t, 4, alpha=0.5, domain="fiqa") for t in titles]
-        moved = sum(1 for a, b in zip(home, mixed) if a != b)
-        assert 0 < moved < len(titles), f"alpha=0.5 moved {moved} of {len(titles)}"
-
-    def test_default_alpha_is_the_domain_partition(self) -> None:
-        for domain in DOMAINS:
-            assert collection_of("t", 4, domain=domain) == collection_of(
-                "t", 4, alpha=0.0, domain=domain
-            )
-
-
-class TestPooledPartition:
-    def test_names_are_distinct_from_the_single_domain_set(self) -> None:
-        """The two datasets must never share database paths."""
-        assert not set(pooled_collection_names(4)) & set(collection_names(4))
-
-    def test_database_paths_separate_alpha_and_n(self) -> None:
-        a = pooled_database_paths(4, 0.0)
-        b = pooled_database_paths(4, 1.0)
-        c = pooled_database_paths(8, 0.0)
-        assert not set(a.values()) & set(b.values())
-        assert not set(a.values()) & set(c.values())
-
-    def test_routes_each_record_by_its_own_domain(self) -> None:
-        records = [
-            {
-                "_id": f"{domain}-{i}",
-                "title": f"{domain} t{i}",
-                "text": "x",
-                "domain": domain,
-            }
-            for domain in DOMAINS
-            for i in range(20)
-        ]
-        grouped = partition_pooled(records, 4, alpha=0.0)
-        for name, rows in grouped.items():
-            domains = {row["domain"] for row in rows}
-            assert len(domains) == 1, f"{name} mixes domains at alpha=0: {domains}"
-
-    def test_alpha_one_mixes_domains_in_every_collection(self) -> None:
-        records = [
-            {"_id": f"{domain}-{i}", "title": f"t{i}", "text": "x", "domain": domain}
-            for domain in DOMAINS
-            for i in range(60)
-        ]
-        grouped = partition_pooled(records, 4, alpha=1.0)
-        assert all(len({r["domain"] for r in rows}) > 1 for rows in grouped.values())
-
-    def test_keeps_every_record(self) -> None:
-        records = [
-            {"_id": f"{d}-{i}", "title": f"{d} t{i}", "text": "x", "domain": d}
-            for d in DOMAINS
-            for i in range(15)
-        ]
-        for alpha in (0.0, 0.5, 1.0):
-            grouped = partition_pooled(records, 8, alpha=alpha)
-            assert sum(len(v) for v in grouped.values()) == len(records)
-
-
-class TestPooledSampler:
-    """Two of the four domains have no titles at all, so the pooled corpus is
-    sampled and partitioned at passage level rather than by title."""
-
-    @staticmethod
-    def _pool(per_domain: int = 40) -> list[dict[str, str]]:
-        return [
-            {
-                "_id": f"{domain}-{i}",
-                # cloud and fiqa carry an empty title upstream.
-                "title": "" if domain in ("cloud", "fiqa") else f"{domain} t{i}",
-                "text": "x",
-                "domain": domain,
-            }
-            for domain in DOMAINS
-            for i in range(per_domain)
-        ]
-
-    def test_keeps_every_gold_passage(self) -> None:
-        pool = self._pool()
-        gold = {"cloud-3", "fiqa-7", "clapnq-1", "govt-39"}
-        kept = sample_pooled_records(pool, gold, budget=20)
-        assert gold <= {row["_id"] for row in kept}
-
-    def test_respects_the_budget_above_the_gold_floor(self) -> None:
-        pool = self._pool()
-        kept = sample_pooled_records(pool, {"cloud-3"}, budget=25)
-        assert len(kept) == 25
-
-    def test_a_titleless_domain_does_not_drag_in_its_whole_corpus(self) -> None:
-        """The failure this replaces: whole-title keeping pulled all 72,442 cloud
-        passages in because they share one empty title."""
-        pool = self._pool()
-        kept = sample_pooled_records(pool, {"cloud-3"}, budget=10)
-        cloud = [row for row in kept if row["domain"] == "cloud"]
-        assert len(cloud) < 40, f"kept {len(cloud)} of 40 cloud passages"
-
-    def test_rejects_gold_the_pool_does_not_hold(self) -> None:
-        with pytest.raises(ValueError, match="do not resolve"):
-            sample_pooled_records(self._pool(), {"nope-1"}, budget=10)
-
-    def test_partition_is_passage_level_not_title_level(self) -> None:
-        """A titleless domain must still spread across its own collections."""
-        pool = self._pool(per_domain=200)
-        grouped = partition_pooled(pool, 8, alpha=0.0)
-        cloud_collections = {
-            name
-            for name, rows in grouped.items()
-            if any(row["domain"] == "cloud" for row in rows)
-        }
-        assert len(cloud_collections) == 2, (
-            f"cloud landed in {len(cloud_collections)} collections; with one empty "
-            "title a title-keyed partition would give 1"
-        )

examples/README.md

@@ -26,8 +26,8 @@ uv run python examples/custom_agent.py /path/to/db.lancedb
 
 **Script:** `custom_agent_agui.py`
 
-A Starlette app that adapts a native RAG-capable agent to AG-UI.
+A Starlette app that adapts a native RAG-capable agent to AG-UI. The configuration places the database (`HAIKU_RAG_CONFIG_PATH`, or `./haiku.rag.yaml`):
 
 ```bash
-DB_PATH=/path/to/db.lancedb uv run uvicorn examples.custom_agent_agui:app --reload --port 8000
+uv run uvicorn examples.custom_agent_agui:app --reload --port 8000
 ```

examples/custom_agent_agui.py

@@ -8,13 +8,13 @@ Requirements:
     - An Anthropic API key (for the QA model) or adjust the model below
 
 Usage:
-    DB_PATH=/path/to/db.lancedb uv run uvicorn examples.custom_agent_agui:app --reload --port 8000
+    uv run uvicorn examples.custom_agent_agui:app --reload --port 8000
+
+    The configuration places the database (HAIKU_RAG_CONFIG_PATH, or
+    ./haiku.rag.yaml).
 """
 
-import os
-import sys
 from dataclasses import dataclass, field
-from pathlib import Path
 from typing import Any
 
 from ag_ui.core import EventType, StateSnapshotEvent
@@ -30,14 +30,7 @@ from haiku.rag.capabilities.compaction import create_capability as compaction
 from haiku.rag.capabilities.policy import create_capability as citation_policy
 from haiku.rag.capabilities.rag import RAGState, create_capability
 
-db_path = os.environ.get("DB_PATH")
-if not db_path:
-    print(
-        "Set DB_PATH environment variable to your haiku.rag database", file=sys.stderr
-    )
-    sys.exit(1)
-
-capability = create_capability(db_path=Path(db_path), defer_loading=False)
+capability = create_capability(defer_loading=False)
 
 
 @dataclass

examples/docker/docker-compose.yml

@@ -103,7 +103,6 @@ services:
         "haiku-rag",
         "--config",
         "/app/haiku.rag.yaml",
-        "--read-only",
         "mcp",
         "--host",
         "0.0.0.0",

haiku_rag_slim/haiku/rag/app.py

@@ -1,5 +1,4 @@
 import logging
-from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -64,21 +63,10 @@ class HaikuRAGApp:
         [ref] = self.scope.databases
         return ref
 
-    @cached_property
-    def _connection(self) -> "tuple[AppConfig, Path]":
-        """How to open the one database this command works on, directly.
-
-        Derived per database from its configured location.
-        """
-        from haiku.rag.client.session import default_db_path
-
-        config, db_path = self._one.connection(self.config)
-        return config, db_path or default_db_path(config)
-
     @property
-    def _store_config(self) -> AppConfig:
-        """The configuration for opening the one database directly."""
-        return self._connection[0]
+    def _location(self) -> "Path | str":
+        """Where the one database this command works on is."""
+        return self._one.location
 
     @property
     def _is_local(self) -> bool:
@@ -91,17 +79,14 @@ class HaikuRAGApp:
 
     @property
     def _path(self) -> Path:
-        """The path of the one database this command works on.
-
-        A database behind a URI has none of its own, and the default stands in:
-        the URI in `_store_config` is what decides where it connects.
-        """
-        return self._connection[1]
+        """The path of the one local database this command works on."""
+        assert self._one.db_path is not None
+        return self._one.db_path
 
     @property
     def display_path(self) -> "Path | str":
         """What a one-database command calls the database it opened."""
-        return self._one.db_path or self._one.uri
+        return self._one.location
 
     @property
     def database_missing(self) -> bool:
@@ -140,7 +125,7 @@ class HaikuRAGApp:
             self.console.print("[red]Database path does not exist.[/red]")
             return
 
-        info = await gather_database_info(self._store_config, self._path)
+        info = await gather_database_info(self._location, self.config)
 
         if not info.exists:
             self.console.print(
@@ -282,8 +267,8 @@ class HaikuRAGApp:
         cm = status if status is not None else nullcontext()
         with cm:
             report = await run_doctor(
-                self._store_config,
-                self._path,
+                self.config,
+                self._location,
                 dict(os.environ),
                 duplicates_out=duplicates_out,
                 on_progress=on_progress,
@@ -340,8 +325,8 @@ class HaikuRAGApp:
             return
 
         async with Store(
-            self._path,
-            config=self._store_config,
+            self._location,
+            config=self.config,
             skip_validation=True,
             read_only=True,
             skip_migration_check=True,
@@ -415,15 +400,15 @@ class HaikuRAGApp:
         """
         from haiku.rag.store.engine import Store
 
-        return Store(self._path, config=self._store_config, read_only=self.read_only)
+        return Store(self._location, config=self.config, read_only=self.read_only)
 
     def _tag_read_store(self) -> "Store":
         """Read-only store for tag inspection; works on old or drifted DBs."""
         from haiku.rag.store.engine import Store
 
         return Store(
-            self._path,
-            config=self._store_config,
+            self._location,
+            config=self.config,
             skip_validation=True,
             skip_migration_check=True,
             read_only=True,
@@ -760,8 +745,8 @@ class HaikuRAGApp:
         from haiku.rag.store.engine import Store
 
         async with Store(
-            self._path,
-            config=self._store_config,
+            self._location,
+            config=self.config,
             skip_validation=True,
             skip_migration_check=True,
             read_only=self.read_only,
@@ -949,7 +934,7 @@ class HaikuRAGApp:
         # The resolved scope: a path overrides a configured URI, and a derived
         # single-database configuration drops the name results and citations
         # carry.
-        server = _mcp_server_covering(self.scope, self.config, self.read_only)
+        server = _mcp_server_covering(self.scope, self.config)
         try:
             if transport == "stdio":
                 await server.run_stdio_async()

haiku_rag_slim/haiku/rag/capabilities/_base.py

@@ -1,5 +1,4 @@
 import asyncio
-import os
 from dataclasses import dataclass, field, replace
 from difflib import get_close_matches
 from pathlib import Path
@@ -14,6 +13,7 @@ from pydantic_ai import (
 )
 from pydantic_ai.capabilities import AbstractCapability
 from pydantic_ai.messages import (
+    BinaryContent,
     InstructionPart,
     ModelMessage,
     ModelRequest,
@@ -29,6 +29,7 @@ from pydantic_ai.toolsets import AgentToolset
 
 from haiku.rag.capabilities._tools import (
     CodeExecutionEntry,
+    EvidenceKey,
     merge_results,
     search_corpus,
 )
@@ -43,7 +44,7 @@ from haiku.rag.store.models.citation import (
     ambiguous_citation,
     resolve_citations,
 )
-from haiku.rag.tools.search import build_image_content_from_results
+from haiku.rag.tools.search import PictureKey, build_image_content_from_results
 
 CITATION_GRACE_REQUESTS = 2
 """Requests calling this capability's tools that its cite tool outlives the rest by.
@@ -61,6 +62,15 @@ Calibration knob. Two unrelated UUID4s reach about 0.5, while dropping or
 duplicating a character or a whole group stays above 0.75, so the gap is wide.
 """
 
+FREE_SIBLINGS_PER_ROUND = 3
+"""Searches one budget unit covers when emitted in the same model response.
+
+Calibration knob, sized to the measured modal burst. ``qa.max_searches``
+counts units, so a model rephrasing its query a few times in one response
+spends one unit, while every search of a sequential searcher is a unit of its
+own.
+"""
+
 
 def _ambiguous_retry(error: AmbiguousCitationError) -> ModelRetry:
     """The only way out of an id that names a chunk in two databases.
@@ -88,12 +98,7 @@ def _nearest_known_id(chunk_id: str, known_ids: list[str]) -> str:
 
 
 def resolve_scope(db_path: Path | str | None, config: AppConfig) -> DatabaseScope:
-    """The databases a capability covers, resolved once at its entry point.
-
-    ``HAIKU_RAG_DB`` is read here and nowhere else.
-    """
-    if db_path is None and (env_db := os.environ.get("HAIKU_RAG_DB")):
-        db_path = Path(env_db).expanduser()
+    """The databases a capability covers, resolved once at its entry point."""
     return DatabaseScope.resolve(config, database_path=db_path)
 
 
@@ -177,6 +182,12 @@ class RAGCapabilityBase[StateT: EvidenceState](AbstractCapability[Any]):
     rag_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
     resource_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
     search_count: int = field(default=0, repr=False)
+    search_step: int = field(default=0, repr=False)
+    """The run_step whose searches are being priced and deduplicated."""
+    step_searches: int = field(default=0, repr=False)
+    step_rejected: bool = field(default=False, repr=False)
+    step_shown: set[EvidenceKey] = field(default_factory=set, repr=False)
+    step_pictures: set[PictureKey] = field(default_factory=set, repr=False)
     request_count: int = field(default=0, repr=False)
     grace_requests_used: int = field(default=0, repr=False)
     epoch: int = field(default=0, repr=False)
@@ -226,6 +237,11 @@ class RAGCapabilityBase[StateT: EvidenceState](AbstractCapability[Any]):
             rag_lock=asyncio.Lock(),
             resource_lock=asyncio.Lock(),
             search_count=0,
+            search_step=0,
+            step_searches=0,
+            step_rejected=False,
+            step_shown=set(),
+            step_pictures=set(),
             request_count=0,
             grace_requests_used=0,
             epoch=0,
@@ -493,32 +509,54 @@ class RAGCapabilityBase[StateT: EvidenceState](AbstractCapability[Any]):
             retrieved_now=retrieved,
         )
 
-    async def _search(self, query: str, limit: int | None) -> str | ToolReturn:
+    async def _search(
+        self, query: str, limit: int | None, run_step: int
+    ) -> str | ToolReturn:
         assert self.state is not None
-        self.search_count += 1
-        if self.search_count > self._max_searches:
+        if run_step != self.search_step:
+            self.search_step = run_step
+            self.step_searches = 0
+            self.step_rejected = False
+            self.step_shown = set()
+            self.step_pictures = set()
+        self.step_searches += 1
+        if (self.step_searches - 1) % FREE_SIBLINGS_PER_ROUND == 0:
+            self.search_count += 1
+        if self.step_rejected or self.search_count > self._max_searches:
+            self.step_rejected = True
             raise ToolFailed(
                 "Search limit reached. Answer the question using "
                 "the results you already have."
             )
         async with self.rag_lock:
-            formatted, results, include_collection = await search_corpus(
+            formatted, results, rendered, include_collection = await search_corpus(
                 await self._ensure_rag(),
                 query,
                 limit=limit,
                 document_filter=self.state.document_filter,
                 sources=self.state.sources,
+                shown=self.step_shown,
             )
+        parts: list[str | BinaryContent] = []
+        emitted: set[PictureKey] = set()
+        if self.vision:
+            parts, emitted = build_image_content_from_results(
+                results,
+                include_collection=include_collection,
+                exclude=self.step_pictures,
+            )
+        # Everything the search produced commits together, after formatting and
+        # image construction have both succeeded: a search that raises must not
+        # leave results citable, note evidence the model never received, or
+        # suppress a later sibling's results.
         state = self.state
         # A model can search the same query twice with different limits, and the
         # narrower return must not drop what the wider one already showed it.
         merge_results(state.searches.setdefault(query, []), results)
         self._note_evidence()
-        if self.vision and (
-            parts := build_image_content_from_results(
-                results, include_collection=include_collection
-            )
-        ):
+        self.step_shown |= rendered
+        self.step_pictures |= emitted
+        if parts:
             return ToolReturn(return_value=formatted, content=parts)
         return formatted
 

haiku_rag_slim/haiku/rag/capabilities/_tools.py

@@ -1,9 +1,11 @@
 from collections.abc import Iterable
+from collections.abc import Set as AbstractSet
 
 from pydantic import BaseModel
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.store.models.chunk import SearchResult, qualified_id
+from haiku.rag.tools.search import picture_keys
 
 
 class CodeExecutionEntry(BaseModel):
@@ -13,14 +15,48 @@ class CodeExecutionEntry(BaseModel):
     success: bool = True
 
 
+EvidenceKey = tuple[tuple[str | None, str | None], tuple[str, frozenset]]
+"""What tells one rendered result from another: qualified id, then signature.
+
+The qualified id comes first because the rendered string alone would conflate
+identical renderings of the same chunk id held by two databases.
+"""
+
+
+def evidence_signature(result: SearchResult, include_collection: bool) -> tuple:
+    """The rendered evidence a result shows the model, as an equivalence key.
+
+    Rank and total are held at neutral values: they vary with a result's
+    position, and position (like score) must not tell two renderings apart.
+    """
+    return (
+        result.format_for_agent(rank=0, total=0, include_collection=include_collection),
+        picture_keys(result),
+    )
+
+
+def evidence_key(result: SearchResult, include_collection: bool) -> EvidenceKey:
+    return (
+        qualified_id(result.source, result.chunk_id),
+        evidence_signature(result, include_collection),
+    )
+
+
 async def search_corpus(
     rag: HaikuRAG,
     query: str,
     limit: int | None = None,
     document_filter: str | None = None,
     sources: list[str] | None = None,
-) -> tuple[str, list[SearchResult], bool]:
-    """Search and context-expand results, and whether they name their collection."""
+    shown: AbstractSet[EvidenceKey] = frozenset(),
+) -> tuple[str, list[SearchResult], set[EvidenceKey], bool]:
+    """Search and context-expand results, eliding evidence already shown.
+
+    Returns the formatted results, the full result list, the evidence keys the
+    formatting rendered in full, and whether results name their collection. A
+    result whose key is in ``shown`` keeps its slot but collapses to one line;
+    the result list is never filtered.
+    """
     results = await rag.search(
         query, limit=limit, filter=document_filter, sources=sources
     )
@@ -29,13 +65,25 @@ async def search_corpus(
     # two collections names them even when everything came back from one.
     selected = rag.source_names if sources is None else sources
     include_collection = len(set(selected)) > 1
-    formatted = "\n\n---\n\n".join(
-        result.format_for_agent(
-            rank=index + 1, total=len(results), include_collection=include_collection
-        )
-        for index, result in enumerate(results)
-    )
-    return formatted or "No results found.", list(results), include_collection
+    rendered: set[EvidenceKey] = set()
+    parts: list[str] = []
+    total = len(results)
+    for index, result in enumerate(results):
+        key = evidence_key(result, include_collection)
+        if key in shown or key in rendered:
+            parts.append(
+                f"Also matched, shown above: [{result.chunk_id}] "
+                f"[rank {index + 1} of {total}]"
+            )
+        else:
+            parts.append(
+                result.format_for_agent(
+                    rank=index + 1, total=total, include_collection=include_collection
+                )
+            )
+            rendered.add(key)
+    formatted = "\n\n---\n\n".join(parts)
+    return formatted or "No results found.", list(results), rendered, include_collection
 
 
 def merge_results(
@@ -56,6 +104,9 @@ def merge_results(
 
 __all__ = [
     "CodeExecutionEntry",
+    "EvidenceKey",
+    "evidence_key",
+    "evidence_signature",
     "merge_results",
     "search_corpus",
 ]

haiku_rag_slim/haiku/rag/capabilities/analysis.py

@@ -19,7 +19,7 @@ from haiku.rag.capabilities._base import (
 )
 from haiku.rag.capabilities._tools import merge_results
 from haiku.rag.config.models import AppConfig
-from haiku.rag.sandbox import AnalysisContext, Sandbox
+from haiku.rag.sandbox import AnalysisContext, Sandbox, recovery_hint
 
 STATE_NAMESPACE = "analysis"
 _CAPABILITY_ID = "haiku-rag-analysis"
@@ -49,21 +49,6 @@ def multiple_collections_instructions() -> str:
     return _multiple_collections_path.read_text().rstrip()
 
 
-def _recovery_hint(stderr: str) -> str:
-    """Name the workaround for sandbox limits models trip over repeatedly.
-
-    The instructions already say file objects are not iterable, and models write
-    ``for line in open(...)`` regardless. Carrying the fix in the error gives
-    them something to act on for the retry.
-    """
-    if "TextIOWrapper" in stderr and "not iterable" in stderr:
-        return (
-            "\n\nHint: file objects cannot be iterated here. Read lines with "
-            '.readlines() or .read().split("\\n").'
-        )
-    return ""
-
-
 @dataclass
 class AnalysisCapability(RAGCapabilityBase[AnalysisState]):
     """Deferred capability for sandboxed computation over a RAG corpus."""
@@ -139,7 +124,7 @@ class AnalysisCapability(RAGCapabilityBase[AnalysisState]):
         )
         if not result.success:
             raise ToolFailed(
-                f"{result.stderr}{_recovery_hint(result.stderr)}"
+                f"{result.stderr}{recovery_hint(result.stderr)}"
                 f"\n\nOutput: {result.stdout}"
             )
         return result.stdout or "No output."
@@ -172,7 +157,7 @@ class AnalysisCapability(RAGCapabilityBase[AnalysisState]):
             ctx: RunContext[Any], query: str, limit: int | None = None
         ) -> str | ToolReturn:
             """Search the knowledge base for evidence to analyze."""
-            return await self._with_state(self._search(query, limit))
+            return await self._with_state(self._search(query, limit, ctx.run_step))
 
         async def analysis_execute_code(ctx: RunContext[Any], code: str) -> Any:
             """Execute Python against the sandboxed document filesystem."""

haiku_rag_slim/haiku/rag/capabilities/instructions/analysis.md

@@ -13,11 +13,11 @@ You can mix the two. The rule: always call `analysis_cite` before answering —
 Execute Python code in a sandboxed interpreter. Variables persist between calls — you can build state incrementally. Use `print()` to output results.
 
 Inside the code, these functions are available (use `await`):
-- `await search(query, limit=10)` → list of dicts with keys: chunk_id, content, document_id, document_title, document_uri, score, page_numbers, headings, doc_item_refs, labels, picture_refs (subset of doc_item_refs labeled `picture`)
-- `await list_documents()` → list of dicts with keys: id, title, uri, created_at
+- `await search(query, limit=10)` → list of dicts with keys: chunk_id, content, document_id, document_title, document_uri, score, page_numbers, headings, doc_item_refs, labels, picture_refs (subset of doc_item_refs labeled `picture`), chunk_meta (the matched chunk's stored metadata, custom keys included)
+- `await list_documents()` → list of dicts with keys: id, title, uri, created_at, metadata
 
-Available modules: `json`, `re`, `math`, `pathlib`
-Not supported: class inheritance and metaclasses, generators/yield, match statements, decorators, `collections`, iterating a file object (`for line in f`)
+Useful modules include `json`, `re`, `math`, `pathlib`, `datetime`, `collections`, `itertools`, `functools` and `dataclasses`. `decimal` and `statistics` do not exist.
+Not supported: class inheritance and metaclasses, generators/yield, match statements, iterating a file object (`for line in f`)
 
 ### analysis_search
 Search the knowledge base directly (outside code execution). Each result has a `Type:` (paragraph, table, code, list_item, picture). When the Type is `picture`, the corresponding figure may also be attached to the tool response as an image alongside the text — use it directly to answer questions about figures, diagrams, charts, screenshots.
@@ -39,16 +39,17 @@ All documents are mounted as a virtual filesystem at `/documents/`:
 
 ```
 /documents/{document_id}/
-    metadata.json    # {"id", "title", "uri", "created_at"}
+    metadata.json    # {"id", "title", "uri", "created_at", "metadata"}
     content.txt      # Full document text
     items.jsonl      # Structured items (one JSON object per line)
+    chunks.jsonl     # Chunks in order with their metadata (one JSON object per line)
     toc.json         # Section tree derived from heading_level
 ```
 
 `{document_id}` is an internal identifier, not the user-facing `uri` (filename, URL, etc.). When you only know a document by its URI or title, use `await list_documents()` to enumerate ids and match against `uri` / `title` — that's a single call to the host. Iterating `/documents/` and reading every `metadata.json` works too but is much slower on portal-scale corpora.
 
 ### Reading files
-Read with `Path.read_text()` or `open()` (including `with` blocks); file objects support `.read()`, `.readline()`, and `.readlines()`. A file object cannot be iterated, so read line-wise with `.readlines()` or `.read().split("\n")` instead of `for line in f`. Files are read-only; writing raises `PermissionError`.
+Read with `Path.read_text()` or `open()` (including `with` blocks); file objects support `.read()`, `.readline()`, and `.readlines()`. A file object cannot be iterated, so read line-wise with `.readlines()` or `.read().split("\n")` instead of `for line in f`. Files are read-only; writing raises `PermissionError`. There is no network. A call has a time limit, named in the error when it is hit, and output past a size is cut with an `... (output truncated)` marker.
 
 ```python
 from pathlib import Path
@@ -70,7 +71,7 @@ for line in Path(f'/documents/{doc_id}/items.jsonl').read_text().strip().split("
 ```
 
 ### metadata.json
-Document metadata: `id`, `title`, `uri`, `created_at`.
+Document metadata: `id`, `title`, `uri`, `created_at`, and `metadata`, the keys stored with the document.
 
 ### content.txt
 Full text content. Use for regex or keyword search across a whole document.
@@ -86,6 +87,9 @@ Each row carries:
 - `chunk_ids`: chunks that contain this item — pass to `analysis_cite()` to ground an answer that read this item directly
 - `heading_level`: H-level for `section_header` rows; `0` on non-header rows
 
+### chunks.jsonl
+The document's chunks in order, one JSON object per line: `chunk_id` and `metadata`, the chunk's stored metadata (`doc_item_refs`, `headings`, `labels`, `page_numbers`, and any custom keys such as paragraph or footnote numbers). To read by chunk metadata, keep the matching rows and take the `items.jsonl` rows whose `chunk_ids` name them.
+
 ### toc.json
 Section tree derived from `heading_level`: `{"doc_id", "title", "tree": [...]}` where each node has `{self_ref, level, title, page_numbers, item_range: [start, end_exclusive], chunk_ids, children}`. `item_range` is a line slice into `items.jsonl` — `items[start:end]`. `chunk_ids` aggregates the citable chunks across all items in the section — pass directly to `analysis_cite()` to ground a section-scoped answer without a corpus-wide `search()` call. `tree: []` for docs with no headers.
 
@@ -112,6 +116,6 @@ You MUST call `analysis_cite` before producing your final answer, every time, wi
 - Use `print()` to output results — the output is your only feedback
 - When you write code, execute it — don't describe what code would do. But not every question needs code; simple lookups are best answered by `analysis_search → analysis_cite`.
 - Use `await` for all async functions inside `analysis_execute_code` (`search`, `list_documents`)
-- Read files with `Path.read_text()` or `open()`/`with`. For lines use `.readlines()` or `.read().split("\n")`, never `for line in f`. The `collections` module is unavailable.
+- Read files with `Path.read_text()` or `open()`/`with`. For lines use `.readlines()` or `.read().split("\n")`, never `for line in f`.
 - Do NOT include chunk IDs or UUIDs in your answer text — your answer should read naturally. Use the `analysis_cite` tool separately to register citations. `cite{...}` markdown-style inline references do nothing; only an actual `analysis_cite` tool call registers a citation.
 - **Before you write your final answer, invoke the `analysis_cite` tool with the supporting chunk_ids, or with an empty list if there are none.** This is the last tool call before answering, every time.

haiku_rag_slim/haiku/rag/capabilities/rag.py

@@ -81,7 +81,7 @@ class RAGCapability(RAGCapabilityBase[RAGState]):
             ctx: RunContext[Any], query: str, limit: int | None = None
         ) -> str | ToolReturn:
             """Search the knowledge base using hybrid vector and full-text search."""
-            return await self._with_state(self._search(query, limit))
+            return await self._with_state(self._search(query, limit, ctx.run_step))
 
         async def rag_cite(ctx: RunContext[Any], chunk_ids: list[str]) -> Any:
             """Register exact search-result chunk IDs as citations for the answer."""

haiku_rag_slim/haiku/rag/chat/__init__.py

@@ -42,14 +42,8 @@ def run_chat(
         config.qa.model = model_config
         config.analysis.model = model_config
 
-    # The capabilities read the databases the scope covers, not what the
-    # configuration names: a `--db PATH` selection is outside the
-    # configuration, and a `--db-name NAME` selection is narrower than it.
-    if scope.covers_multiple:
-        capability_config, capability_db_path = config, None
-    else:
-        capability_config, capability_db_path = scope.databases[0].connection(config)
-
+    # The app opens the scope and lends that client to the capabilities, which
+    # read what `--db PATH` or `--db-name NAME` selected.
     enabled = capabilities or ["rag"]
     capability_list = []
     defer_loading = len(enabled) > 1
@@ -68,8 +62,7 @@ def run_chat(
 
         capability_list.append(
             create_capability(
-                db_path=capability_db_path,
-                config=capability_config,
+                config=config,
                 defer_loading=defer_loading,
                 vision=driving_model.vision,
             )
@@ -80,8 +73,7 @@ def run_chat(
 
         capability_list.append(
             create_capability(
-                db_path=capability_db_path,
-                config=capability_config,
+                config=config,
                 defer_loading=defer_loading,
                 vision=driving_model.vision,
             )

haiku_rag_slim/haiku/rag/chat/app.py

@@ -148,10 +148,12 @@ class ChatApp(App):
         # a client whose __aenter__ failed.
         await client.__aenter__()
         self.client = client
-        # Lent to the capabilities: already the databases they were built for,
-        # and one connection per database however many capabilities read it.
+        # Lent to the capabilities, with the scope it covers: one connection
+        # per database however many capabilities read it, and the analysis
+        # sandbox is built over the same selection.
         for capability in self._capabilities:
             capability.borrowed_rag = client
+            capability.scope = self.scope
 
         self._agent = Agent(
             self._model,
@@ -425,7 +427,8 @@ class ChatApp(App):
 
     def on_document_filter_modal_filter_changed(self, event: Any) -> None:
         """Scope the conversation to the selection: the filter carries the ids,
-        and `sources` restricts the search to the databases the selection names.
+        and over a set `sources` restricts the search to the databases the
+        selection names. One database needs no narrowing by source.
         """
         from haiku.rag.tools.filters import build_document_id_filter
 
@@ -434,10 +437,9 @@ class ChatApp(App):
         doc_filter = build_document_id_filter(
             sorted({doc_id for _, doc_id in event.selected})
         )
-        selected_sources = {source for source, _ in event.selected}
-        sources: list[str] | None = None
-        if selected_sources and None not in selected_sources:
-            sources = sorted(s for s in selected_sources if s is not None)
+        selected_sources = sorted({source for source, _ in event.selected if source})
+        covers_multiple = self.client is not None and self.client.covers_multiple
+        sources = selected_sources if covers_multiple and selected_sources else None
         for namespace, state_type in (
             (RAG_STATE_NAMESPACE, RAGState),
             (ANALYSIS_STATE_NAMESPACE, AnalysisState),

haiku_rag_slim/haiku/rag/chat/widgets/document_filter_modal.py

@@ -24,15 +24,18 @@ class DocumentCheckbox(Checkbox):
         self.doc_id = doc_id
 
 
-def _labelled(docs) -> list[tuple[str, str | None, str]]:
-    """Each document's label, database and id, sorted by label. The database is
-    named alongside the title, which a title alone does not say. Labels are
-    escaped: titles and database names are data, not Textual markup."""
+def _labelled(
+    docs, *, name_database: bool = False
+) -> list[tuple[str, str | None, str]]:
+    """Each document's label, database and id, sorted by label. Across several
+    databases the database is named alongside the title, which a title alone
+    does not say. Labels are escaped: titles and database names are data, not
+    Textual markup."""
     rows = [
         (
             escape(
                 f"{doc.title or doc.uri or doc.id}"
-                + (f"  ({doc.source})" if doc.source else "")
+                + (f"  ({doc.source})" if name_database and doc.source else "")
             ),
             doc.source,
             doc.id,
@@ -221,7 +224,9 @@ class DocumentFilterModal(ModalScreen):
             DocumentCheckbox(
                 label, source, doc_id, value=(source, doc_id) in self._selected
             )
-            for label, source, doc_id in _labelled(docs)
+            for label, source, doc_id in _labelled(
+                docs, name_database=self.client.covers_multiple
+            )
         ]
         if boxes:
             await filter_list.mount_all(boxes)

haiku_rag_slim/haiku/rag/cli.py

@@ -90,9 +90,10 @@ def resolve_scope(
     """The databases a command works on, resolved once.
 
     The CLI decides only what it alone knows: that `--db` and `--db-name` are
-    the same thing said twice, and whether this command can read more than one.
-    Everything else — an unknown name, a `lancedb.uri`, the default location —
-    is `DatabaseScope.resolve`'s to answer.
+    the same thing said twice, that a human typing `--db PATH` means that
+    database whatever is configured, and whether this command can read more
+    than one. Everything else — an unknown name, the default location — is
+    `DatabaseScope.resolve`'s to answer.
     """
     from haiku.rag.client.scope import DatabaseScope
 
@@ -100,9 +101,12 @@ def resolve_scope(
         raise AmbiguousDatabaseError(
             "pass --db or --db-name, not both: they name the same thing"
         )
-    scope = DatabaseScope.resolve(
-        get_config(), database_name=_db_name, database_path=db
-    )
+    if db is not None:
+        try:
+            return DatabaseScope.at(db)
+        except ValueError as error:
+            raise typer.BadParameter(str(error), param_hint="--db") from error
+    scope = DatabaseScope.resolve(get_config(), database_name=_db_name)
     if scope.covers_multiple and not covers_set:
         raise AmbiguousDatabaseError(
             f"lancedb.databases names {', '.join(sorted(scope.names))}; this "
@@ -882,7 +886,7 @@ def mcp(
     ),
 ) -> None:
     """Run the MCP server."""
-    app = create_app(db)
+    app = create_app(db, covers_set=True)
 
     transport = "stdio" if stdio else None
 

haiku_rag_slim/haiku/rag/client/__init__.py

@@ -21,7 +21,6 @@ from haiku.rag.client.session import (
     FederatedSession,
     SingleDatabaseSession,
     aclose_quietly,
-    default_db_path,
 )
 from haiku.rag.config import AppConfig, get_config
 from haiku.rag.converters import get_converter
@@ -129,25 +128,22 @@ class HaikuRAG:
         """Initialize the RAG client with a database path.
 
         Args:
-            db_path: Path or string path to the database. When omitted, resolves
-                ``lancedb.databases``, then ``lancedb.uri``, then the default
-                path under ``storage.data_dir``.
+            db_path: Path or string path to the database, named by its stem.
+                Valid where the configuration places no database; beside
+                ``lancedb.databases`` it raises ``AmbiguousDatabaseError``.
+                When omitted, the configured databases are covered, or the
+                default database ``haiku.rag`` under ``storage.data_dir``.
             config: Configuration to use. Defaults to the current global config.
             skip_validation: Whether to skip configuration validation on database load.
             create: Whether to create the database if it doesn't exist.
             read_only: Whether to open the database in read-only mode.
-            sources: Names from ``config.lancedb.databases`` this client covers,
-                None for all of them. Only that setting names databases, so a
-                name raises when ``lancedb.uri`` placed the database, and is
-                rejected alongside ``db_path``, which says the same thing
+            sources: Names of the databases this client covers, None for all of
+                them. Rejected alongside ``db_path``, which says the same thing
                 another way. ``[]`` raises too: a client over no database can do
                 nothing, unlike ``sources=[]`` on a search, which is a selection
                 of nothing to search.
         """
         self._configured = config if config is not None else get_config()
-        # What the caller configured, kept intact: entering derives a
-        # single-database configuration from it, and every re-entry derives
-        # from the configured set.
         self._config = self._configured
         self._requested_db_path = Path(db_path) if db_path is not None else None
         if self._requested_db_path is not None and sources is not None:
@@ -169,24 +165,31 @@ class HaikuRAG:
 
     @property
     def covers_multiple(self) -> bool:
-        """Whether this client reads from more than one database."""
-        return isinstance(self._session, FederatedSession)
+        """Whether this client reads from more than one database.
+
+        Known before the client enters: coverage is a fact of the resolved
+        scope.
+        """
+        if self._session is not None:
+            return isinstance(self._session, FederatedSession)
+        return self._resolve_scope().covers_multiple
 
     @property
     def source_names(self) -> tuple[str, ...]:
-        """The configured databases this client covers, in configured order.
+        """The databases this client covers, by name, in configured order.
 
-        A single database contributes its own name, or nothing where the
-        configuration named none.
+        Known before the client enters: coverage is a fact of the resolved
+        scope.
         """
         if isinstance(self._session, FederatedSession):
             return self._session.names
-        return () if self.source is None else (self.source,)
+        if isinstance(self._session, SingleDatabaseSession):
+            return (self._session.source,)
+        return self._resolve_scope().names
 
     @property
     def source(self) -> str | None:
-        """The configured database this client reads, or None while covering a
-        set or reading a database the configuration did not name."""
+        """The database this client reads, or None while covering a set."""
         if isinstance(self._session, SingleDatabaseSession):
             return self._session.source
         return None
@@ -339,15 +342,12 @@ class HaikuRAG:
             return self
 
         [ref] = scope.databases
-        self._config, db_path = ref.connection(self._configured)
-
         self._session = await SingleDatabaseSession(
-            db_path if db_path is not None else default_db_path(self._config),
+            ref,
             self._config,
             skip_validation=self._skip_validation,
             create=self._create,
             read_only=self._read_only,
-            source=ref.name,
         ).open()
         return self
 
@@ -410,9 +410,8 @@ class HaikuRAG:
 
         `lender` is the client that opened it, whose reranker this one borrows.
         """
-        client = cls(
-            session.db_path, config=session.config, read_only=session.read_only
-        )
+        client = cls(config=session.config, read_only=session.read_only)
+        client._scope = DatabaseScope((session.ref,))
         client._session = session
         client._owns_session = False
         client._lender = lender
@@ -858,7 +857,7 @@ class HaikuRAG:
         if unknown:
             raise UnknownDatabaseError(
                 f"unknown database(s) {', '.join(sorted(set(unknown)))}; this "
-                f"client covers {', '.join(sorted(covered)) or 'a single unnamed database'}"
+                f"client covers {', '.join(sorted(covered))}"
             )
 
     async def clients_covering(
@@ -882,8 +881,8 @@ class HaikuRAG:
             return []
         if sources != [self.source]:
             raise UnknownDatabaseError(
-                f"unknown database(s) {', '.join(sources) or '(none)'}; this "
-                f"client covers {self.source or 'a single unnamed database'}"
+                f"unknown database(s) {', '.join(sources)}; this client covers "
+                f"{self.source}"
             )
         return [self]
 
@@ -905,9 +904,6 @@ class HaikuRAG:
         if not await self.clients_covering(sources):
             return []
         results = await search(self, query, limit, search_type, filter, include_images)
-        # A database named in config keeps its name even when it is the only one
-        # this client covers. Only an unnamed `lancedb.uri` database leaves
-        # source unset.
         for result in results:
             result.source = self.source
         return results

haiku_rag_slim/haiku/rag/client/rebuild.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import logging
 from collections.abc import AsyncGenerator
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING
 
 from docling_core.types.doc.document import DescriptionMetaField, PictureMeta
@@ -531,7 +531,7 @@ async def _flush_rebuild_batch(
     if not documents:
         return
 
-    now = datetime.now().isoformat()
+    now = datetime.now(UTC).isoformat()
 
     # Batch update documents and document_meta using merge_insert (one LanceDB
     # version per table). Content+blobs go to documents; mutable attributes go

haiku_rag_slim/haiku/rag/client/scope.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -8,50 +9,60 @@ from haiku.rag.store.exceptions import (
 )
 from haiku.rag.utils import locate_database
 
+DEFAULT_DATABASE_FILENAME = "haiku.rag.lancedb"
+
+
+def database_name(path: Path) -> str:
+    """The name a database at `path` answers to: the path's stem."""
+    if not path.stem:
+        raise ValueError(f"a database at {path} has no name: the path has no stem")
+    return path.stem
+
 
 @dataclass(frozen=True)
 class DatabaseRef:
-    """A resolved database location, and the configured name it answers to.
+    """A resolved database: the name it answers to, and where it is.
 
-    Exactly one of ``uri`` and ``db_path`` is set. ``name`` is the key from
-    ``lancedb.databases``, and the only identity that leaves the configuration:
-    it travels in results, citations and errors, where a location must not.
-    None where nothing names the database.
+    ``name`` is the key from ``lancedb.databases``, or the stem of a path the
+    caller gave. It is the only identity that leaves the configuration: it
+    travels in results, citations and errors, where a location must not.
+    ``location`` is a local path, or a URI. ``given`` marks a path the caller
+    gave, whose errors may name it: the caller already knows where it is.
     """
 
-    name: str | None
-    uri: str
-    db_path: Path | None
+    name: str
+    location: Path | str
+    given: bool = False
 
     def __post_init__(self) -> None:
-        if bool(self.uri) == (self.db_path is not None):
+        if not self.name.strip():
+            raise ValueError(f"a database at {self.location} has no name")
+        if isinstance(self.location, str):
+            if not self.location.strip():
+                raise ValueError(f"database {self.name!r} has no location")
+            object.__setattr__(self, "location", locate_database(self.location))
+        if self.given and not isinstance(self.location, Path):
             raise ValueError(
-                "a database is either a URI or a local path: "
-                f"got uri={self.uri!r} and db_path={self.db_path!r}"
+                f"database {self.name!r} is given as a path, and {self.location} "
+                "is a URI"
             )
 
     @classmethod
-    def at(cls, path: Path | str, *, name: str | None = None) -> "DatabaseRef":
+    def at(cls, path: Path | str) -> "DatabaseRef":
         """A database at a path the caller named, taken as given."""
-        return cls(name=name, uri="", db_path=Path(path))
+        path = Path(path)
+        return cls(name=database_name(path), location=path, given=True)
 
     @classmethod
-    def configured(cls, name: str | None, location: str) -> "DatabaseRef":
-        """A database the configuration placed, by ``lancedb.uri`` or an entry in
-        ``lancedb.databases``. A location carrying a scheme is a URI, anything
-        else a local path."""
-        uri, db_path = locate_database(location)
-        return cls(name=name, uri=uri, db_path=db_path)
+    def configured(cls, name: str, location: str | Path) -> "DatabaseRef":
+        """A database the configuration placed. A location carrying a scheme is
+        a URI, anything else a local path."""
+        return cls(name=name, location=location)
 
-    def connection(self, config: AppConfig) -> tuple[AppConfig, Path | None]:
-        """The configuration and path to open this one database with.
-
-        A copy: the caller's configuration still names whatever set it named.
-        """
-        one = config.model_copy(deep=True)
-        one.lancedb.databases = {}
-        one.lancedb.uri = self.uri
-        return one, self.db_path
+    @property
+    def db_path(self) -> Path | None:
+        """The local path, or None for a database behind a URI."""
+        return self.location if isinstance(self.location, Path) else None
 
 
 @dataclass(frozen=True)
@@ -59,10 +70,7 @@ class DatabaseScope:
     """The databases an operation covers.
 
     Resolved once, from configuration plus at most one selector, then passed
-    down. Never empty.
-
-    Nothing here reads the environment: ``HAIKU_RAG_DB`` is the capability entry
-    point's to honour.
+    down. Never empty. Nothing here reads the environment.
     """
 
     databases: tuple[DatabaseRef, ...]
@@ -71,6 +79,14 @@ class DatabaseScope:
         if not self.databases:
             raise ValueError("a scope covers at least one database")
 
+    @classmethod
+    def at(cls, path: Path | str) -> "DatabaseScope":
+        """One database at a path the caller named, whatever is configured.
+
+        The CLI's ``--db``: a human typing a path means that database.
+        """
+        return cls((DatabaseRef.at(path),))
+
     @classmethod
     def resolve(
         cls,
@@ -81,9 +97,10 @@ class DatabaseScope:
     ) -> "DatabaseScope":
         """The databases named by `config` and at most one selector.
 
-        A path names one database that nothing calls anything; a name selects one
-        of the configured set and keeps its name. With no selector the configured
-        set is covered in configuration order, a set of one included.
+        The configuration places databases: ``lancedb.databases``, or where it
+        names none, the default database under ``storage.data_dir`` as the entry
+        ``haiku.rag``. A name selects one of them. A path places a database
+        where the configuration places none, and is refused beside one it does.
         """
         if database_name is not None and database_path is not None:
             raise AmbiguousDatabaseError(
@@ -91,33 +108,37 @@ class DatabaseScope:
                 "pass one of them"
             )
 
-        declared = config.lancedb.databases
+        configured = config.lancedb.databases
 
         if database_path is not None:
-            return cls((DatabaseRef.at(database_path),))
+            if configured:
+                raise AmbiguousDatabaseError(
+                    "a database path and lancedb.databases both place the "
+                    f"database: db_path={Path(database_path)} and databases "
+                    f"name {', '.join(sorted(configured))}; pass one of them"
+                )
+            return cls.at(database_path)
+
+        declared: Mapping[str, str | Path] = configured or {
+            "haiku.rag": config.storage.data_dir / DEFAULT_DATABASE_FILENAME
+        }
 
         if database_name is not None:
             if database_name not in declared:
                 raise UnknownDatabaseError(
-                    f"unknown database {database_name!r}; lancedb.databases names "
-                    f"{', '.join(sorted(declared)) or 'nothing'}"
+                    f"unknown database {database_name!r}; the databases are "
+                    f"{', '.join(sorted(declared))}"
                 )
             return cls(
                 (DatabaseRef.configured(database_name, declared[database_name]),)
             )
 
-        if declared:
-            return cls(
-                tuple(
-                    DatabaseRef.configured(name, location)
-                    for name, location in declared.items()
-                )
+        return cls(
+            tuple(
+                DatabaseRef.configured(name, location)
+                for name, location in declared.items()
             )
-
-        if config.lancedb.uri:
-            return cls((DatabaseRef.configured(None, config.lancedb.uri),))
-
-        return cls((DatabaseRef.at(config.storage.data_dir / "haiku.rag.lancedb"),))
+        )
 
     def select(self, names: list[str]) -> "DatabaseScope":
         """The databases in this scope named by `names`, in the order given.
@@ -128,7 +149,7 @@ class DatabaseScope:
             raise ValueError(
                 "sources=[] selects no database; pass None for all of them"
             )
-        by_name = {ref.name: ref for ref in self.databases if ref.name is not None}
+        by_name = {ref.name: ref for ref in self.databases}
         missing = [name for name in names if name not in by_name]
         if missing:
             raise UnknownDatabaseError(
@@ -144,5 +165,5 @@ class DatabaseScope:
 
     @property
     def names(self) -> tuple[str, ...]:
-        """The configured names covered, in order. Empty where none is named."""
-        return tuple(ref.name for ref in self.databases if ref.name is not None)
+        """The names of the databases covered, in order."""
+        return tuple(ref.name for ref in self.databases)

haiku_rag_slim/haiku/rag/client/search.py

@@ -105,6 +105,11 @@ async def search_sources(
     fetch_limit = _fetch_limit(client, query, limit)
     query_vector = await _embed_query(selected[0], query, resolved)
     text = query if isinstance(query, str) else ""
+    # Embeddings are read only by cosine fusion: a reranker scores the union
+    # itself, and its 10x over-fetch would materialize them for nothing.
+    uses_cosine = query_vector is not None and (
+        not isinstance(query, str) or client.reranker is None
+    )
     per_source = await gather_all(
         *(
             c.chunk_repository.search(
@@ -113,12 +118,15 @@ async def search_sources(
                 search_type=resolved,
                 filter=filter,
                 query_vector=query_vector,
+                with_vectors=uses_cosine,
             )
             for c in selected
         )
     )
 
-    ranked = await _fuse(client, selected, query, per_source, limit)
+    ranked = await _fuse(
+        client, selected, query, per_source, limit, query_vector=query_vector
+    )
 
     results: list[SearchResult] = []
     for owner, chunk, score in ranked:
@@ -149,13 +157,21 @@ async def _fuse(
     query: "str | bytes | PILImage.Image",
     per_source: list[list[tuple[Chunk, float]]],
     limit: int,
+    query_vector: list[float] | None = None,
 ) -> list[tuple["HaikuRAG", Chunk, float]]:
     """One ranked list from several, keeping each candidate's owner.
 
     A configured reranker scores the union directly, which is what makes ranking
     across databases tractable: it compares query against document and does not
-    care where a candidate came from. Without one, reciprocal rank fusion over the
-    per-database rankings, since scores from separate indexes are not comparable.
+    care where a candidate came from. Without one, the union is ordered by
+    cosine similarity to the query vector: the databases in a selection share an
+    embedder, so similarity in that one space is the signal that is comparable
+    across databases by construction, where retrieval scores are each database's
+    own rank arithmetic. A search with no query vector (full-text) orders by the
+    retrieval score instead. In both, ties resolve by within-database rank — the
+    candidate nothing in its own database beat wins — and only a tie on both
+    falls to configured order. The returned score is the one the union was
+    ordered by, so downstream re-sorts (context expansion) preserve this order.
     """
     owned = [
         (client, chunk, score)
@@ -165,8 +181,9 @@ async def _fuse(
     if not owned:
         return []
 
-    # An image query has no text for a reranker to score against, and the check
-    # precedes `reranker`, which builds the reranker on first access.
+    # The reranker interface takes a text query, so an image query skips it,
+    # and the check precedes `reranker`, which builds the reranker on first
+    # access.
     if isinstance(query, str):
         reranker = federator.reranker
         if reranker is not None:
@@ -191,12 +208,38 @@ async def _fuse(
                 )
             return [(owner_of[id(chunk)], chunk, score) for chunk, score in reranked]
 
-    scored: list[tuple[float, HaikuRAG, Chunk]] = []
+    scored: list[tuple[float, float, HaikuRAG, Chunk]] = []
     for client, candidates in zip(clients, per_source, strict=True):
-        for rank, (chunk, _) in enumerate(candidates):
-            scored.append((1.0 / (_RRF_K + rank + 1), client, chunk))
-    scored.sort(key=lambda item: item[0], reverse=True)
-    return [(client, chunk, score) for score, client, chunk in scored[:limit]]
+        for rank, (chunk, score) in enumerate(candidates):
+            scored.append((1.0 / (_RRF_K + rank + 1), score, client, chunk))
+
+    embeddings = [chunk.embedding for _, _, _, chunk in scored]
+    if query_vector is not None and all(e is not None for e in embeddings):
+        similarities = _cosine_to(query_vector, embeddings)  # ty: ignore[invalid-argument-type]
+        scored = [
+            (rank_score, similarity, client, chunk)
+            for (rank_score, _, client, chunk), similarity in zip(
+                scored, similarities, strict=True
+            )
+        ]
+    scored.sort(key=lambda item: (item[1], item[0]), reverse=True)
+    return [(client, chunk, score) for _, score, client, chunk in scored[:limit]]
+
+
+def _cosine_to(query_vector: list[float], embeddings: list[list[float]]) -> list[float]:
+    """Cosine similarity of each embedding to the query vector.
+
+    A zero-norm vector has no direction, so its similarity is 0 rather than a
+    division error.
+    """
+    import numpy as np
+
+    query = np.asarray(query_vector, dtype=np.float32)
+    matrix = np.asarray(embeddings, dtype=np.float32)
+    norms = np.linalg.norm(matrix, axis=1) * np.linalg.norm(query)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        similarities = np.where(norms > 0, matrix @ query / norms, 0.0)
+    return [float(s) for s in similarities]
 
 
 # Reciprocal rank fusion's smoothing constant, the value the literature uses.
@@ -268,10 +311,9 @@ async def _rank(
 ) -> list[tuple[Chunk, float]]:
     """Order candidates and cut them to `limit`.
 
-    An image query carries no text for a reranker to score against, so its
-    candidates keep the vector ranking. Its type is checked before
-    `client.reranker`, which builds the reranker on first access and loads model
-    weights for a local one.
+    The reranker interface takes a text query, so an image query keeps the
+    vector ranking. Its type is checked before `client.reranker`, which builds
+    the reranker on first access and loads model weights for a local one.
     """
     if not isinstance(query, str):
         return candidates[:limit]

haiku_rag_slim/haiku/rag/client/session.py

@@ -43,36 +43,29 @@ async def aclose_quietly(closeable: Any, what: str) -> None:
         logger.debug("Closing the %s failed on teardown", what, exc_info=True)
 
 
-def default_db_path(config: AppConfig) -> Path:
-    """Where a database lives when its location names no path."""
-    return config.storage.data_dir / "haiku.rag.lancedb"
-
-
 class SingleDatabaseSession:
     """One database: its store, its repositories, and their lifecycle.
 
     Everything that needs a store lives here, so nothing above has to ask whether
-    it has one. ``source`` is the configured name this database answers to, or
-    None where nothing names it.
+    it has one. Built from the resolved reference: ``source`` is the name it
+    answers to, and the store receives its location.
 
-    ``db_path``, ``config``, ``read_only`` and ``source`` are readable: a client
+    ``ref``, ``config``, ``read_only`` and ``source`` are readable: a client
     borrowing this session reports them as its own.
     """
 
     def __init__(
         self,
-        db_path: Path | str,
+        ref: DatabaseRef,
         config: AppConfig,
         *,
         skip_validation: bool = False,
         create: bool = False,
         read_only: bool = False,
-        source: str | None = None,
     ) -> None:
-        self.db_path = db_path
+        self.ref = ref
         self.config = config
         self.read_only = read_only
-        self.source = source
         self._skip_validation = skip_validation
         self._create = create
         self._vacuum_tasks: set[asyncio.Task] = set()
@@ -80,19 +73,25 @@ class SingleDatabaseSession:
         self._vacuum_dirty = False
 
     @property
-    def location(self) -> Path | str:
-        """Configured URI or local path for this database.
+    def source(self) -> str:
+        return self.ref.name
 
-        Not `db_path`, which is a placeholder where a URI holds the database.
-        """
-        return self.config.lancedb.uri or self.db_path
+    @property
+    def location(self) -> Path | str:
+        """Where this database is: its path, or its URI."""
+        return self.ref.location
+
+    @property
+    def db_path(self) -> Path | None:
+        """The local path, or None for a database behind a URI."""
+        return self.ref.db_path
 
     async def open(self) -> "SingleDatabaseSession":
         """Connect, validate, and build the repositories."""
         failure: str | None = None
         try:
             self.store = Store(
-                self.db_path,
+                self.location,
                 config=self.config,
                 skip_validation=self._skip_validation,
                 create=self._create,
@@ -107,20 +106,22 @@ class SingleDatabaseSession:
                 raise
         except _NAMEABLE_FAILURES as error:
             # The message keeps its remedy and gains the database's name.
-            if self.source is None:
+            if self.ref.given:
                 raise
             raise type(error)(f"database {self.source!r}: {error}") from error
         except Exception as error:
-            # Without a name there is nothing to report in the location's place.
-            if self.source is None:
+            # A path the caller gave may be named: the caller knows it already.
+            if self.ref.given:
                 raise
-            failure = type(error).__name__
+            failure = (
+                "does not exist; create it with `haiku-rag init` or `create=True`"
+                if isinstance(error, FileNotFoundError)
+                else f"could not be opened: {type(error).__name__}"
+            )
         if failure is not None:
             # Raised outside the handler: the exception carries neither a cause
             # nor a location-bearing context.
-            raise SourceUnavailableError(
-                f"database {self.source!r} could not be opened: {failure}"
-            )
+            raise SourceUnavailableError(f"database {self.source!r} {failure}")
         self.document_repository = DocumentRepository(self.store)
         self.chunk_repository = ChunkRepository(self.store)
         self.document_item_repository = DocumentItemRepository(self.store)
@@ -266,9 +267,7 @@ class FederatedSession:
         skip_validation: bool = False,
         read_only: bool = False,
     ) -> None:
-        self._refs: dict[str, DatabaseRef] = {
-            ref.name: ref for ref in scope.databases if ref.name is not None
-        }
+        self._refs: dict[str, DatabaseRef] = {ref.name: ref for ref in scope.databases}
         self._config = config
         self._skip_validation = skip_validation
         self._read_only = read_only
@@ -309,14 +308,11 @@ class FederatedSession:
 
         Registered here because a cancelled `gather` discards its results.
         """
-        ref = self._refs[name]
-        one, db_path = ref.connection(self._config)
         self._sessions[name] = await SingleDatabaseSession(
-            db_path if db_path is not None else default_db_path(one),
-            one,
+            self._refs[name],
+            self._config,
             skip_validation=self._skip_validation,
             read_only=self._read_only,
-            source=ref.name,
         ).open()
 
     async def aclose(self) -> None:

haiku_rag_slim/haiku/rag/config/models.py

@@ -40,7 +40,7 @@ class ModelConfig(ConfigModel):
     """
 
     provider: str = "ollama"
-    name: str = "gpt-oss"
+    name: str = "qwen3.8"
     base_url: str | None = None
     api_key: str | None = None
 
@@ -102,12 +102,12 @@ class LanceDBConfig(ConfigModel):
     The cache sizes are per process, since the session is shared across
     connections.
 
-    `databases` maps a name to a location, for searching multiple at once. The
-    name is what results and citations carry, so a location never leaves the
-    configuration. Mutually exclusive with `uri`.
+    `databases` maps a name to a location, a local path or a URI, and is the one
+    way to place databases. The name is what results and citations carry, so a
+    location never leaves the configuration. Empty means the default database,
+    `haiku.rag`, under `storage.data_dir`.
     """
 
-    uri: str = ""
     api_key: str = ""
     region: str = ""
     storage_options: dict[str, str] = Field(default_factory=dict)
@@ -116,13 +116,23 @@ class LanceDBConfig(ConfigModel):
     index_cache_size_bytes: int | None = Field(default=None, ge=0)
     metadata_cache_size_bytes: int | None = Field(default=None, ge=0)
 
-    @model_validator(mode="after")
-    def _one_way_of_naming_databases(self) -> "LanceDBConfig":
-        if self.uri and self.databases:
+    @model_validator(mode="before")
+    @classmethod
+    def _uri_names_its_replacement(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "uri" in data:
+            if str(data["uri"]).strip():
+                raise ValueError(
+                    "lancedb.uri was removed; write lancedb.databases: {NAME: "
+                    f"{data['uri']!r}}} instead"
+                )
             raise ValueError(
-                "lancedb.uri and lancedb.databases are mutually exclusive: "
-                "use uri for one unnamed location, or databases for named ones"
+                "lancedb.uri was removed; remove the empty key. With no "
+                "lancedb.databases the database is haiku.rag under storage.data_dir"
             )
+        return data
+
+    @model_validator(mode="after")
+    def _every_database_is_named_and_placed(self) -> "LanceDBConfig":
         for name, location in self.databases.items():
             # A blank name is falsy, so source routing reads it as absent; a
             # blank location resolves to the working directory.
@@ -155,9 +165,10 @@ class QAConfig(ConfigModel):
     model: ModelConfig = Field(
         default_factory=lambda: ModelConfig(
             provider="ollama",
-            name="gpt-oss",
+            name="qwen3.8",
             enable_thinking=True,
             temperature=0.3,
+            vision=True,
         )
     )
     max_searches: int = Field(default=5, ge=0)
@@ -206,7 +217,8 @@ class PictureDescriptionConfig(ConfigModel):
     model: ModelConfig = Field(
         default_factory=lambda: ModelConfig(
             provider="ollama",
-            name="ministral-3",
+            name="qwen3.8",
+            enable_thinking=False,
             temperature=0.0,
         )
     )
@@ -293,7 +305,7 @@ class ProcessingConfig(ConfigModel):
     title_model: ModelConfig = Field(
         default_factory=lambda: ModelConfig(
             provider="ollama",
-            name="gpt-oss",
+            name="qwen3.8",
             enable_thinking=False,
             temperature=0.3,
             max_tokens=100,
@@ -306,6 +318,7 @@ class SearchConfig(ConfigModel):
     max_context_chars: int = Field(default=5000, gt=0)
     vector_index_metric: Literal["cosine", "l2"] = "cosine"
     vector_refine_factor: int = Field(default=30, gt=0)
+    vector_nprobes: int = Field(default=20, gt=0)
 
 
 class OllamaConfig(ConfigModel):

haiku_rag_slim/haiku/rag/context.py

@@ -32,6 +32,8 @@ In both cases:
   - Results without doc_item_refs pass through unexpanded
 """
 
+from typing import Any
+
 from haiku.rag.store.models.chunk import SearchResult
 from haiku.rag.store.models.document_item import DocumentItem
 
@@ -488,3 +490,77 @@ def expand_with_items(
         final_results.append(built)
 
     return final_results + passthrough
+
+
+def build_toc(
+    items: list["DocumentItem"],
+    chunk_index: dict[str, list[str]],
+) -> list[dict[str, Any]]:
+    """Build a nested section tree from items in position order.
+
+    Each ``section_header`` with ``heading_level > 0`` becomes a node. Nesting
+    follows the explicit levels: a header pops the stack until the top is at
+    a strictly shallower level, then becomes a child of that top (or a root).
+
+    ``item_range = [start, end_exclusive]`` indexes the position-ordered item
+    list, which is the line numbering of the sandbox's ``items.jsonl``: ``start``
+    is the header's index and ``end_exclusive`` the index of the next header
+    whose level is the same or shallower (the next sibling or ancestor that
+    ends this section), or the item count if no such header exists. Indices,
+    not positions: positions may have gaps.
+
+    ``chunk_ids`` aggregates the chunks covered by all items in the section's
+    ``item_range`` (deduped, order preserved). Pass directly to ``cite()`` to
+    ground a section-scoped answer without a corpus-wide ``search()`` call.
+
+    Items without a section_header label (or with ``heading_level == 0``) are
+    skipped. When all section_headers carry the same level the output is a
+    flat sibling list (see docling-project/docling#2121 for an upstream case
+    where every PDF section_header is emitted at level=1).
+    """
+    # Defensive: every consumer is supposed to pass items in position order,
+    # but the end_exclusive lookahead below silently miscomputes section
+    # boundaries if it's not — better to sort once than trust the caller.
+    items = sorted(items, key=lambda i: i.position)
+    header_indices = [
+        idx
+        for idx, i in enumerate(items)
+        if i.label == "section_header" and i.heading_level > 0
+    ]
+    if not header_indices:
+        return []
+
+    ends: list[int] = []
+    for n, idx in enumerate(header_indices):
+        end = len(items)
+        for later in header_indices[n + 1 :]:
+            if items[later].heading_level <= items[idx].heading_level:
+                end = later
+                break
+        ends.append(end)
+
+    roots: list[dict[str, Any]] = []
+    stack: list[tuple[int, dict[str, Any]]] = []
+    for idx, end in zip(header_indices, ends, strict=True):
+        h = items[idx]
+        seen: set[str] = set()
+        chunk_ids: list[str] = []
+        for item in items[idx:end]:
+            for cid in chunk_index.get(item.self_ref, []):
+                if cid not in seen:
+                    seen.add(cid)
+                    chunk_ids.append(cid)
+        node: dict[str, Any] = {
+            "self_ref": h.self_ref,
+            "level": h.heading_level,
+            "title": h.text,
+            "page_numbers": list(h.page_numbers),
+            "item_range": [idx, end],
+            "chunk_ids": chunk_ids,
+            "children": [],
+        }
+        while stack and stack[-1][0] >= h.heading_level:
+            stack.pop()
+        (stack[-1][1]["children"] if stack else roots).append(node)
+        stack.append((h.heading_level, node))
+    return roots

haiku_rag_slim/haiku/rag/converters/base.py

@@ -42,6 +42,20 @@ def vlm_api_headers(model: "ModelConfig") -> dict[str, str]:
     return {}
 
 
+def vlm_api_params(model: "ModelConfig", max_tokens: int) -> dict[str, object]:
+    """Request body fields docling posts alongside the picture."""
+    from haiku.rag.utils import reasoning_effort
+
+    params: dict[str, object] = {
+        "model": model.name,
+        "max_completion_tokens": max_tokens,
+    }
+    effort = reasoning_effort(model)
+    if effort is not None:
+        params["reasoning_effort"] = effort
+    return params
+
+
 class DocumentConverter(ABC):
     """Abstract base class for document converters.
 

haiku_rag_slim/haiku/rag/converters/docling_local.py

@@ -12,6 +12,7 @@ from haiku.rag.config import AppConfig
 from haiku.rag.converters.base import (
     DocumentConverter,
     vlm_api_headers,
+    vlm_api_params,
     vlm_api_url,
 )
 from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
@@ -153,10 +154,7 @@ class DoclingLocalConverter(DocumentConverter):
             pipeline_options.picture_description_options = PictureDescriptionApiOptions(
                 url=AnyUrl(vlm_api_url(self.config, pic_desc.model)),
                 headers=vlm_api_headers(pic_desc.model),
-                params=dict(
-                    model=pic_desc.model.name,
-                    max_completion_tokens=pic_desc.max_tokens,
-                ),
+                params=vlm_api_params(pic_desc.model, pic_desc.max_tokens),
                 prompt=self.config.prompts.picture_description,
                 timeout=pic_desc.timeout,
             )

haiku_rag_slim/haiku/rag/converters/docling_serve.py

@@ -9,6 +9,7 @@ from haiku.rag.config import AppConfig
 from haiku.rag.converters.base import (
     DocumentConverter,
     vlm_api_headers,
+    vlm_api_params,
     vlm_api_url,
 )
 from haiku.rag.converters.text_utils import TextFileHandler, docling_safe_name
@@ -110,10 +111,7 @@ class DoclingServeConverter(DocumentConverter):
             picture_description_api = {
                 "url": vlm_api_url(self.config, pic_desc.model),
                 "headers": vlm_api_headers(pic_desc.model),
-                "params": {
-                    "model": pic_desc.model.name,
-                    "max_completion_tokens": pic_desc.max_tokens,
-                },
+                "params": vlm_api_params(pic_desc.model, pic_desc.max_tokens),
                 "prompt": prompt,
                 "timeout": pic_desc.timeout,
             }

haiku_rag_slim/haiku/rag/doctor.py

@@ -1080,7 +1080,7 @@ async def run_provider_checks(
 
 async def run_doctor(
     config: AppConfig,
-    db_path: Path,
+    location: Path | str,
     environ: dict[str, str],
     duplicates_out: Path | None = None,
     on_progress: Callable[[str], None] | None = None,
@@ -1092,7 +1092,7 @@ async def run_doctor(
     """
     notify = on_progress or (lambda _label: None)
     notify("Inspecting tables")
-    db = await connect_lancedb(config, db_path)
+    db = await connect_lancedb(location, config)
     stats = await get_database_stats(db)
 
     results: list[CheckResult] = []
@@ -1110,7 +1110,7 @@ async def run_doctor(
         missing = [name for name in REQUIRED_TABLES if not stats[name]["exists"]]
         if not missing:
             async with Store(
-                db_path,
+                location,
                 config=config,
                 skip_validation=True,
                 read_only=True,

haiku_rag_slim/haiku/rag/ingester/api/routes/database.py

@@ -1,6 +1,5 @@
 from fastapi import APIRouter, Depends, HTTPException, status
 
-from haiku.rag.client.session import default_db_path
 from haiku.rag.ingester.api.server import APIState, get_state
 from haiku.rag.store.info import DatabaseInfo, gather_database_info
 
@@ -21,5 +20,4 @@ async def database(state: APIState = Depends(get_state)) -> DatabaseInfo:
             detail="database not configured",
         )
     [ref] = state.scope.databases
-    one, db_path = ref.connection(state.config)
-    return await gather_database_info(one, db_path or default_db_path(one))
+    return await gather_database_info(ref.location, state.config)

haiku_rag_slim/haiku/rag/ingester/app.py

@@ -4,7 +4,6 @@ import signal
 from collections.abc import Callable
 from contextlib import asynccontextmanager
 from datetime import UTC, datetime
-from pathlib import Path
 from typing import TYPE_CHECKING
 
 from pydantic import BaseModel
@@ -22,6 +21,8 @@ from haiku.rag.ingester.workers.retry import RetryPolicy
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncEngine
 
+    from haiku.rag.client.scope import DatabaseScope
+
 logger = logging.getLogger(__name__)
 
 _MANIFEST_EXTRA_KEY = "_manifest"
@@ -72,14 +73,14 @@ class IngesterApp:
     WorkerPool, and a HaikuRAG client for the worker pool to ingest through.
     """
 
-    def __init__(self, *, config: AppConfig, db_path: Path | None = None):
+    def __init__(self, *, config: AppConfig, scope: "DatabaseScope | None" = None):
+        """The ingester over the database `scope` covers, or the one the
+        configuration places when no scope is handed in."""
         from haiku.rag.client.scope import DatabaseScope
         from haiku.rag.store.exceptions import AmbiguousDatabaseError
 
         self._config = config
-        # `--db` is an explicit override; None leaves placement to the
-        # configuration.
-        self._scope = DatabaseScope.resolve(config, database_path=db_path)
+        self._scope = scope if scope is not None else DatabaseScope.resolve(config)
         if self._scope.covers_multiple:
             raise AmbiguousDatabaseError(
                 "haiku-ingester writes one database, and lancedb.databases "

haiku_rag_slim/haiku/rag/ingester/cli.py

@@ -42,6 +42,7 @@ from haiku.rag.store.exceptions import (  # noqa: E402
 )
 
 if TYPE_CHECKING:
+    from haiku.rag.client.scope import DatabaseScope
     from haiku.rag.ingester.app import BatchProgress, BatchProgressCallback
 
 _cli = typer.Typer(
@@ -218,6 +219,19 @@ def _load_manifest(path: Path) -> BatchManifest:
     return BatchManifest.model_validate(data)
 
 
+def _scope_for(db: Path | None) -> "DatabaseScope | None":
+    """`--db PATH` is the operator's explicit override: that database, whatever
+    is configured. None leaves placement to the configuration."""
+    from haiku.rag.client.scope import DatabaseScope
+
+    if db is None:
+        return None
+    try:
+        return DatabaseScope.at(db)
+    except ValueError as error:
+        raise typer.BadParameter(str(error), param_hint="--db") from error
+
+
 @_cli.command("serve")
 def serve(
     db: Path | None = typer.Option(
@@ -259,7 +273,7 @@ def serve(
         app_config.ingester.api.port = port
     if root_path is not None:
         app_config.ingester.api.root_path = root_path
-    app = IngesterApp(config=app_config, db_path=db)
+    app = IngesterApp(config=app_config, scope=_scope_for(db))
     asyncio.run(app.serve(api=not no_api))
 
 
@@ -318,7 +332,7 @@ async def _run_batch(
 ) -> None:
     from haiku.rag.ingester.app import IngesterApp
 
-    app = IngesterApp(config=app_config, db_path=db_path)
+    app = IngesterApp(config=app_config, scope=_scope_for(db_path))
     if dry_run:
         report = await app.run_batch_dry_run()
         if report.failed_sweeps:

haiku_rag_slim/haiku/rag/inspector/widgets/info_modal.py

@@ -20,13 +20,12 @@ async def database_lines(client: "HaikuRAG") -> list[str]:
     Reported through the connection the client already holds. A failure becomes
     a line of the report, and the other databases still report.
     """
-    from haiku.rag.store.engine import ConnectionMode
     from haiku.rag.store.info import get_database_stats
 
     lines: list[str] = []
     db_path = client.store.db_path
 
-    if client.store._connection_mode == ConnectionMode.LOCAL and not db_path.exists():
+    if db_path is not None and not db_path.exists():
         return ["[red]Database path does not exist.[/red]"]
 
     try:

haiku_rag_slim/haiku/rag/mcp.py

@@ -1,66 +1,168 @@
 import asyncio
+import base64
 from collections.abc import AsyncIterator
 from contextlib import AsyncExitStack, asynccontextmanager
+from importlib import metadata
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Annotated
 
 from fastmcp import FastMCP
+from fastmcp.exceptions import ToolError
+from fastmcp.tools import ToolResult
+from mcp.types import ContentBlock, ImageContent, TextContent, ToolAnnotations
+from pydantic import Field
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.config import AppConfig, get_config
+from haiku.rag.context import build_toc
+from haiku.rag.sandbox import AnalysisContext, Sandbox, recovery_hint
 from haiku.rag.store.models import Document, SearchResult
-from haiku.rag.tools.document import DocumentInfo
-from haiku.rag.utils import format_citations
+from haiku.rag.store.schema import DocumentMetaRecord
+from haiku.rag.tools.document import DocumentInfo, DocumentSection, OutlineNode
+from haiku.rag.tools.search import collect_pictures
 
 if TYPE_CHECKING:
+    from typing import Any
+
     from haiku.rag.client.scope import DatabaseScope
+    from haiku.rag.store.models.document_item import DocumentItem
+
+_FILTER_COLUMNS = ", ".join(DocumentMetaRecord.model_fields)
+
+Filter = Annotated[
+    str | None,
+    Field(
+        description=(
+            f"SQL WHERE clause over the document columns {_FILTER_COLUMNS}, "
+            "restricting which documents are used. `metadata` is a JSON string, "
+            'so match its keys with LIKE: metadata LIKE \'%"author": "Smith"%\'. '
+            "Also uri LIKE '%.pdf', title = 'Q3 report'."
+        )
+    ),
+]
+Sources = Annotated[
+    list[str] | None,
+    Field(description="Collections to use, by name. All of them by default."),
+]
 
 
-def _decode_images(images_base64: list[str] | None) -> list[bytes] | None:
-    if not images_base64:
-        return None
-    import base64
+def _read_only(title: str) -> ToolAnnotations:
+    return ToolAnnotations(title=title, read_only_hint=True, open_world_hint=False)
 
-    return [base64.b64decode(b64, validate=True) for b64 in images_base64]
+
+def _decode_image(image_base64: str) -> bytes:
+    try:
+        return base64.b64decode(image_base64, validate=True)
+    except ValueError as e:
+        # binascii.Error for characters outside the alphabet or bad padding,
+        # ValueError itself for non-ASCII input.
+        raise ToolError("Invalid base64 image") from e
+
+
+def _instructions(scope: "DatabaseScope", config: AppConfig) -> str:
+    """What the server is for, naming no tools: the client has every tool's
+    description from the listing."""
+    lines = [
+        "haiku-rag is the user's knowledge base: documents they ingested, "
+        "searchable by meaning and keyword, readable whole or section by section, "
+        "or computed across with code."
+    ]
+    lines.append(
+        "Use it whenever a question could be answered from those documents, "
+        "before answering from memory, and say when it had nothing relevant."
+    )
+    if scope.covers_multiple:
+        lines.append(
+            f"It holds several collections: {', '.join(scope.names)}. Results "
+            "name theirs in `source`; pass `sources` to use a subset."
+        )
+    if config.prompts.domain_preamble:
+        lines.append(config.prompts.domain_preamble)
+    return "\n".join(lines)
+
+
+def _search_result(results: list[SearchResult], covers_multiple: bool) -> ToolResult:
+    """Results as the in-process agents read them, plus the matched chunk's
+    metadata, then each distinct picture as an image block labelled with its
+    result. No structured content: a client given both shows the model the
+    JSON and drops the text, or shows both."""
+    total = len(results)
+    text = "\n\n".join(
+        result.format_for_agent(
+            rank=rank,
+            total=total,
+            include_collection=covers_multiple,
+            include_document_id=True,
+            include_chunk_meta=True,
+        )
+        for rank, result in enumerate(results, 1)
+    )
+    content: list[ContentBlock] = [
+        TextContent(type="text", text=text or "No results found.")
+    ]
+    pictures, _ = collect_pictures(results)
+    for source, chunk_id, self_ref, picture in pictures:
+        collection = f" in {source}" if covers_multiple and source else ""
+        content.append(
+            TextContent(
+                type="text",
+                text=f"Picture {self_ref} of search result [{chunk_id}]{collection}",
+            )
+        )
+        content.append(
+            ImageContent(
+                type="image",
+                data=base64.b64encode(picture.data).decode("ascii"),
+                mime_type="image/png",
+            )
+        )
+    return ToolResult(content=content)
+
+
+def _node(toc: "dict[str, Any]") -> OutlineNode:
+    return OutlineNode(
+        id=toc["self_ref"],
+        title=toc["title"],
+        level=toc["level"],
+        page_numbers=toc["page_numbers"],
+        children=[_node(child) for child in toc["children"]],
+    )
+
+
+def _find(toc: list["dict[str, Any]"], section_id: str) -> "dict[str, Any] | None":
+    for node in toc:
+        if node["self_ref"] == section_id:
+            return node
+        found = _find(node["children"], section_id)
+        if found is not None:
+            return found
+    return None
 
 
 def create_mcp_server(
-    db_path: Path | None = None,
-    config: AppConfig | None = None,
-    read_only: bool = False,
+    db_path: Path | None = None, config: AppConfig | None = None
 ) -> FastMCP:
-    """Create an MCP server over one database.
+    """Create an MCP server over the databases the configuration places.
 
     Args:
-        db_path: Path to the database file, or None to let `config` place it. A
-            path overrides a configured `lancedb.uri`: for a URI-backed
-            database, pass None.
+        db_path: Path to the database file, where `config` places none; or
+            None to serve the databases the configuration places. Beside
+            `lancedb.databases` a path raises `AmbiguousDatabaseError`.
         config: Configuration to use.
-        read_only: If True, write tools (add_document_*, delete_document) are not registered.
     """
     from haiku.rag.client.scope import DatabaseScope
 
     config = config if config is not None else get_config()
-    return _covering(
-        DatabaseScope.resolve(config, database_path=db_path), config, read_only
-    )
+    return _covering(DatabaseScope.resolve(config, database_path=db_path), config)
 
 
-def _covering(scope: "DatabaseScope", config: AppConfig, read_only: bool) -> FastMCP:
+def _covering(scope: "DatabaseScope", config: AppConfig) -> FastMCP:
     """An MCP server over databases someone already resolved.
 
     Internal, as ``HaikuRAG._covering`` is: the public factory takes a path and
     resolves it, which is its own job. A caller that resolved already passes the
-    scope, so the configured name survives, which results and citations carry as
-    ``source``.
+    scope, so the configured name survives, which results carry as ``source``.
     """
-    from haiku.rag.store.exceptions import AmbiguousDatabaseError
-
-    if scope.covers_multiple:
-        raise AmbiguousDatabaseError(
-            "an MCP server serves one database, and this scope covers "
-            f"{', '.join(scope.names)}; name the one to serve"
-        )
     client: HaikuRAG | None = None
     stack = AsyncExitStack()
     client_lock = asyncio.Lock()
@@ -76,7 +178,7 @@ def _covering(scope: "DatabaseScope", config: AppConfig, read_only: bool) -> Fas
         async with client_lock:
             if client is None:
                 client = await stack.enter_async_context(
-                    HaikuRAG._covering(scope, config, read_only=read_only)
+                    HaikuRAG._covering(scope, config, read_only=True)
                 )
         return client
 
@@ -95,90 +197,53 @@ def _covering(scope: "DatabaseScope", config: AppConfig, read_only: bool) -> Fas
             finally:
                 client = None
 
-    mcp = FastMCP("haiku-rag", lifespan=lifespan)
+    # Explicit: the setting is also read from the environment, and the contract
+    # is that every failure reaches the client with its message.
+    mcp = FastMCP(
+        "haiku-rag",
+        instructions=_instructions(scope, config),
+        version=metadata.version("haiku.rag-slim"),
+        lifespan=lifespan,
+        mask_error_details=False,
+    )
 
-    # Write tools - only registered when not in read-only mode
-    if not read_only:
-
-        @mcp.tool()
-        async def add_document_from_file(
-            file_path: str,
-            metadata: dict[str, Any] | None = None,
-            title: str | None = None,
-        ) -> str | None:
-            """Add a document to the RAG system from a file path."""
-            try:
-                rag = await _client()
-                result = await rag.create_document_from_source(
-                    Path(file_path), title=title, metadata=metadata or {}
-                )
-                # Handle both single document and list of documents (directories)
-                if isinstance(result, list):
-                    return result[0].id if result else None
-                return result.id
-            except Exception:
-                return None
-
-        @mcp.tool()
-        async def add_document_from_url(
-            url: str, metadata: dict[str, Any] | None = None, title: str | None = None
-        ) -> str | None:
-            """Add a document to the RAG system from a URL."""
-            try:
-                rag = await _client()
-                result = await rag.create_document_from_source(
-                    url, title=title, metadata=metadata or {}
-                )
-                # Handle both single document and list of documents
-                if isinstance(result, list):
-                    return result[0].id if result else None
-                return result.id
-            except Exception:
-                return None
-
-        @mcp.tool()
-        async def add_document_from_text(
-            content: str,
-            uri: str | None = None,
-            metadata: dict[str, Any] | None = None,
-            title: str | None = None,
-        ) -> str | None:
-            """Add a document to the RAG system from text content."""
-            try:
-                rag = await _client()
-                document = await rag.create_document(
-                    content, uri, title=title, metadata=metadata or {}
-                )
-                return document.id
-            except Exception:
-                return None
-
-        @mcp.tool()
-        async def delete_document(document_id: str) -> bool:
-            """Delete a document by its ID."""
-            try:
-                rag = await _client()
-                return await rag.delete_document(document_id)
-            except Exception:
-                return False
-
-    # Read tools - always registered
-    @mcp.tool()
+    @mcp.tool(annotations=_read_only("Search documents"))
     async def search_documents(
-        query: str, limit: int | None = None, include_images: bool = True
-    ) -> list[SearchResult]:
-        """Search the RAG system for documents using hybrid search (vector similarity + full-text search).
+        query: str,
+        limit: int | None = None,
+        include_images: bool = True,
+        filter: Filter = None,
+        sources: Sources = None,
+    ) -> ToolResult:
+        """Search the knowledge base by meaning and keyword.
 
-        When include_images is True (default) and a picture-labeled chunk is
-        in the result set, ``SearchResult.image_data`` carries base64-encoded
-        PNG bytes keyed by self_ref. Set to False to omit the bytes from the
-        response (smaller JSON payload for plain-text consumers).
+        Use this first for any question the documents might answer; it needs
+        no model and is the cheapest call. Results come best first, each with
+        its rank, `Document ID`, `Collection` when the server covers several,
+        the document title, section headings, the matched chunk's metadata
+        when it has any, and the matching passage expanded to its section;
+        pass the id and collection to the document tools. Pictures in the
+        results follow as images, each labelled with its result. Ranks, not scores,
+        are the signal: scores are not comparable across queries. If nothing
+        relevant comes back, rephrase once or narrow with `filter` before
+        concluding the material is absent.
+
+        Args:
+            query: What to look for, in natural language or keywords.
+            limit: How many results to return; the server's configured default
+                when omitted.
+            include_images: Return the pictures in the results as images.
+                False for a smaller response.
         """
-        try:
-            rag = await _client()
-            return await rag.search(query, limit=limit, include_images=include_images)
-        except Exception:
-            return []
+        rag = await _client()
+        results = await rag.search(
+            query,
+            limit=limit,
+            filter=filter,
+            include_images=include_images,
+            sources=sources,
+        )
+        return _search_result(await rag.expand_context(results), rag.covers_multiple)
 
     # Image-as-query tool, only registered when the configured embedder
     # supports image embeddings. Probed at server-build time when no Store is
@@ -188,123 +253,208 @@ def _covering(scope: "DatabaseScope", config: AppConfig, read_only: bool) -> Fas
 
     if get_embedder(config).supports_images:
 
-        @mcp.tool()
+        @mcp.tool(annotations=_read_only("Search documents by image"))
         async def search_documents_by_image(
             image_base64: str,
             limit: int | None = None,
             include_images: bool = True,
-        ) -> list[SearchResult]:
-            """Search the RAG system using an image as the query.
+            filter: Filter = None,
+            sources: Sources = None,
+        ) -> ToolResult:
+            """Search the knowledge base with an image as the query.
 
-            ``image_base64`` is a base64-encoded image (PNG/JPEG bytes). The
-            image is embedded via the configured multimodal embedder and the
-            chunks table is searched vector-only. ``include_images`` controls
-            whether picture bytes are attached to picture-labeled results.
+            Use this when the question is about a picture rather than words.
+            The image is embedded and matched against document text and
+            figures by vector similarity alone. Results have the shape of
+            `search_documents` results.
+
+            Args:
+                image_base64: The query image, PNG or JPEG bytes as base64.
+                limit: How many results to return; the server's configured
+                    default when omitted.
+                include_images: Return the pictures in the results as images.
+                    False for a smaller response.
             """
-            import base64
-
-            try:
-                raw = base64.b64decode(image_base64)
-            except Exception:
-                return []
-            try:
-                rag = await _client()
-                return await rag.search(raw, limit=limit, include_images=include_images)
-            except Exception:
-                return []
-
-    @mcp.tool()
-    async def get_document(document_id: str) -> Document | None:
-        """Get a document by its ID."""
-        try:
+            raw = _decode_image(image_base64)
             rag = await _client()
-            return await rag.get_document_by_id(document_id)
-        except Exception:
-            return None
+            results = await rag.search(
+                raw,
+                limit=limit,
+                filter=filter,
+                include_images=include_images,
+                sources=sources,
+            )
+            return _search_result(
+                await rag.expand_context(results), rag.covers_multiple
+            )
 
-    @mcp.tool()
+    @mcp.tool(annotations=_read_only("Get document"))
+    async def get_document(document_id: str, source: str | None = None) -> Document:
+        """Read one document whole, in reading order.
+
+        Use this after a search when a passage is not enough. Returns the
+        document's content, title, uri and metadata. Ids come from search
+        results and `list_documents`.
+
+        Args:
+            document_id: The document's id.
+            source: The collection holding it. Without one every collection
+                is asked.
+        """
+        rag = await _client()
+        document = await rag.get_document_by_id(document_id, source)
+        if document is None:
+            raise ToolError(f"No document with id {document_id!r}")
+        return document
+
+    async def _items_of(document_id: str, source: str | None) -> list["DocumentItem"]:
+        """A document's items in reading order, from the database holding it."""
+        rag = await _client()
+        document = await rag.get_document_by_id(document_id, source)
+        if document is None:
+            raise ToolError(f"No document with id {document_id!r}")
+        owner = await rag.reader_for(source or document.source)
+        assert owner is not None, "a stored document names its database"
+        return await owner.document_item_repository.get_all_items(document_id)
+
+    @mcp.tool(annotations=_read_only("Document outline"))
+    async def get_document_outline(
+        document_id: str, source: str | None = None
+    ) -> list[OutlineNode]:
+        """The heading tree of a document, with page numbers.
+
+        Use this on a long document to see its structure before reading, then
+        pass a node's `id` to `get_document_section`. Returns the headings
+        nested by level; an empty list means the document has no headings,
+        so read it with `get_document`.
+
+        Args:
+            document_id: The document's id.
+            source: The collection holding it. Without one every collection
+                is asked.
+        """
+        return [
+            _node(toc) for toc in build_toc(await _items_of(document_id, source), {})
+        ]
+
+    @mcp.tool(annotations=_read_only("Document section"))
+    async def get_document_section(
+        document_id: str, section_id: str, source: str | None = None
+    ) -> DocumentSection:
+        """The text of one section of a document, subsections included.
+
+        Use this to read a part of a long document instead of the whole.
+        `section_id` is a node `id` from `get_document_outline`. Returns the
+        section's heading, page numbers and text in reading order, up to the
+        next heading of the same or a higher level.
+
+        Args:
+            document_id: The document's id.
+            section_id: The `id` of a node in the document's outline.
+            source: The collection holding it. Without one every collection
+                is asked.
+        """
+        items = await _items_of(document_id, source)
+        node = _find(build_toc(items, {}), section_id)
+        if node is None:
+            raise ToolError(f"No section {section_id!r} in document {document_id!r}")
+        start, end = node["item_range"]
+        ordered = sorted(items, key=lambda item: item.position)
+        return DocumentSection(
+            id=node["self_ref"],
+            title=node["title"],
+            page_numbers=node["page_numbers"],
+            content="\n\n".join(item.text for item in ordered[start:end] if item.text),
+        )
+
+    @mcp.tool(annotations=_read_only("List documents"))
     async def list_documents(
         limit: int | None = None,
         offset: int | None = None,
-        filter: str | None = None,
+        filter: Filter = None,
     ) -> list[DocumentInfo]:
-        """List all documents with optional pagination and filtering.
+        """List what the knowledge base holds.
+
+        Use this to see which documents exist, their titles, URIs and
+        metadata, and so what a `filter` can match. Not a search: it returns
+        no passages.
 
         Args:
-            limit: Maximum number of documents to return.
-            offset: Number of documents to skip.
-            filter: Optional SQL WHERE clause to filter documents.
+            limit: How many documents to return.
+            offset: How many documents to skip, for paging.
         """
-        try:
-            rag = await _client()
-            documents = await rag.list_documents(limit, offset, filter)
+        rag = await _client()
+        documents = await rag.list_documents(limit, offset, filter)
+        return [
+            DocumentInfo(
+                id=doc.id,
+                title=doc.title or "Untitled",
+                uri=doc.uri or "",
+                created=doc.created_at.strftime("%Y-%m-%d"),
+                source=doc.source,
+                metadata=doc.metadata,
+            )
+            for doc in documents
+        ]
 
-            return [
-                DocumentInfo(
-                    id=doc.id,
-                    title=doc.title or "Untitled",
-                    uri=doc.uri or "",
-                    created=doc.created_at.strftime("%Y-%m-%d"),
-                )
-                for doc in documents
-            ]
-        except Exception:
-            return []
-
-    @mcp.tool()
-    async def ask_question(
-        question: str,
-        cite: bool = False,
-        images_base64: list[str] | None = None,
+    @mcp.tool(annotations=_read_only("Run code over the documents"))
+    async def execute_code(
+        code: str, filter: Filter = None, sources: Sources = None
     ) -> str:
-        """Ask a question using the QA agent.
+        """Run a Python program over the documents and return what it printed.
+
+        Use this when the answer is a count, an aggregate, a comparison across
+        many documents, a lookup by document or chunk metadata, or a pattern
+        over whole documents: whatever a search cannot rank. The program runs
+        in a sandboxed interpreter on the server. Each call is one program,
+        nothing carries over between calls, and `print` is the only output.
+
+        Inside the program, `/documents/{document_id}/` holds `metadata.json`
+        (id, title, uri, created_at, metadata), `content.txt` (the whole text),
+        `items.jsonl` (one item per line: self_ref, label, text, page_numbers,
+        heading_level, chunk_ids), `chunks.jsonl` (one chunk per line: chunk_id,
+        metadata) and `toc.json` (`doc_id`, `title`, `tree`; each node has
+        self_ref, level, title, page_numbers, item_range as a slice into
+        items.jsonl, chunk_ids and children; an empty tree means no headings).
+        Read files with `Path.read_text()` or `open()`; a file object cannot be
+        iterated, use `.readlines()`. `await search(query, limit=10)` returns
+        dicts with chunk_id, content, document_id, document_title, document_uri,
+        source, score, page_numbers, headings, doc_item_refs, labels,
+        picture_refs (the doc_item_refs that are pictures) and chunk_meta.
+        `await list_documents()` returns dicts with id, title, uri, created_at,
+        source and metadata. Both see the documents `filter` and `sources`
+        select. Useful modules include json, re, math, pathlib, datetime,
+        collections, itertools, functools and dataclasses; decimal and
+        statistics do not exist. No generator functions, match statements or
+        class inheritance.
+        Files are read-only, there is no network, a call has a time limit named
+        in the error when it is hit, and output past a size is truncated.
+
+        Map a title or URI to a document id with one `list_documents()` call
+        rather than reading every `metadata.json`. The files carry no `source`,
+        so over several collections group by the `source` of `list_documents()`
+        rows. For a known document's structure read its `toc.json` before
+        searching: `search()` ranks across every document. A hit's
+        `doc_item_refs` are `self_ref` values in `items.jsonl`, which places it
+        in its section. `chunk_ids` on items and `chunk_id` in `chunks.jsonl`
+        join the two files; they are not citations.
 
         Args:
-            question: The question to ask.
-            cite: Whether to include citations in the response.
-            images_base64: Base64-encoded images attached to the question
-                (requires a vision-capable QA model).
-
-        Returns:
-            The answer as a string.
+            code: The program. Use `await` on search and list_documents.
         """
+        rag = await _client()
+        sandbox = Sandbox._covering(
+            scope, config, AnalysisContext(filter=filter, sources=sources), rag=rag
+        )
         try:
-            images = _decode_images(images_base64)
-            rag = await _client()
-            answer, citations = await rag.ask(question, images=images)
-            if cite and citations:
-                answer += "\n\n" + format_citations(citations)
-            return answer
-        except Exception as e:
-            return f"Error answering question: {e!s}"
-
-    @mcp.tool()
-    async def analyze(
-        question: str,
-        filter: str | None = None,
-        images_base64: list[str] | None = None,
-    ) -> str:
-        """Answer complex questions using the analysis capability.
-
-        Use this for questions requiring computation, aggregation, or
-        structural traversal across documents. The capability can write and
-        execute Python code in a sandboxed interpreter.
-
-        Args:
-            question: The question to answer.
-            filter: Optional SQL WHERE clause to filter documents.
-            images_base64: Base64-encoded images attached to the question
-                (requires a vision-capable analysis model).
-
-        Returns:
-            The answer as a string.
-        """
-        try:
-            images = _decode_images(images_base64)
-            rag = await _client()
-            result = await rag.analyze(question, filter=filter, images=images)
-            return result.answer
-        except Exception as e:
-            return f"Error running analysis capability: {e!s}"
+            result = await sandbox.execute(code)
+        finally:
+            await sandbox.close()
+        if not result.success:
+            raise ToolError(
+                f"{result.stderr}{recovery_hint(result.stderr)}"
+                f"\n\nOutput: {result.stdout}"
+            )
+        return result.stdout or "No output."
 
     return mcp

haiku_rag_slim/haiku/rag/sandbox/__init__.py

@@ -1,10 +1,11 @@
 from haiku.rag.sandbox.dependencies import AnalysisContext
 from haiku.rag.sandbox.models import AnalysisResult
-from haiku.rag.sandbox.sandbox import Sandbox, SandboxResult
+from haiku.rag.sandbox.sandbox import Sandbox, SandboxResult, recovery_hint
 
 __all__ = [
     "AnalysisContext",
     "AnalysisResult",
     "Sandbox",
     "SandboxResult",
+    "recovery_hint",
 ]

haiku_rag_slim/haiku/rag/sandbox/sandbox.py

@@ -17,8 +17,9 @@ from pydantic_monty import (
 )
 
 from haiku.rag.config.models import AppConfig
+from haiku.rag.context import build_toc
 from haiku.rag.sandbox.dependencies import AnalysisContext
-from haiku.rag.store.models.chunk import SearchResult
+from haiku.rag.store.models.chunk import Chunk, SearchResult
 from haiku.rag.store.models.document_item import PICTURE_REF_PREFIX, DocumentItem
 from haiku.rag.utils import gather_all
 
@@ -29,6 +30,9 @@ if TYPE_CHECKING:
     from haiku.rag.client.scope import DatabaseScope
 
 
+_MAX_HOST_CALLS = 10_000_000
+
+
 @dataclass
 class SandboxResult:
     """Result of executing code in the sandbox."""
@@ -38,79 +42,19 @@ class SandboxResult:
     success: bool
 
 
-def _build_toc(
-    items: list["DocumentItem"],
-    chunk_index: dict[str, list[str]],
-) -> list[dict[str, Any]]:
-    """Build a nested section tree from items in position order.
+def recovery_hint(stderr: str) -> str:
+    """Name the workaround for sandbox limits models trip over repeatedly.
 
-    Each ``section_header`` with ``heading_level > 0`` becomes a node. Nesting
-    follows the explicit levels: a header pops the stack until the top is at
-    a strictly shallower level, then becomes a child of that top (or a root).
-
-    ``item_range = [position, end_exclusive]`` where ``end_exclusive`` is the
-    position of the next header whose level is the same or shallower (i.e.
-    the next sibling or ancestor that ends this section), or the total item
-    count if no such header exists.
-
-    ``chunk_ids`` aggregates the chunks covered by all items in the section's
-    ``item_range`` (deduped, order preserved). Pass directly to ``cite()`` to
-    ground a section-scoped answer without a corpus-wide ``search()`` call.
-
-    Items without a section_header label (or with ``heading_level == 0``) are
-    skipped. When all section_headers carry the same level the output is a
-    flat sibling list (see docling-project/docling#2121 for an upstream case
-    where every PDF section_header is emitted at level=1).
+    The instructions already say file objects are not iterable, and models write
+    ``for line in open(...)`` regardless. Carrying the fix in the error gives
+    them something to act on for the retry.
     """
-    # Defensive: every consumer is supposed to pass items in position order,
-    # but the end_exclusive lookahead below silently miscomputes section
-    # boundaries if it's not — better to sort once than trust the caller.
-    items = sorted(items, key=lambda i: i.position)
-    headers: list[DocumentItem] = [
-        i for i in items if i.label == "section_header" and i.heading_level > 0
-    ]
-    if not headers:
-        return []
-
-    total = max((i.position for i in items), default=-1) + 1
-    items_by_position: dict[int, DocumentItem] = {i.position: i for i in items}
-
-    ends: list[int] = []
-    for idx, h in enumerate(headers):
-        end = total
-        for j in range(idx + 1, len(headers)):
-            if headers[j].heading_level <= h.heading_level:
-                end = headers[j].position
-                break
-        ends.append(end)
-
-    roots: list[dict[str, Any]] = []
-    stack: list[tuple[int, dict[str, Any]]] = []
-    for h, end in zip(headers, ends, strict=True):
-        seen: set[str] = set()
-        chunk_ids: list[str] = []
-        for pos in range(h.position, end):
-            item = items_by_position.get(pos)
-            if item is None:
-                continue
-            for cid in chunk_index.get(item.self_ref, []):
-                if cid not in seen:
-                    seen.add(cid)
-                    chunk_ids.append(cid)
-        node: dict[str, Any] = {
-            "self_ref": h.self_ref,
-            "level": h.heading_level,
-            "title": h.text,
-            "page_numbers": list(h.page_numbers),
-            "item_range": [h.position, end],
-            "chunk_ids": chunk_ids,
-            "children": [],
-        }
-        while stack and stack[-1][0] >= h.heading_level:
-            stack.pop()
-        (stack[-1][1]["children"] if stack else roots).append(node)
-        stack.append((h.heading_level, node))
-    return roots
+    if "TextIOWrapper" in stderr and "not iterable" in stderr:
+        return (
+            "\n\nHint: file objects cannot be iterated here. Read lines with "
+            '.readlines() or .read().split("\\n").'
+        )
+    return ""
 
 
 class Sandbox:
@@ -120,7 +64,8 @@ class Sandbox:
     The interpreter runs in a subprocess worker checked out of an ``AsyncMonty``
     pool. External functions (search, list_documents) are called by Monty code
     using ``await`` and resolved asynchronously on the host. Documents are
-    exposed via a virtual filesystem at ``/documents/{id}/``.
+    exposed via a virtual filesystem at ``/documents/{id}/``: ``metadata.json``,
+    ``content.txt``, ``items.jsonl``, ``chunks.jsonl`` and ``toc.json``.
 
     The session persists across ``execute()`` calls within the same Sandbox
     instance — variables carry over. Call ``close()`` to return the worker to
@@ -150,6 +95,7 @@ class Sandbox:
     _doc_items: dict[str, list["DocumentItem"]]
     _doc_chunk_index: dict[str, dict[str, list[str]]]
     _items_jsonl_cache: dict[str, str]
+    _chunks_jsonl_cache: dict[str, str]
     _toc_json_cache: dict[str, str]
     _opened: "HaikuRAG | None"
     _pool: AsyncMonty | None
@@ -216,6 +162,7 @@ class Sandbox:
         self._doc_items = {}
         self._doc_chunk_index = {}
         self._items_jsonl_cache = {}
+        self._chunks_jsonl_cache = {}
         self._toc_json_cache = {}
         self._pool = None
         self._session = None
@@ -328,14 +275,43 @@ class Sandbox:
         assert self._loop is not None, (
             "VFS reads happen during execute(); the loop must be captured first."
         )
-        if self._deadline is not None and self._loop.time() > self._deadline:
+        if self._past_deadline():
             coro.close()
-            raise TimeoutError(
-                "time limit exceeded: no further document reads after "
-                f"{self._config.analysis.code_timeout}s"
-            )
+            raise self._time_limit()
         return asyncio.run_coroutine_threadsafe(coro, self._loop).result()
 
+    def _past_deadline(self) -> bool:
+        return (
+            self._deadline is not None
+            and self._loop is not None
+            and self._loop.time() > self._deadline
+        )
+
+    def _time_limit(self) -> TimeoutError:
+        return TimeoutError(
+            "time limit exceeded: no further document reads or calls after "
+            f"{self._config.analysis.code_timeout}s"
+        )
+
+    def _check_deadline(self) -> None:
+        """Refuse a host call once the call's time is up.
+
+        Monty's watchdog counts only time the worker spends computing, so every
+        host call, a file served from memory and an in-code search included,
+        checks the deadline before it runs.
+        """
+        if self._past_deadline():
+            raise self._time_limit()
+
+    def _timed(
+        self, read: Callable[["PurePosixPath"], str]
+    ) -> Callable[["PurePosixPath"], str]:
+        def call(path: "PurePosixPath") -> str:
+            self._check_deadline()
+            return read(path)
+
+        return call
+
     async def _discard_session(self) -> None:
         """Drop a session whose worker is gone.
 
@@ -371,6 +347,7 @@ class Sandbox:
         context = self._context
 
         async def search(query: str, limit: int = 10) -> list[dict[str, Any]]:
+            self._check_deadline()
             # Picture bytes are deliberately not attached to in-code search
             # results: the Monty interpreter has no PIL/base64/hashlib, so the
             # agent's Python can't do anything with them. The driving model
@@ -404,11 +381,13 @@ class Sandbox:
                         "doc_item_refs": r.doc_item_refs,
                         "labels": r.labels,
                         "picture_refs": picture_refs,
+                        "chunk_meta": r.chunk_meta,
                     }
                 )
             return out
 
         async def list_documents() -> list[dict[str, Any]]:
+            self._check_deadline()
             docs, _ = await self._documents()
             return [
                 {
@@ -417,6 +396,7 @@ class Sandbox:
                     "uri": d.uri,
                     "created_at": str(d.created_at),
                     "source": d.source,
+                    "metadata": d.metadata,
                 }
                 for d in docs
             ]
@@ -433,6 +413,7 @@ class Sandbox:
         - metadata.json: CallbackFile (eager, small)
         - content.txt: CallbackFile (lazy, can be large)
         - items.jsonl: CallbackFile (lazy, bulk-cached)
+        - chunks.jsonl: CallbackFile (lazy, bulk-cached)
         - toc.json: CallbackFile (lazy, bulk-cached)
         """
         files: list[CallbackFile] = []
@@ -507,6 +488,31 @@ class Sandbox:
 
             return read_items
 
+        def _make_chunks_reader(
+            did: str,
+        ) -> Callable[["PurePosixPath"], str]:
+            def read_chunks(_path: "PurePosixPath") -> str:
+                cached = sandbox._chunks_jsonl_cache.get(did)
+                if cached is not None:
+                    return cached
+
+                async def _fetch() -> list[Chunk]:
+                    async with sandbox._connection(sandbox._owners.get(did)) as rag:
+                        return await rag.chunk_repository.get_by_document_id(did)
+
+                chunks = sandbox._run_on_loop(_fetch())
+                jsonl = "\n".join(
+                    json.dumps(
+                        {"chunk_id": chunk.id, "metadata": chunk.metadata},
+                        ensure_ascii=False,
+                    )
+                    for chunk in chunks
+                )
+                sandbox._chunks_jsonl_cache[did] = jsonl
+                return jsonl
+
+            return read_chunks
+
         def _make_toc_reader(
             did: str,
         ) -> Callable[["PurePosixPath"], str]:
@@ -520,7 +526,7 @@ class Sandbox:
                     {
                         "doc_id": did,
                         "title": doc_titles.get(did),
-                        "tree": _build_toc(items, chunk_index),
+                        "tree": build_toc(items, chunk_index),
                     },
                     ensure_ascii=False,
                 )
@@ -541,6 +547,7 @@ class Sandbox:
                     "title": doc.title,
                     "uri": doc.uri,
                     "created_at": str(doc.created_at),
+                    "metadata": doc.metadata,
                 },
                 ensure_ascii=False,
             )
@@ -550,7 +557,7 @@ class Sandbox:
             files.append(
                 CallbackFile(
                     f"{doc_dir}/metadata.json",
-                    read=lambda _path, text=metadata: text,
+                    read=self._timed(lambda _path, text=metadata: text),
                     write=_deny_write,
                 )
             )
@@ -571,14 +578,21 @@ class Sandbox:
             files.append(
                 CallbackFile(
                     f"{doc_dir}/content.txt",
-                    read=_make_content_reader(doc_id),
+                    read=self._timed(_make_content_reader(doc_id)),
                     write=_deny_write,
                 )
             )
             files.append(
                 CallbackFile(
                     f"{doc_dir}/items.jsonl",
-                    read=_make_items_reader(doc_id),
+                    read=self._timed(_make_items_reader(doc_id)),
+                    write=_deny_write,
+                )
+            )
+            files.append(
+                CallbackFile(
+                    f"{doc_dir}/chunks.jsonl",
+                    read=self._timed(_make_chunks_reader(doc_id)),
                     write=_deny_write,
                 )
             )
@@ -589,7 +603,7 @@ class Sandbox:
                 files.append(
                     CallbackFile(
                         f"{doc_dir}/toc.json",
-                        read=_make_toc_reader(doc_id),
+                        read=self._timed(_make_toc_reader(doc_id)),
                         write=_deny_write,
                     )
                 )
@@ -601,12 +615,19 @@ class Sandbox:
 
         Monty spends ``max_duration_secs`` across the session's whole life, and
         the session is reused so variables persist between calls: the budget
-        covers the whole run. ``code_timeout`` is enforced per call elsewhere: the read
-        deadline in ``_run_on_loop`` bounds a call that reads, and the pool's
-        ``request_timeout`` bounds one that computes.
+        covers the whole run. ``code_timeout`` is enforced per call elsewhere: past
+        its deadline no further host call starts (``_check_deadline``), and the
+        pool's ``request_timeout`` bounds compute.
+
+        ``max_suspensions`` counts host callbacks per session, document reads
+        included, defaults to 1000 and cannot be disabled. The time budgets are
+        the governors here, so it is set where no program reaches it.
         """
         analysis = self._config.analysis
-        return {"max_duration_secs": analysis.code_timeout * analysis.max_executions}
+        return {
+            "max_duration_secs": analysis.code_timeout * analysis.max_executions,
+            "max_suspensions": _MAX_HOST_CALLS,
+        }
 
     async def _ensure_initialized(self) -> tuple[AsyncMontySession, OSAccess]:
         """Check out a worker session and build the VFS on first use."""

haiku_rag_slim/haiku/rag/store/engine.py

@@ -38,11 +38,12 @@ class ConnectionMode(Enum):
     OBJECT_STORAGE = "object_storage"
 
     @staticmethod
-    def from_config(config: AppConfig) -> "ConnectionMode":
-        uri = config.lancedb.uri
-        if not uri:
+    def of(location: Path | str) -> "ConnectionMode":
+        """How a location is connected to: a path is local, `db://` is LanceDB
+        Cloud, any other scheme is object storage."""
+        if isinstance(location, Path) or "://" not in location:
             return ConnectionMode.LOCAL
-        if uri.startswith("db://"):
+        if location.startswith("db://"):
             return ConnectionMode.CLOUD
         return ConnectionMode.OBJECT_STORAGE
 
@@ -72,8 +73,10 @@ def _session(config: AppConfig) -> lancedb.Session:
 
 
 async def connect_lancedb(
-    config: AppConfig, db_path: Path | None = None
+    location: Path | str, config: AppConfig
 ) -> lancedb.AsyncConnection:
+    """Connect to the database at `location`, with the connection settings
+    (credentials, storage options, caches, consistency) from `config`."""
     interval = config.lancedb.read_consistency_interval_seconds
     kwargs: dict[str, Any] = {
         "session": _session(config),
@@ -81,22 +84,19 @@ async def connect_lancedb(
             timedelta(seconds=interval) if interval is not None else None
         ),
     }
-    mode = ConnectionMode.from_config(config)
+    mode = ConnectionMode.of(location)
     if mode == ConnectionMode.CLOUD:
         return await lancedb.connect_async(
-            uri=config.lancedb.uri,
+            uri=str(location),
             api_key=config.lancedb.api_key,
             region=config.lancedb.region,
             **kwargs,
         )
-    elif mode == ConnectionMode.OBJECT_STORAGE:
+    if mode == ConnectionMode.OBJECT_STORAGE:
         if config.lancedb.storage_options:
             kwargs["storage_options"] = config.lancedb.storage_options
-        return await lancedb.connect_async(uri=config.lancedb.uri, **kwargs)
-    else:
-        if db_path is None:
-            raise ValueError("No lancedb.uri configured and no db_path provided")
-        return await lancedb.connect_async(db_path.absolute(), **kwargs)
+        return await lancedb.connect_async(uri=str(location), **kwargs)
+    return await lancedb.connect_async(Path(location).absolute(), **kwargs)
 
 
 def _stored_vector_dim(settings: dict) -> int | None:
@@ -180,14 +180,24 @@ class TagInfo:
 class Store:
     def __init__(
         self,
-        db_path: Path | str,
+        location: Path | str,
         config: AppConfig | None = None,
         skip_validation: bool = False,
         create: bool = False,
         read_only: bool = False,
         skip_migration_check: bool = False,
     ):
-        self.db_path: Path = Path(db_path)
+        """A store over the database at `location`, a local path or a URI.
+
+        `config` supplies connection settings; where the database is comes
+        from `location` alone.
+        """
+        self._location: Path | str = location
+        self.db_path: Path | None = (
+            Path(location)
+            if ConnectionMode.of(location) == ConnectionMode.LOCAL
+            else None
+        )
         self._config = config if config is not None else get_config()
         self._read_only = read_only
         self._create = create
@@ -200,7 +210,7 @@ class Store:
         self._rebuild_lock = asyncio.Lock()
         self._is_new_db = False
 
-        if self._connection_mode == ConnectionMode.LOCAL:
+        if self.db_path is not None:
             if not self.db_path.exists():
                 if not create:
                     raise FileNotFoundError(
@@ -231,7 +241,7 @@ class Store:
     async def _initialize(self):
         """Perform async initialization: connect to LanceDB, init tables, validate."""
         self.db: lancedb.AsyncConnection = await connect_lancedb(
-            self._config, self.db_path
+            self.location, self._config
         )
 
         # Read once and thread onward: on object storage each of these is a
@@ -392,9 +402,14 @@ class Store:
         needed = datetime.now() - oldest + TAG_RETENTION_MARGIN
         return max(retention, needed)
 
+    @property
+    def location(self) -> Path | str:
+        """Where this store connected: a local path, or a URI."""
+        return self._location
+
     @property
     def _connection_mode(self) -> ConnectionMode:
-        return ConnectionMode.from_config(self._config)
+        return ConnectionMode.of(self._location)
 
     async def _ensure_vector_index(self) -> None:
         """Create or rebuild vector index on chunks table.

haiku_rag_slim/haiku/rag/store/info.py

@@ -96,15 +96,15 @@ class DatabaseInfo(BaseModel):
     packages: dict[str, str] = Field(default_factory=dict)
 
 
-async def gather_database_info(config: AppConfig, db_path: Path) -> DatabaseInfo:
+async def gather_database_info(location: Path | str, config: AppConfig) -> DatabaseInfo:
     """Collect read-only database state without going through Store, so a
     database missing tables (e.g. pre-migration) still reports what it can."""
     from haiku.rag.store.upgrades import get_pending_upgrades
     from haiku.rag.utils import get_package_versions
 
-    display_path = config.lancedb.uri or str(db_path)
+    display_path = str(location)
 
-    db = await connect_lancedb(config, db_path)
+    db = await connect_lancedb(location, config)
     stats = await get_database_stats(db)
 
     if not any(entry["exists"] for entry in stats.values()):

haiku_rag_slim/haiku/rag/store/models/chunk.py

@@ -1,3 +1,4 @@
+import json
 from typing import TYPE_CHECKING, Literal
 
 from pydantic import BaseModel, PrivateAttr
@@ -143,13 +144,14 @@ class SearchResult(BaseModel):
     consumers (UIs). Never part of ``format_for_agent`` output.
 
     ``chunk_meta`` is the anchor chunk's unparsed ``Chunk.metadata`` and does not
-    include the metadata of any other chunks merged with it. Never part of
-    ``format_for_agent`` output.
+    include the metadata of any other chunks merged with it. Left out of
+    ``format_for_agent`` output unless ``include_chunk_meta`` asks for its custom
+    keys.
 
-    ``source`` names the configured database a result came from: the name from
-    ``lancedb.databases``, never a path or URI, so a location cannot travel in a
-    result, a citation or a log. It is None only where no database is named, as
-    with the single ``lancedb.uri``.
+    ``source`` names the database a result came from: the name from
+    ``lancedb.databases`` or a path's stem, never a path or URI, so a location
+    cannot travel in a result, a citation or a log. Every result a search
+    produces carries it; None only on a result built by hand.
     """
 
     content: str
@@ -202,6 +204,8 @@ class SearchResult(BaseModel):
         total: int | None = None,
         *,
         include_collection: bool = False,
+        include_document_id: bool = False,
+        include_chunk_meta: bool = False,
     ) -> str:
         """Format this search result for inclusion in agent context.
 
@@ -215,7 +219,11 @@ class SearchResult(BaseModel):
 
         `include_collection` is the caller's decision, not this result's: a
         search spanning one collection has nothing to distinguish, whether or
-        not that collection is named.
+        not that collection is named. `include_document_id` is for a reader
+        that will fetch the document by id from the text alone.
+        `include_chunk_meta` renders the metadata stored with the matched
+        chunk beyond haiku.rag's own structural keys; on an expanded result it
+        locates the hit, not the whole passage.
         """
         if rank is not None and total is not None:
             parts = [f"[{self.chunk_id}] [rank {rank} of {total}]"]
@@ -224,6 +232,9 @@ class SearchResult(BaseModel):
         else:
             parts = [f"[{self.chunk_id}] (score: {self.score:.2f})"]
 
+        if include_document_id and self.document_id:
+            parts.append(f"Document ID: {self.document_id}")
+
         if include_collection and self.source:
             parts.append(f"Collection: {self.source}")
 
@@ -242,6 +253,16 @@ class SearchResult(BaseModel):
             if primary_label:
                 parts.append(f"Type: {primary_label}")
 
+        if include_chunk_meta:
+            custom = {
+                key: value
+                for key, value in self.chunk_meta.items()
+                if key not in ChunkMetadata.model_fields
+            }
+            if custom:
+                rendered = json.dumps(custom, ensure_ascii=False, sort_keys=True)
+                parts.append(f"Matched chunk metadata: {rendered}")
+
         # Surface picture captions when present. Order matches the binary
         # attachments emitted by build_image_content_from_results, so the model
         # can correlate caption ↔ attached image by position (BinaryContent

haiku_rag_slim/haiku/rag/store/models/citation.py

@@ -24,9 +24,9 @@ class Citation(BaseModel):
     ``chunk_ids`` lists the ids of all chunks whose expansion ranges merged
     into the cited result (always includes ``chunk_id``).
 
-    ``source`` names the configured database the cited chunk came from: the name
-    from ``lancedb.databases``, never a path or URI. It is None only where no
-    database is named, as with the single ``lancedb.uri``.
+    ``source`` names the database the cited chunk came from: the name from
+    ``lancedb.databases`` or a path's stem, never a path or URI. None only on a
+    citation resolved from a hand-built result.
 
     ``doc_item_refs`` are the ``self_ref`` values of every item in the cited
     content — the exact items the model saw. Visual grounding resolves bounding

haiku_rag_slim/haiku/rag/store/models/document.py

@@ -1,8 +1,8 @@
 import json
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 from haiku.rag.store.compression import compress_docling_split, decompress_json
 
@@ -14,10 +14,10 @@ class Document(BaseModel):
     """
     Represents a document with an ID, content, and metadata.
 
-    ``source`` names the configured database a document came from: the name
-    from ``lancedb.databases``, never a path or URI. It is None where no
-    database is named, as with the single ``lancedb.uri``, and is never
-    persisted.
+    ``source`` names the database a document came from: the name from
+    ``lancedb.databases`` or a path's stem, never a path or URI. Every document
+    a database returns carries it; it is never persisted, and None only on a
+    document built by hand.
     """
 
     id: str | None = None
@@ -29,8 +29,13 @@ class Document(BaseModel):
     docling_document: bytes | None = Field(default=None, exclude=True)
     docling_pages: bytes | None = Field(default=None, exclude=True)
     docling_version: str | None = Field(default=None, exclude=True)
-    created_at: datetime = Field(default_factory=datetime.now)
-    updated_at: datetime = Field(default_factory=datetime.now)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    @field_validator("created_at", "updated_at")
+    @classmethod
+    def _to_utc(cls, value: datetime) -> datetime:
+        return value if value.tzinfo else value.astimezone(UTC)
 
     def set_docling(self, docling_doc: "DoclingDocument") -> None:
         """Serialize and store a DoclingDocument, splitting structure and pages.

haiku_rag_slim/haiku/rag/store/repositories/chunk.py

@@ -240,6 +240,7 @@ class ChunkRepository:
         search_type: SearchType = "hybrid",
         filter: str | None = None,
         query_vector: list[float] | None = None,
+        with_vectors: bool = False,
     ) -> list[tuple[Chunk, float]]:
         """Search for relevant chunks using the specified search method.
 
@@ -294,6 +295,7 @@ class ChunkRepository:
                 .column("vector")
                 .distance_type(self.store._config.search.vector_index_metric)
                 .refine_factor(self.store._config.search.vector_refine_factor)
+                .nprobes(self.store._config.search.vector_nprobes)
             )
             # An image query has no text to match, so it stays vector-only.
             if search_type != "vector" and query.strip():
@@ -304,7 +306,7 @@ class ChunkRepository:
         if chunk_filter is not None:
             results = results.where(chunk_filter)
         results = results.limit(limit)
-        return await self._process_search_results(results)
+        return await self._process_search_results(results, with_vectors=with_vectors)
 
     async def get_by_document_id(
         self,
@@ -405,7 +407,7 @@ class ChunkRepository:
         return len(df)
 
     async def _process_search_results(
-        self, query_result: "AsyncQueryBase"
+        self, query_result: "AsyncQueryBase", with_vectors: bool = False
     ) -> list[tuple[Chunk, float]]:
         """Process search results into chunks with document info and scores."""
         import pandas as pd
@@ -456,6 +458,13 @@ class ChunkRepository:
             )
             documents_map = {str(row["id"]): row for row in doc_rows}
 
+        # The query projects no columns, so the vectors are already in the
+        # frame; only the federated fusion path reads them, so materializing
+        # per-chunk lists is gated on the caller asking.
+        vectors = (
+            df["vector"].tolist() if with_vectors and "vector" in df.columns else None
+        )
+
         chunks_with_scores = []
         for i, chunk_record in enumerate(pydantic_results):
             doc = documents_map.get(chunk_record.document_id)
@@ -468,6 +477,7 @@ class ChunkRepository:
                 document_uri=doc["uri"] if doc else None,
                 document_title=doc["title"] if doc else None,
                 document_meta=json.loads(doc.get("metadata", "{}") if doc else "{}"),
+                embedding=list(vectors[i]) if vectors is not None else None,
             )
             score = scores[i] if i < len(scores) else 1.0
             chunks_with_scores.append((chunk, score))

haiku_rag_slim/haiku/rag/store/repositories/document.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import overload
 from uuid import uuid4
 
@@ -77,8 +77,12 @@ class DocumentRepository:
             docling_document=doc.docling_document,
             docling_pages=doc.docling_pages,
             docling_version=doc.docling_version,
-            created_at=datetime.fromisoformat(created) if created else datetime.now(),
-            updated_at=datetime.fromisoformat(updated) if updated else datetime.now(),
+            created_at=datetime.fromisoformat(created)
+            if created
+            else datetime.now(UTC),
+            updated_at=datetime.fromisoformat(updated)
+            if updated
+            else datetime.now(UTC),
         )
 
     def _to_documents_record(self, entity: Document, doc_id: str) -> DocumentRecord:
@@ -138,7 +142,7 @@ class DocumentRepository:
         # document_meta) would surface.
         if isinstance(entity, Document):
             doc_id = str(uuid4())
-            now = datetime.now().isoformat()
+            now = datetime.now(UTC).isoformat()
             await self.store.document_meta_table.add(
                 [self._to_meta_record(entity, doc_id, now, now)]
             )
@@ -159,7 +163,7 @@ class DocumentRepository:
         if not documents:
             return []
 
-        now = datetime.now().isoformat()
+        now = datetime.now(UTC).isoformat()
         created_at = datetime.fromisoformat(now)
         doc_records = []
         meta_records = []
@@ -272,7 +276,7 @@ class DocumentRepository:
         self.store._assert_writable()
         assert entity.id, "Document ID is required for update"
 
-        now = datetime.now().isoformat()
+        now = datetime.now(UTC).isoformat()
         entity.updated_at = datetime.fromisoformat(now)
         created = entity.created_at.isoformat() if entity.created_at else now
         record = self._to_meta_record(entity, entity.id, created, now)

haiku_rag_slim/haiku/rag/store/upgrades/v0_58_0.py

@@ -77,8 +77,11 @@ async def _apply_split_document_meta(store: Store) -> None:
         Exception
     ):  # pragma: no cover - defensive; stats() failure shouldn't block the split
         live_bytes = 0
-    free_bytes = shutil.disk_usage(store.db_path).free
-    if live_bytes and free_bytes < live_bytes:
+    # A database behind a URI has no local disk to run out of.
+    free_bytes = (
+        shutil.disk_usage(store.db_path).free if store.db_path is not None else None
+    )
+    if live_bytes and free_bytes is not None and free_bytes < live_bytes:
         logger.warning(
             "Skipping post-migration vacuum: need ~%.2f GB free to compact the "
             "documents table, have %.2f GB. Run `haiku-rag vacuum` once you have "

haiku_rag_slim/haiku/rag/tools/document.py

@@ -27,6 +27,27 @@ class DocumentInfo(BaseModel):
     title: str
     uri: str
     created: str
+    source: str | None = None
+    metadata: dict = {}
+
+
+class OutlineNode(BaseModel):
+    """A heading in a document's outline. `id` is the heading item's self_ref."""
+
+    id: str
+    title: str
+    level: int
+    page_numbers: list[int] = []
+    children: list["OutlineNode"] = []
+
+
+class DocumentSection(BaseModel):
+    """One section's text in reading order, subsections included."""
+
+    id: str
+    title: str
+    page_numbers: list[int] = []
+    content: str
 
 
 class DocumentListResponse(BaseModel):

haiku_rag_slim/haiku/rag/tools/search.py

@@ -1,5 +1,6 @@
 import base64
 from collections.abc import Callable
+from collections.abc import Set as AbstractSet
 from io import BytesIO
 
 from PIL import Image
@@ -20,6 +21,22 @@ their own picture had it removed, along with their text.
 """
 
 
+PictureKey = tuple[str | None, str | None, str]
+"""Identity of one attached picture: (source, document_id, self_ref).
+
+``self_ref`` alone collides across documents, and a copy of a document in
+another collection carries its own pictures.
+"""
+
+
+def picture_keys(result: SearchResult) -> frozenset[PictureKey]:
+    """The identity of every picture this result carries."""
+    return frozenset(
+        (result.source, result.document_id, self_ref)
+        for self_ref in (result.image_data or {})
+    )
+
+
 def decode_picture(data: bytes, self_ref: str) -> BinaryContent | None:
     """Wrap picture bytes for the wire, or return nothing if they will not decode.
 
@@ -35,31 +52,20 @@ def decode_picture(data: bytes, self_ref: str) -> BinaryContent | None:
     return BinaryContent(data=data, media_type="image/png", identifier=self_ref)
 
 
-def build_image_content_from_results(
-    results: list[SearchResult],
-    include_collection: bool = False,
-) -> list[str | BinaryContent]:
-    """Decode and validate picture bytes attached to search results, labelled.
+def collect_pictures(
+    results: list[SearchResult], exclude: AbstractSet[PictureKey] = frozenset()
+) -> tuple[list[tuple[str | None, str | None, str, BinaryContent]], set[PictureKey]]:
+    """Every distinct, decodable picture attached to ``results``, in order.
 
-    Dedup keyed on ``(source, document_id, self_ref)`` so the same picture in
-    different chunks is sent once, and a copy in another collection is its own. Pictures that fail
-    ``PIL.Image.verify()`` are skipped — the model adapter renders one
-    vision placeholder per ``BinaryContent``, so emitting one for an
-    image the server can't decode leaves the processor with an
-    off-by-one count.
-
-    Every picture is preceded by a line naming the result it belongs to.
-    ``ToolReturn.content`` reaches the model as a user-role message, so
-    retrieved pictures are otherwise indistinguishable from ones the user
-    attached, and models narrate them as part of the question: unlabelled,
-    gemma4-26b answered about a figure from an unrelated document, and with a
-    single note ahead of the batch it still called them "images in the prompt".
-    The label also names the chunk to cite for a figure, which
-    ``BinaryContent.identifier`` cannot do — it does not survive serialization
-    to the vision API.
+    Returns ``(source, chunk_id, self_ref, picture)`` per picture and the
+    ``PictureKey`` of each. Dedup keyed on ``PictureKey`` so the same picture in
+    different chunks is emitted once, and a copy in another collection is its
+    own; ``exclude`` seeds that dedup with pictures already sent. Pictures that
+    fail ``PIL.Image.verify()`` are skipped.
     """
     collected: list[tuple[str | None, str | None, str, BinaryContent]] = []
-    seen: set[tuple[str | None, str | None, str]] = set()
+    seen: set[PictureKey] = set(exclude)
+    emitted: set[PictureKey] = set()
     for result in results:
         if not result.image_data:
             continue
@@ -72,7 +78,34 @@ def build_image_content_from_results(
                 continue
             collected.append((result.source, result.chunk_id, self_ref, picture))
             seen.add(key)
+            emitted.add(key)
+    return collected, emitted
 
+
+def build_image_content_from_results(
+    results: list[SearchResult],
+    include_collection: bool = False,
+    exclude: AbstractSet[PictureKey] = frozenset(),
+) -> tuple[list[str | BinaryContent], set[PictureKey]]:
+    """Decode and validate picture bytes attached to search results, labelled.
+
+    Returns the labelled content and the ``PictureKey`` of every picture it
+    emitted, as ``collect_pictures`` decides them. An undecodable picture is
+    skipped because the model adapter renders one vision placeholder per
+    ``BinaryContent``, so emitting one for an image the server can't decode
+    leaves the processor with an off-by-one count.
+
+    Every picture is preceded by a line naming the result it belongs to.
+    ``ToolReturn.content`` reaches the model as a user-role message, so
+    retrieved pictures are otherwise indistinguishable from ones the user
+    attached, and models narrate them as part of the question: unlabelled,
+    gemma4-26b answered about a figure from an unrelated document, and with a
+    single note ahead of the batch it still called them "images in the prompt".
+    The label also names the chunk to cite for a figure, which
+    ``BinaryContent.identifier`` cannot do — it does not survive serialization
+    to the vision API.
+    """
+    collected, emitted = collect_pictures(results, exclude)
     content: list[str | BinaryContent] = []
     total = len(collected)
     for position, (source, chunk_id, self_ref, picture) in enumerate(collected, 1):
@@ -83,7 +116,7 @@ def build_image_content_from_results(
             f"Not provided by the user. {RETRIEVED_IMAGE_TAG}"
         )
         content.append(picture)
-    return content
+    return content, emitted
 
 
 def create_search_toolset(
@@ -174,7 +207,7 @@ def create_search_toolset(
         if not config.qa.model.vision:
             return text
 
-        image_content = build_image_content_from_results(
+        image_content, _ = build_image_content_from_results(
             results_list, include_collection=include_collection
         )
         if image_content:

haiku_rag_slim/haiku/rag/utils.py

@@ -4,7 +4,7 @@ import sys
 from collections.abc import Awaitable
 from importlib import metadata
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, NoReturn, cast
+from typing import TYPE_CHECKING, Any, Literal, NoReturn, cast
 
 from packaging.version import Version, parse
 
@@ -41,7 +41,7 @@ def parse_model_option(value: str) -> "ModelConfig":
     parts = value.split(":", 1)
     if len(parts) != 2 or not parts[0] or not parts[1]:
         raise ValueError(
-            f"Invalid model format '{value}'. Expected 'provider:name' (e.g. 'ollama:gpt-oss')."
+            f"Invalid model format '{value}'. Expected 'provider:name' (e.g. 'ollama:qwen3.8')."
         )
     return ModelConfig(provider=parts[0], name=parts[1])
 
@@ -182,6 +182,20 @@ _OPENAI_COMPAT_PROFILE: "OpenAIModelProfile" = {
 }
 
 
+def reasoning_effort(
+    model_config: "ModelConfig",
+) -> Literal["none", "low", "high"] | None:
+    """OpenAI `reasoning_effort` for a model config, or None when unset.
+
+    "low" is gpt-oss's floor; its template rejects "none".
+    """
+    if model_config.enable_thinking is None:
+        return None
+    if model_config.enable_thinking:
+        return "high"
+    return "low" if model_config.name == "gpt-oss" else "none"
+
+
 def get_model(
     model_config: "ModelConfig",
     app_config: "AppConfig | None" = None,
@@ -213,12 +227,9 @@ def get_model(
     if provider == "ollama":
         model_settings = None
 
-        # Apply thinking control for gpt-oss
-        if model == "gpt-oss" and model_config.enable_thinking is not None:
-            if model_config.enable_thinking is False:
-                model_settings = OpenAIChatModelSettings(openai_reasoning_effort="low")
-            else:
-                model_settings = OpenAIChatModelSettings(openai_reasoning_effort="high")
+        effort = reasoning_effort(model_config)
+        if effort is not None:
+            model_settings = OpenAIChatModelSettings(openai_reasoning_effort=effort)
 
         model_settings = apply_common_settings(
             model_settings, model_config, map_thinking=False
@@ -382,43 +393,6 @@ def _citation_label(c: "Citation") -> str:
     return c.document_title or c.document_uri
 
 
-def format_citations(citations: "list[Citation]") -> str:
-    """Format citations as plain text with preserved formatting.
-
-    Used by things like the MCP server where Rich renderables are not available.
-    Pictures referenced by the chunk are surfaced as ``[Figure: <ref>]`` markers.
-    """
-    if not citations:
-        return ""
-
-    lines = ["## Citations\n"]
-
-    for i, c in enumerate(citations):
-        idx = c.index if c.index is not None else (i + 1)
-        title = c.document_title or c.document_uri
-        header = f"[{idx}] {title}"
-
-        location_parts = []
-        pages = _citation_pages(c)
-        if pages:
-            location_parts.append(pages)
-        section = _citation_section(c)
-        if section:
-            location_parts.append(f"Section: {section}")
-
-        source = c.document_uri
-        if location_parts:
-            source += f" - {', '.join(location_parts)}"
-
-        lines.append(f"{header} {source}")
-        for ref in c.picture_refs:
-            lines.append(f"[Figure: {ref}]")
-        lines.append(c.content)
-        lines.append("")
-
-    return "\n".join(lines)
-
-
 def truncated(text: str, limit: int) -> str:
     """The first `limit` characters of `text`, with `…` appended when anything
     was dropped. A cut result is `limit` characters plus the mark."""
@@ -549,17 +523,15 @@ def raise_missing_extra(module: str, extra: str, exc: ModuleNotFoundError) -> No
     ) from exc
 
 
-def locate_database(location: str) -> tuple[str, Path | None]:
-    """Split a configured location into (uri, db_path).
+def locate_database(location: str) -> Path | str:
+    """A configured location as a URI, or as a local path.
 
-    A value with a scheme is a `lancedb.uri`; anything else is a local path.
-    `ConnectionMode` classifies a `uri` as object storage and opens it without
-    the existence check a local database gets, so a local path never travels
-    as one.
+    A value with a scheme is a URI, which `ConnectionMode` opens without the
+    existence check a local database gets; anything else is a local path.
     """
     if "://" in location:
-        return location, None
-    return "", Path(location)
+        return location
+    return Path(location)
 
 
 def get_default_data_dir() -> Path:

haiku_rag_slim/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag-slim"
 description = "Local-first agentic RAG with citations - hybrid search, reranking and multimodal retrieval over your own documents, no database server required - Minimal dependencies"
-version = "0.79.0"
+version = "0.82.1"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -40,12 +40,12 @@ dependencies = [
     "docling-core>=2.82.0,<3.0.0",
     "httpx>=0.28.1",
     "jinja2>=3.1.0",
-    "fastmcp>=3.3.0",
+    "fastmcp>=4.0.2,<5.0.0",
     "lancedb==0.37.1",
     "pathspec>=1.0.4",
     "pydantic>=2.12.5",
     "pydantic-ai-slim[openai,logfire,ag-ui]>=2.18.0,<3.0.0",
-    "pydantic-monty>=0.0.19",
+    "pydantic-monty>=0.0.23",
     "pypdfium2>=5.0",
     "python-dotenv>=1.2.2",
     "pyyaml>=6.0.3",

plugins/haiku-rag/.claude-plugin/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "haiku-rag",
+  "version": "0.82.1",
+  "description": "Search, read and analyze your haiku.rag knowledge base from Claude Code.",
+  "author": {
+    "name": "Yiorgis Gozadinos",
+    "email": "ggozadinos@gmail.com"
+  },
+  "homepage": "https://ggozad.github.io/haiku.rag/mcp/",
+  "repository": "https://github.com/ggozad/haiku.rag",
+  "license": "MIT"
+}

plugins/haiku-rag/.codex-plugin/plugin.json

@@ -0,0 +1,26 @@
+{
+  "name": "haiku-rag",
+  "version": "0.82.1",
+  "description": "Search, read and analyze your haiku.rag knowledge base from Codex.",
+  "author": {
+    "name": "Yiorgis Gozadinos",
+    "email": "ggozadinos@gmail.com",
+    "url": "https://github.com/ggozad"
+  },
+  "homepage": "https://ggozad.github.io/haiku.rag/mcp/",
+  "repository": "https://github.com/ggozad/haiku.rag",
+  "license": "MIT",
+  "keywords": ["rag", "knowledge-base", "search", "documents", "mcp"],
+  "skills": "./skills/",
+  "mcpServers": "./.mcp.json",
+  "interface": {
+    "displayName": "haiku.rag",
+    "shortDescription": "Search and analyze your haiku.rag knowledge base",
+    "longDescription": "Search, read, and compute over documents in your local haiku.rag knowledge base through MCP tools.",
+    "developerName": "Yiorgis Gozadinos",
+    "category": "Productivity",
+    "capabilities": ["Interactive", "Read"],
+    "websiteURL": "https://ggozad.github.io/haiku.rag/",
+    "defaultPrompt": "Search my haiku.rag knowledge base and cite the relevant documents."
+  }
+}

plugins/haiku-rag/.mcp.json

@@ -0,0 +1,8 @@
+{
+  "mcpServers": {
+    "haiku-rag": {
+      "command": "haiku-rag",
+      "args": ["mcp", "--stdio"]
+    }
+  }
+}

plugins/haiku-rag/skills/haiku-rag/SKILL.md

@@ -0,0 +1,77 @@
+---
+name: haiku-rag
+description: Search, read and compute over the user's haiku.rag knowledge base
+  through the haiku-rag MCP tools. Use whenever a request could be answered
+  from the user's ingested documents, when asked to find, look up, check or
+  cite something in their documents or knowledge base, or when the question is
+  about the user's own material rather than general knowledge.
+compatibility: Requires the haiku-rag MCP server to be registered in the client.
+allowed-tools:
+  - mcp__plugin_haiku-rag_haiku-rag__search_documents
+  - mcp__plugin_haiku-rag_haiku-rag__search_documents_by_image
+  - mcp__plugin_haiku-rag_haiku-rag__get_document
+  - mcp__plugin_haiku-rag_haiku-rag__get_document_outline
+  - mcp__plugin_haiku-rag_haiku-rag__get_document_section
+  - mcp__plugin_haiku-rag_haiku-rag__list_documents
+  - mcp__plugin_haiku-rag_haiku-rag__execute_code
+---
+
+# Working with the knowledge base
+
+Check the knowledge base before answering from memory whenever the question
+could be about the user's documents. Say so when it has nothing relevant.
+
+## Find
+
+`search_documents` is the first call. Results come best first with the document
+title, section headings, the matched chunk's metadata when it has any, and the
+passage in its section. Pictures in the results arrive as images: answer
+figure questions from them. `filter` restricts which documents are searched,
+`limit` how many results come back. If it misses, rephrase once or narrow with
+a filter before concluding the material is not there. When the question is
+about an image rather than words and the server offers
+`search_documents_by_image`, it takes the image as the query.
+
+## Read
+
+Every search result shows its `Document ID` (and `Collection` when there are
+several); pass them to the read tools. `get_document` returns a document's
+whole text in reading order. For a long one, `get_document_outline` gives the
+heading tree with page numbers and `get_document_section` the text of one
+section, subsections included.
+
+## Compute
+
+`execute_code` runs a Python program on the server over the same documents.
+Under `/documents/{id}/` each has `metadata.json`, `content.txt`, `items.jsonl`,
+`chunks.jsonl` and `toc.json`, and the program can `await search(query)` and
+`await list_documents()`. Write code when the answer is a count, an aggregate, a
+comparison across many documents, a lookup by document or chunk metadata, or a
+pattern over whole documents: whatever search cannot rank. Each call is one
+program and variables do not carry over, so gather, compute and `print` a
+compact result in the same program. `filter` and `sources` select the documents
+it sees. For a known document's structure read its `toc.json` first; `search()`
+ranks across every document. Map a title or URI to an id with one
+`list_documents()` call rather than reading every `metadata.json`; the files
+carry no `source`, so over several collections group by its rows. Answer and
+cite from what it printed.
+
+## Explore
+
+`list_documents` shows what is stored: titles, URIs and metadata. It is how you
+learn what a filter can match.
+
+## Filters
+
+A SQL WHERE clause over the document columns `id`, `uri`, `title`,
+`created_at`, `updated_at`, `metadata`. `metadata` is a JSON string, so match
+it with LIKE: `metadata LIKE '%"author": "Smith"%'`. Also `uri LIKE '%.pdf'`,
+`title = 'Q3 report'`.
+
+## Results and citations
+
+Rank is the signal; scores are not comparable across queries and are never
+confidence. Cite the document title or URI, the section heading and page
+numbers when present, and the matched chunk's metadata when it carries locators
+such as paragraph or footnote numbers. When results carry `source`, the server
+covers several collections: name it, and pass `sources` to search a subset.

pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag"
 description = "Local-first agentic RAG with citations - hybrid search, reranking and multimodal retrieval over your own documents, no database server required"
-version = "0.79.0"
+version = "0.82.1"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -37,7 +37,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "haiku.rag-slim[docling,voyageai,cohere,zeroentropy,tui,cross-encoder,jina]==0.79.0",
+    "haiku.rag-slim[docling,voyageai,cohere,zeroentropy,tui,cross-encoder,jina]==0.82.1",
 ]
 
 [project.urls]
@@ -52,9 +52,9 @@ haiku-rag = "haiku.rag.cli:cli"
 
 [project.optional-dependencies]
 tui = ["textual>=8.2.4"]
-s3 = ["haiku.rag-slim[s3]==0.79.0"]
-cross-encoder = ["haiku.rag-slim[cross-encoder]==0.79.0"]
-ingester = ["haiku.rag-slim[ingester]==0.79.0"]
+s3 = ["haiku.rag-slim[s3]==0.82.1"]
+cross-encoder = ["haiku.rag-slim[cross-encoder]==0.82.1"]
+ingester = ["haiku.rag-slim[ingester]==0.82.1"]
 
 [build-system]
 requires = ["hatchling"]

scripts/bump_version.py

@@ -2,7 +2,8 @@
 """
 Version bumping script for haiku.rag workspace.
 
-Updates version in all pyproject.toml files and CHANGELOG.md.
+Updates version in all pyproject.toml files, both plugin manifests, and
+CHANGELOG.md.
 """
 
 import re
@@ -54,6 +55,19 @@ def update_example_dependencies(file_path: Path, new_version: str) -> None:
     print(f"✓ Updated example dependencies in {file_path.relative_to(Path.cwd())}")
 
 
+def update_plugin_version(file_path: Path, new_version: str) -> None:
+    """Update the version in a plugin manifest."""
+    content = file_path.read_text()
+    updated = re.sub(
+        r'^(\s*"version": )"[^"]+"',
+        rf'\1"{new_version}"',
+        content,
+        flags=re.MULTILINE,
+    )
+    file_path.write_text(updated)
+    print(f"✓ Updated {file_path.relative_to(Path.cwd())}")
+
+
 def update_changelog(changelog_path: Path, new_version: str) -> None:
     """Update CHANGELOG.md with new version."""
     content = changelog_path.read_text()
@@ -122,10 +136,16 @@ def main():
         root / "app" / "backend" / "pyproject.toml",
     ]
 
+    plugin_files = [
+        root / "plugins" / "haiku-rag" / ".claude-plugin" / "plugin.json",
+        root / "plugins" / "haiku-rag" / ".codex-plugin" / "plugin.json",
+    ]
     changelog_file = root / "CHANGELOG.md"
 
     # Check all files exist
-    for file in pyproject_files + example_pyproject_files + [changelog_file]:
+    for file in (
+        pyproject_files + example_pyproject_files + plugin_files + [changelog_file]
+    ):
         if not file.exists():
             print(f"Error: {file} not found")
             sys.exit(1)
@@ -155,6 +175,9 @@ def main():
     for file in example_pyproject_files:
         update_example_dependencies(file, new_version)
 
+    for file in plugin_files:
+        update_plugin_version(file, new_version)
+
     # Update CHANGELOG.md
     update_changelog(changelog_file, new_version)
 

tests/capabilities/test_capabilities.py

@@ -98,14 +98,10 @@ def _placed(capability) -> "Path | None":
     return ref.db_path
 
 
-def test_capability_factories_resolve_environment_and_defaults(
-    temp_db_path, monkeypatch
-):
+def test_capability_factories_resolve_defaults(temp_db_path, monkeypatch):
+    """The configuration places the database; the environment plays no part."""
     config = AppConfig()
     monkeypatch.setenv("HAIKU_RAG_DB", str(temp_db_path))
-    assert _placed(create_rag(config=config)) == temp_db_path
-
-    monkeypatch.delenv("HAIKU_RAG_DB")
     assert _placed(create_rag(config=config)) == (
         config.storage.data_dir / "haiku.rag.lancedb"
     )
@@ -124,30 +120,28 @@ class TestACapabilityFollowsTheConfiguredLocation:
     """A capability nobody handed a client opens one for itself, at the
     database the configuration places."""
 
-    def _config(self, tmp_path, uri: str) -> AppConfig:
+    def _config(self, tmp_path, location: str) -> AppConfig:
         from haiku.rag.config.models import LanceDBConfig, StorageConfig
 
         return AppConfig(
-            lancedb=LanceDBConfig(uri=uri),
+            lancedb=LanceDBConfig(databases={"notes": location}),
             storage=StorageConfig(data_dir=tmp_path / "elsewhere"),
         )
 
-    def test_a_configured_uri_is_left_to_the_client(self, tmp_path):
-        """A path overrides a configured location, so the capability passes
-        None and the client resolves the configured URI."""
+    def test_a_configured_location_is_the_capability_scope(self, tmp_path):
         located = tmp_path / "notes.lancedb"
         for factory in (create_rag, create_analysis):
             [local] = factory(
                 config=self._config(tmp_path, str(located))
             ).scope.databases
-            assert local == DatabaseRef.configured(None, str(located))
+            assert local == DatabaseRef("notes", located)
 
             remote = self._config(tmp_path, "s3://bucket/one.lancedb")
             [ref] = factory(config=remote).scope.databases
-            assert ref == DatabaseRef(None, "s3://bucket/one.lancedb", None)
+            assert ref == DatabaseRef("notes", "s3://bucket/one.lancedb")
 
     @pytest.mark.asyncio
-    async def test_it_opens_the_database_the_uri_places(self, tmp_path):
+    async def test_it_opens_the_database_the_configuration_places(self, tmp_path):
         from haiku.rag.client import HaikuRAG
 
         located = tmp_path / "notes.lancedb"
@@ -162,19 +156,15 @@ class TestACapabilityFollowsTheConfiguredLocation:
         finally:
             await capability._close()
 
-    def test_an_explicit_path_still_overrides_the_configured_uri(self, tmp_path):
+    def test_a_path_beside_the_configured_placement_is_refused(self, tmp_path):
+        from haiku.rag.store.exceptions import AmbiguousDatabaseError
+
         config = self._config(tmp_path, str(tmp_path / "notes.lancedb"))
         chosen = tmp_path / "chosen.lancedb"
 
-        assert _placed(create_rag(db_path=chosen, config=config)) == chosen
-
-    def test_the_environment_still_overrides_the_configured_uri(
-        self, tmp_path, monkeypatch
-    ):
-        config = self._config(tmp_path, "s3://bucket/one.lancedb")
-        monkeypatch.setenv("HAIKU_RAG_DB", str(tmp_path / "from-env.lancedb"))
-
-        assert _placed(create_rag(config=config)) == tmp_path / "from-env.lancedb"
+        for factory in (create_rag, create_analysis):
+            with pytest.raises(AmbiguousDatabaseError, match="notes"):
+                factory(db_path=chosen, config=config)
 
 
 @pytest.mark.asyncio
@@ -206,15 +196,15 @@ def test_domain_preamble_is_added_to_capability_instructions(temp_db_path):
 
 
 def _single_database_client() -> AsyncMock:
-    """A stand-in for a client covering one unnamed database.
+    """A stand-in for a client covering one database.
 
-    `covers_multiple`, `source` and `clients_covering` answer as one unnamed
-    database does; a bare AsyncMock answers every attribute with a truthy Mock.
+    `covers_multiple`, `source` and `clients_covering` answer as one database
+    does; a bare AsyncMock answers every attribute with a truthy Mock.
     """
     client = AsyncMock()
     client.covers_multiple = False
-    client.source_names = ()
-    client.source = None
+    client.source_names = ("test",)
+    client.source = "test"
     client.clients_covering.return_value = [client]
     return client
 
@@ -412,7 +402,7 @@ async def test_a_spent_search_budget_fails_the_tool(temp_db_path):
     capability.state = RAGState()
 
     with pytest.raises(ToolFailed, match="Search limit reached"):
-        await capability._search("anything", None)
+        await capability._search("anything", None, 1)
 
 
 def _stub_client(*batches: list[SearchResult]) -> AsyncMock:
@@ -452,7 +442,7 @@ async def _labels_of_search(temp_db_path, *sources: str) -> list[str]:
     client.source_names = sources
 
     with patch.object(RAGCapability, "_ensure_rag", AsyncMock(return_value=client)):
-        returned = await capability._search("cats", None)
+        returned = await capability._search("cats", None, 1)
 
     assert isinstance(returned, ToolReturn)
     assert returned.content is not None
@@ -483,7 +473,9 @@ async def test_a_fruitless_search_says_so(temp_db_path):
     capability.state = RAGState()
     capability.borrowed_rag = _stub_client([])
 
-    assert await capability._search("nothing about this", None) == "No results found."
+    assert (
+        await capability._search("nothing about this", None, 1) == "No results found."
+    )
 
 
 @pytest.mark.asyncio
@@ -500,8 +492,8 @@ async def test_a_narrower_repeat_keeps_what_the_wider_search_returned(temp_db_pa
         [SearchResult(content="first", score=1.0, chunk_id="chunk-1")],
     )
 
-    await capability._search("Figure 3-1", 20)
-    await capability._search("Figure 3-1", None)
+    await capability._search("Figure 3-1", 20, 1)
+    await capability._search("Figure 3-1", None, 2)
 
     stored = capability.state.searches["Figure 3-1"]
     assert [result.chunk_id for result in stored] == [
@@ -525,8 +517,8 @@ async def test_two_databases_holding_one_chunk_id_both_survive(temp_db_path):
         ],
     )
 
-    await capability._search("cats", 20)
-    await capability._search("cats", None)
+    await capability._search("cats", 20, 1)
+    await capability._search("cats", None, 2)
 
     stored = capability.state.searches["cats"]
     assert [(r.source, r.chunk_id) for r in stored] == [
@@ -1105,7 +1097,7 @@ def _record(deps: Deps, namespace: str) -> CapabilityEvidenceRecord:
     return CapabilityEvidenceRecord.model_validate(deps.state[namespace]["evidence"])
 
 
-async def _stub_search(self, query: str, _limit: int | None) -> str:
+async def _stub_search(self, query: str, _limit: int | None, _run_step: int) -> str:
     """Record a result the way the real search does, so citing resolves."""
     cast(Any, self.state).searches[query] = [
         SearchResult(content="evidence", score=1.0, chunk_id="chunk-1")
@@ -1689,22 +1681,18 @@ class TestMultipleCollectionsInstructions:
             (create_rag, rag_text),
             (create_analysis, analysis_text),
         ):
-            for config in (AppConfig(), self._config(alpha="/a.lancedb")):
-                capability = factory(db_path=Path("/tmp/x.lancedb"), config=config)
-                assert capability.instruction_text == baseline()
+            one_at_a_path = factory(db_path=Path("/tmp/x.lancedb"), config=AppConfig())
+            assert one_at_a_path.instruction_text == baseline()
+            one_configured = factory(config=self._config(alpha="/a.lancedb"))
+            assert one_configured.instruction_text == baseline()
 
-    def test_an_explicit_path_opens_one_database(self):
-        """A path names one database, whatever the configuration names."""
-        from haiku.rag.capabilities.analysis import instructions as analysis_text
-        from haiku.rag.capabilities.rag import instructions as rag_text
+    def test_a_path_beside_a_configured_set_is_refused(self):
+        from haiku.rag.store.exceptions import AmbiguousDatabaseError
 
         config = self._config(alpha="/a.lancedb", beta="/b.lancedb")
-        for factory, baseline in (
-            (create_rag, rag_text),
-            (create_analysis, analysis_text),
-        ):
-            capability = factory(db_path=Path("/tmp/one.lancedb"), config=config)
-            assert capability.instruction_text == baseline()
+        for factory in (create_rag, create_analysis):
+            with pytest.raises(AmbiguousDatabaseError, match="alpha, beta"):
+                factory(db_path=Path("/tmp/one.lancedb"), config=config)
 
     def test_a_lent_client_covering_one_database_is_instructed_as_before(self):
         from haiku.rag.capabilities.analysis import instructions as analysis_text

tests/capabilities/test_citation_policy.py

@@ -29,7 +29,7 @@ class Deps:
     state: dict[str, Any] = field(default_factory=dict)
 
 
-async def stub_search(self, query: str, _limit: int | None) -> str:
+async def stub_search(self, query: str, _limit: int | None, _run_step: int) -> str:
     cast(Any, self.state).searches[query] = [
         SearchResult(content="evidence", score=1.0, chunk_id="chunk-1")
     ]

tests/capabilities/test_evidence_wire.py

@@ -437,7 +437,9 @@ REAL_PNG = base64.b64decode(
 )
 
 
-async def _search_with_a_picture(self, query: str, _limit: int | None) -> str:
+async def _search_with_a_picture(
+    self, query: str, _limit: int | None, _run_step: int
+) -> str:
     """Record a result carrying a page image, the way a real search does."""
     cast(Any, self.state).searches[query] = [
         SearchResult(
@@ -515,6 +517,89 @@ async def test_a_picture_that_will_not_decode_emits_neither_image_nor_label(
     assert texts_of(wire[-1]) == []
 
 
+def _burst_result() -> SearchResult:
+    return SearchResult(
+        content="evidence",
+        score=1.0,
+        chunk_id="chunk-1",
+        document_id="doc-1",
+        source="main",
+        doc_item_refs=["#/pictures/0"],
+        image_data={"#/pictures/0": base64.b64encode(REAL_PNG).decode()},
+    )
+
+
+async def _fanout_question_then_another(temp_db_path, cite: bool) -> list[list[Any]]:
+    """Question 1 fans out over one picture chunk; question 2 follows compacted."""
+    rag = create_rag(
+        db_path=temp_db_path, config=AppConfig(), defer_loading=False, vision=True
+    )
+    client = AsyncMock()
+    client.search.side_effect = [[_burst_result()], [_burst_result()]]
+    client.expand_context.side_effect = lambda results: results
+    client.source_names = ["main"]
+    rag.borrowed_rag = client
+    citing = (
+        [[ToolCallPart("rag_cite", {"chunk_ids": ["chunk-1"]}, "call-3")]]
+        if cite
+        else []
+    )
+    calls = iter(
+        [
+            [
+                ToolCallPart("rag_search", {"query": "figure"}, "call-1"),
+                ToolCallPart("rag_search", {"query": "the figure"}, "call-2"),
+            ],
+            *citing,
+            [TextPart("first answer")],
+            [TextPart("second answer")],
+        ]
+    )
+    wire: list[list[Any]] = []
+
+    async def model(messages, _info):
+        wire.append(list(messages))
+        return ModelResponse(parts=next(calls))
+
+    agent = Agent(
+        FunctionModel(model),
+        deps_type=Deps,
+        capabilities=[rag, create_compaction()],
+    )
+    deps = Deps()
+
+    with patch.object(
+        RAGCapability, "get_picture_bytes", AsyncMock(return_value=REAL_PNG)
+    ):
+        first = await agent.run("what does the figure show?", deps=deps)
+        await agent.run(
+            "and what else?", deps=deps, message_history=first.all_messages()
+        )
+    return wire
+
+
+@pytest.mark.asyncio
+async def test_a_burst_deduplicated_picture_survives_compaction_when_cited(
+    temp_db_path,
+):
+    """Dedup attaches the picture once in its own question; the capsule re-fetches
+    it for the next. Neither pass may leave the model without it."""
+    wire = await _fanout_question_then_another(temp_db_path, cite=True)
+
+    assert [picture.data for picture in images_of(wire[1])] == [REAL_PNG]
+    assert [picture.data for picture in images_of(wire[-1])] == [REAL_PNG]
+
+
+@pytest.mark.asyncio
+async def test_a_burst_deduplicated_picture_is_dropped_by_compaction_uncited(
+    temp_db_path,
+):
+    wire = await _fanout_question_then_another(temp_db_path, cite=False)
+
+    assert [picture.data for picture in images_of(wire[1])] == [REAL_PNG]
+    assert images_of(wire[-1]) == []
+
+
 @pytest.mark.asyncio
 async def test_the_capsule_is_built_once_per_request_and_again_for_the_next(
     temp_db_path,

tests/capabilities/test_search_units.py

@@ -0,0 +1,329 @@
+import base64
+from dataclasses import dataclass, field
+from io import BytesIO
+from typing import Any
+from unittest.mock import AsyncMock
+
+import pytest
+from PIL import Image as PILImage
+from pydantic_ai import Agent
+from pydantic_ai.messages import (
+    BinaryContent,
+    ModelResponse,
+    TextPart,
+    ToolCallPart,
+    ToolReturnPart,
+)
+from pydantic_ai.models.function import FunctionModel
+from pydantic_ai.run import AgentRunResult
+
+from haiku.rag.capabilities.ledger import CapabilityEvidenceRecord
+from haiku.rag.capabilities.rag import RAGState
+from haiku.rag.capabilities.rag import create_capability as create_rag
+from haiku.rag.config.models import AppConfig
+from haiku.rag.store.models.chunk import SearchResult
+
+
+@dataclass
+class Deps:
+    state: dict[str, Any] = field(default_factory=dict)
+
+
+def burst_model(bursts: list[list[str]]) -> FunctionModel:
+    """Emit one `rag_search` call per query in each burst, then answer."""
+    responses = 0
+
+    def model_function(_messages, _info) -> ModelResponse:
+        nonlocal responses
+        responses += 1
+        if responses <= len(bursts):
+            return ModelResponse(
+                parts=[
+                    ToolCallPart("rag_search", {"query": query})
+                    for query in bursts[responses - 1]
+                ]
+            )
+        return ModelResponse(parts=[TextPart("done")])
+
+    return FunctionModel(model_function)
+
+
+def burst_agent(
+    bursts: list[list[str]], db_path, max_searches: int
+) -> Agent[Deps, str]:
+    config = AppConfig()
+    config.qa.max_searches = max_searches
+    return Agent(
+        burst_model(bursts),
+        deps_type=Deps,
+        capabilities=[create_rag(db_path=db_path, config=config, defer_loading=False)],
+    )
+
+
+def search_returns(result: AgentRunResult[Any]) -> list[ToolReturnPart]:
+    return [
+        part
+        for message in result.all_messages()
+        for part in message.parts
+        if isinstance(part, ToolReturnPart) and part.tool_name == "rag_search"
+    ]
+
+
+def outcomes(result: AgentRunResult[Any]) -> list[str]:
+    return [
+        "failed" if part.outcome == "failed" else "ok"
+        for part in search_returns(result)
+    ]
+
+
+@pytest.mark.asyncio
+async def test_a_burst_in_one_response_consumes_one_unit(rag_db):
+    """Three searches emitted together cost one unit and run in emission order."""
+    agent = burst_agent([["ai", "machine learning", "deep learning"]], rag_db, 1)
+
+    result = await agent.run("question", deps=Deps())
+
+    assert outcomes(result) == ["ok", "ok", "ok"]
+    calls = [
+        part
+        for message in result.all_messages()
+        for part in message.parts
+        if isinstance(part, ToolCallPart) and part.tool_name == "rag_search"
+    ]
+    assert [part.tool_call_id for part in search_returns(result)] == [
+        part.tool_call_id for part in calls
+    ]
+
+
+@pytest.mark.asyncio
+async def test_sequential_searches_pay_one_unit_each(rag_db):
+    agent = burst_agent([["ai"], ["machine learning"]], rag_db, 1)
+
+    result = await agent.run("question", deps=Deps())
+
+    assert outcomes(result) == ["ok", "failed"]
+    assert "Search limit reached" in str(search_returns(result)[1].content)
+
+
+@pytest.mark.asyncio
+async def test_max_searches_zero_fails_every_sibling(rag_db):
+    agent = burst_agent([["ai", "machine learning", "deep learning"]], rag_db, 0)
+
+    result = await agent.run("question", deps=Deps())
+
+    assert outcomes(result) == ["failed", "failed", "failed"]
+
+
+@pytest.mark.asyncio
+async def test_a_rejected_round_fails_all_its_siblings(rag_db):
+    agent = burst_agent([["ai"], ["ml", "deep learning", "supervised"]], rag_db, 1)
+
+    result = await agent.run("question", deps=Deps())
+
+    assert outcomes(result) == ["ok", "failed", "failed", "failed"]
+
+
+@pytest.mark.asyncio
+async def test_a_sibling_past_the_allowance_pays_its_own_unit(rag_db):
+    burst = [["ai", "machine learning", "deep learning", "supervised learning"]]
+
+    within = await burst_agent(burst, rag_db, 2).run("question", deps=Deps())
+    over = await burst_agent(burst, rag_db, 1).run("question", deps=Deps())
+
+    assert outcomes(within) == ["ok", "ok", "ok", "ok"]
+    assert outcomes(over) == ["ok", "ok", "ok", "failed"]
+
+
+@pytest.mark.asyncio
+async def test_unit_tracking_resets_between_runs(rag_db):
+    """A second run's opening burst prices like a first run's."""
+
+    def model_function(messages, _info) -> ModelResponse:
+        if any(isinstance(part, ToolReturnPart) for part in messages[-1].parts):
+            return ModelResponse(parts=[TextPart("done")])
+        return ModelResponse(
+            parts=[
+                ToolCallPart("rag_search", {"query": query})
+                for query in ["ai", "machine learning", "deep learning"]
+            ]
+        )
+
+    config = AppConfig()
+    config.qa.max_searches = 1
+    agent = Agent(
+        FunctionModel(model_function),
+        deps_type=Deps,
+        capabilities=[create_rag(db_path=rag_db, config=config, defer_loading=False)],
+    )
+    deps = Deps()
+
+    first = await agent.run("question", deps=deps)
+    second = await agent.run("another", deps=deps, message_history=first.all_messages())
+
+    assert outcomes(first) == ["ok", "ok", "ok"]
+    assert outcomes(second)[-3:] == ["ok", "ok", "ok"]
+
+
+def _png() -> str:
+    buffer = BytesIO()
+    PILImage.new("RGB", (4, 4), "red").save(buffer, format="PNG")
+    return base64.b64encode(buffer.getvalue()).decode()
+
+
+def make_result(**overrides: Any) -> SearchResult:
+    fields: dict[str, Any] = {
+        "content": "body",
+        "score": 0.9,
+        "source": "main",
+        "chunk_id": "c1",
+        "document_id": "d1",
+        "image_data": {"#/pictures/0": _png()},
+    }
+    fields.update(overrides)
+    return SearchResult(**fields)
+
+
+def stub_client(
+    *batches: list[SearchResult], sources: list[str] | None = None
+) -> AsyncMock:
+    client = AsyncMock()
+    client.search.side_effect = list(batches)
+    client.expand_context.side_effect = lambda results: results
+    client.source_names = sources or ["main"]
+    return client
+
+
+def dedup_capability(client: AsyncMock, temp_db_path, *, vision: bool = True):
+    capability = create_rag(db_path=temp_db_path, config=AppConfig(), vision=vision)
+    capability.state = RAGState(evidence=CapabilityEvidenceRecord(question=0))
+    capability.borrowed_rag = client
+    return capability
+
+
+def images_of(returned: Any) -> list[BinaryContent]:
+    if isinstance(returned, str):
+        return []
+    return [item for item in returned.content if isinstance(item, BinaryContent)]
+
+
+def text_of(returned: Any) -> str:
+    return returned if isinstance(returned, str) else returned.return_value
+
+
+@pytest.mark.asyncio
+async def test_a_duplicate_sibling_is_elided_and_stays_citable(temp_db_path):
+    duplicate, novel = make_result(), make_result(chunk_id="c2", content="novel")
+    client = stub_client([make_result()], [duplicate, novel])
+    capability = dedup_capability(client, temp_db_path)
+
+    first = await capability._search("q", None, 1)
+    second = await capability._search("q rephrased", None, 1)
+
+    assert len(images_of(first)) == 1
+    assert images_of(second) == []
+    text = text_of(second)
+    assert "Also matched, shown above: [c1] [rank 1 of 2]" in text
+    assert "body" not in text
+    assert "[rank 2 of 2]" in text and "novel" in text
+    assert [r.chunk_id for r in capability.state.searches["q rephrased"]] == [
+        "c1",
+        "c2",
+    ]
+    assert await capability._cite(["c1"]) == "Registered 1 citation(s)."
+
+
+@pytest.mark.asyncio
+async def test_a_new_run_step_formats_shown_results_in_full(temp_db_path):
+    client = stub_client([make_result()], [make_result()])
+    capability = dedup_capability(client, temp_db_path)
+
+    await capability._search("q", None, 1)
+    second = await capability._search("q again", None, 2)
+
+    assert "body" in text_of(second)
+    assert len(images_of(second)) == 1
+
+
+@pytest.mark.asyncio
+async def test_same_chunk_id_from_another_collection_is_not_elided(temp_db_path):
+    client = stub_client(
+        [make_result(source="alpha")],
+        [make_result(source="beta")],
+        sources=["alpha", "beta"],
+    )
+    capability = dedup_capability(client, temp_db_path)
+
+    await capability._search("q", None, 1)
+    second = await capability._search("q rephrased", None, 1)
+
+    assert "body" in text_of(second)
+
+
+@pytest.mark.asyncio
+async def test_same_anchor_with_new_evidence_formats_in_full(temp_db_path):
+    shared, extra = _png(), _png()
+    client = stub_client(
+        [make_result(content="c1 with c2", image_data={"#/pictures/1": shared})],
+        [
+            make_result(
+                content="c1 with c3",
+                image_data={"#/pictures/1": shared, "#/pictures/3": extra},
+            )
+        ],
+    )
+    capability = dedup_capability(client, temp_db_path)
+
+    await capability._search("q", None, 1)
+    second = await capability._search("q rephrased", None, 1)
+
+    assert "c1 with c3" in text_of(second)
+    assert len(images_of(second)) == 1
+    labels = [item for item in second.content if isinstance(item, str)]
+    assert any("#/pictures/3" in label for label in labels)
+
+
+@pytest.mark.parametrize(
+    ("overrides", "elided"),
+    [
+        ({"score": 0.1}, True),
+        ({"content": "different"}, False),
+        ({"document_title": "Other"}, False),
+        ({"headings": ["Heading"]}, False),
+        ({"labels": ["table"]}, False),
+        ({"picture_captions": {"#/pictures/0": "A caption"}}, False),
+        ({"image_data": {"#/pictures/9": _png()}}, False),
+    ],
+)
+@pytest.mark.asyncio
+async def test_equivalence_follows_the_rendered_evidence(
+    temp_db_path, overrides: dict[str, Any], elided: bool
+):
+    """Any rendered field or picture identity defeats elision; score alone does not."""
+    client = stub_client([make_result()], [make_result(**overrides)])
+    capability = dedup_capability(client, temp_db_path)
+
+    await capability._search("q", None, 1)
+    second = await capability._search("q rephrased", None, 1)
+
+    assert ("Also matched, shown above" in text_of(second)) is elided
+
+
+@pytest.mark.asyncio
+async def test_a_failed_sibling_commits_nothing(temp_db_path):
+    client = stub_client(
+        [make_result(image_data={"#/pictures/0": "AAA"})],
+        [make_result()],
+    )
+    capability = dedup_capability(client, temp_db_path)
+    evidence_before = capability.state.evidence.model_dump()
+
+    with pytest.raises(Exception):
+        await capability._search("q", None, 1)
+
+    assert capability.state.searches == {}
+    assert capability.state.evidence.model_dump() == evidence_before
+
+    second = await capability._search("q rephrased", None, 1)
+
+    assert "body" in text_of(second)
+    assert len(images_of(second)) == 1

tests/chat/test_chat_app.py

@@ -84,17 +84,18 @@ def test_chat_capabilities_read_the_named_database(tmp_path, monkeypatch):
         from haiku.rag.chat import run_chat
 
         run_chat(scope=DatabaseScope.resolve(config, database_name="b"))
+        named_scope = chat_app.call_args.kwargs["scope"]
         [named] = chat_app.call_args.kwargs["capabilities"]
 
         run_chat(scope=DatabaseScope.resolve(config))
+        covering_scope = chat_app.call_args.kwargs["scope"]
         [covering] = chat_app.call_args.kwargs["capabilities"]
 
-    # The chat lends its own client, so this scope is the fallback: it places
-    # the named database alone.
-    [placed] = named.scope.databases
-    assert placed.db_path == tmp_path / "b.lancedb"
-    assert named.config.lancedb.databases == {}
-    assert covering.scope.names == ("a", "b")
+    # The app opens the scope it is handed and lends that client to the
+    # capabilities, which keep the configuration as the caller named it.
+    assert named_scope.names == ("b",)
+    assert covering_scope.names == ("a", "b")
+    assert set(named.config.lancedb.databases) == {"a", "b"}
     assert set(covering.config.lancedb.databases) == {"a", "b"}
 
 
@@ -158,8 +159,8 @@ def _make_mock_client():
     # Covers one database; a bare AsyncMock answers `covers_multiple` with a
     # truthy Mock.
     mock_client.covers_multiple = False
-    mock_client.source_names = ()
-    mock_client.source = None
+    mock_client.source_names = ("test",)
+    mock_client.source = "test"
     return mock_client
 
 
@@ -461,9 +462,9 @@ async def test_document_filter_updates_rag_state(temp_db_path: Path):
     ):
         async with app.run_test():
             # The selection is document ids, so a repeated title cannot widen it.
-            selected = [
-                (None, "6f1c2d4e-0000-4000-8000-000000000001"),
-                (None, "6f1c2d4e-0000-4000-8000-000000000002"),
+            selected: list[tuple[str | None, str]] = [
+                ("test", "6f1c2d4e-0000-4000-8000-000000000001"),
+                ("test", "6f1c2d4e-0000-4000-8000-000000000002"),
             ]
             app.on_document_filter_modal_filter_changed(
                 DocumentFilterModal.FilterChanged(selected)
@@ -477,7 +478,7 @@ async def test_document_filter_updates_rag_state(temp_db_path: Path):
             assert rag_state.document_filter == expected_filter
             assert rag_state.document_filter is not None
             assert "LIKE" not in rag_state.document_filter
-            # An unnamed database leaves the question unscoped by source.
+            # One database leaves the question unscoped by source.
             assert rag_state.sources is None
 
             # The state snapshot should also reflect the change
@@ -486,12 +487,15 @@ async def test_document_filter_updates_rag_state(temp_db_path: Path):
 
 @pytest.mark.asyncio
 async def test_document_filter_narrows_sources_to_the_selection(temp_db_path: Path):
-    """The filter carries ids, and `sources` restricts the question to the
-    databases the selection names."""
+    """Over a set, the filter carries ids and `sources` restricts the question
+    to the databases the selection names."""
     from haiku.rag.chat.app import RAG_STATE_NAMESPACE
     from haiku.rag.chat.widgets.document_filter_modal import DocumentFilterModal
 
     app, mock_client = _make_app_with_state(temp_db_path)
+    mock_client.covers_multiple = True
+    mock_client.source_names = ("alpha", "beta")
+    mock_client.source = None
 
     with (
         patch("haiku.rag.chat.app.HaikuRAG") as _stub_rag,
@@ -537,7 +541,7 @@ async def test_document_filter_cleared_when_empty(temp_db_path: Path):
         async with app.run_test():
             # First set a filter
             app.on_document_filter_modal_filter_changed(
-                DocumentFilterModal.FilterChanged([(None, "AI Overview")])
+                DocumentFilterModal.FilterChanged([("test", "AI Overview")])
             )
             rag_state = RAGState.model_validate(app._state[RAG_STATE_NAMESPACE])
             assert rag_state.document_filter is not None
@@ -684,6 +688,37 @@ class TestLendingTheClient:
         assert borrowed == [client] * len(app._capabilities)
         assert borrowed
 
+    @pytest.mark.asyncio
+    async def test_mounting_gives_every_capability_the_apps_scope(self, tmp_path):
+        """A capability built over the configured set covers what the chat
+        selected once mounted: the analysis sandbox is built over that scope."""
+        from haiku.rag.chat.app import ChatApp
+        from haiku.rag.client.scope import DatabaseScope
+        from haiku.rag.config.models import AppConfig, LanceDBConfig
+
+        config = AppConfig(
+            lancedb=LanceDBConfig(
+                databases={
+                    "a": str(tmp_path / "a.lancedb"),
+                    "b": str(tmp_path / "b.lancedb"),
+                }
+            )
+        )
+        selected = DatabaseScope.resolve(config, database_name="b")
+        capability = create_capability(config=config)
+        assert capability.scope.covers_multiple
+
+        client = _make_mock_client()
+        app = ChatApp(scope=selected, capabilities=[capability], read_only=True)
+        with (
+            patch("haiku.rag.chat.app.HaikuRAG") as stub_rag,
+            _covering_returns(stub_rag, client),
+        ):
+            async with app.run_test():
+                pass
+
+        assert capability.scope == selected
+
 
 class TestDocumentSelectionIdentity:
     """Two documents can share a title, within a corpus and across databases, so
@@ -785,11 +820,23 @@ class TestDocumentSelectionIdentity:
             )
         ]
 
-        ((label, source, doc_id),) = _labelled(docs)
+        ((label, source, doc_id),) = _labelled(docs, name_database=True)
         box = DocumentCheckbox(label, source, doc_id, value=False)
 
         assert str(box.label) == "Report [/red]  (alpha [/x])"
 
+    def test_one_database_is_not_named_on_its_labels(self):
+        """A single database names every document alike, so the label says
+        nothing a title does not."""
+        from haiku.rag.chat.widgets.document_filter_modal import _labelled
+        from haiku.rag.store.models.document import Document
+
+        docs = [Document(id="id-one", content="", title="Report", source="test")]
+
+        ((label, _, _),) = _labelled(docs)
+
+        assert label == "Report"
+
 
 def test_a_citation_title_that_looks_like_markup_is_text():
     from rich.text import Text
@@ -1014,17 +1061,20 @@ class TestKeepingSelectionsReachable:
         from haiku.rag.store.models.document import Document
 
         picked = [
-            Document(id=f"sel-{i:04d}", content="", title=f"Selected {i:04d}")
+            Document(
+                id=f"sel-{i:04d}", content="", title=f"Selected {i:04d}", source="test"
+            )
             for i in range(DOCUMENT_PAGE + 20)
         ]
         by_id = {d.id: d for d in picked}
         matched = [
-            Document(id=f"hit-{i}", content="", title=f"Hit {i}") for i in range(5)
+            Document(id=f"hit-{i}", content="", title=f"Hit {i}", source="test")
+            for i in range(5)
         ]
 
         client = AsyncMock()
         client.covers_multiple = False
-        client.source_names = ()
+        client.source_names = ("test",)
         client.count_documents.return_value = 5
 
         async def listing(limit=None, offset=0, filter=None):
@@ -1036,7 +1086,7 @@ class TestKeepingSelectionsReachable:
         client.list_documents.side_effect = listing
 
         modal = DocumentFilterModal(
-            client=client, selected=[(None, d.id or "") for d in picked]
+            client=client, selected=[("test", d.id or "") for d in picked]
         )
         app, _ = _make_app(temp_db_path, client)
         with (
@@ -1083,14 +1133,16 @@ class TestKeepingSelectionsReachable:
         from haiku.rag.store.models.document import Document
 
         picked = [
-            Document(id=f"sel-{i:04d}", content="", title=f"Selected {i:04d}")
+            Document(
+                id=f"sel-{i:04d}", content="", title=f"Selected {i:04d}", source="test"
+            )
             for i in range(DOCUMENT_PAGE + 1)
         ]
         by_id = {d.id: d for d in picked}
 
         client = AsyncMock()
         client.covers_multiple = False
-        client.source_names = ()
+        client.source_names = ("test",)
         client.count_documents.return_value = 0
 
         async def listing(limit=None, offset=0, filter=None):
@@ -1102,7 +1154,7 @@ class TestKeepingSelectionsReachable:
         client.list_documents.side_effect = listing
 
         modal = DocumentFilterModal(
-            client=client, selected=[(None, d.id or "") for d in picked]
+            client=client, selected=[("test", d.id or "") for d in picked]
         )
         app, _ = _make_app(temp_db_path, client)
         with (
@@ -1134,7 +1186,9 @@ class TestKeepingSelectionsReachable:
         # The row is gone from the listing, not merely unchecked.
         assert "sel-0200" not in remaining
         assert len(remaining) == DOCUMENT_PAGE
-        assert modal._selected == {(None, d.id) for d in picked} - {(None, "sel-0200")}
+        assert modal._selected == {("test", d.id) for d in picked} - {
+            ("test", "sel-0200")
+        }
         # The page it was on no longer exists, so the modal does not report it.
         assert modal._page == 0
         assert "page" not in footer
@@ -1154,10 +1208,10 @@ class TestKeepingSelectionsReachable:
 
         client = AsyncMock()
         client.covers_multiple = False
-        client.source_names = ()
+        client.source_names = ("test",)
         client.count_documents.return_value = DOCUMENT_PAGE * 2
         client.list_documents.return_value = [
-            Document(id="d1", content="", title="One")
+            Document(id="d1", content="", title="One", source="test")
         ]
 
         modal = DocumentFilterModal(client=client)
@@ -1187,7 +1241,7 @@ class TestKeepingSelectionsReachable:
 
         client = AsyncMock()
         client.covers_multiple = False
-        client.source_names = ()
+        client.source_names = ("test",)
         client.list_documents.return_value = []
         client.count_documents.return_value = 0
 
@@ -1222,10 +1276,10 @@ class TestKeepingSelectionsReachable:
 
         client = AsyncMock()
         client.covers_multiple = False
-        client.source_names = ()
+        client.source_names = ("test",)
         client.list_documents.return_value = [
-            Document(id="id-one", content="", title="Capital region"),
-            Document(id="id-two", content="", title="Nobel laureates"),
+            Document(id="id-one", content="", title="Capital region", source="test"),
+            Document(id="id-two", content="", title="Nobel laureates", source="test"),
         ]
         client.count_documents.return_value = DOCUMENT_PAGE * 2
 
@@ -1328,10 +1382,10 @@ class TestDocumentSearchFilter:
 
         client = AsyncMock()
         client.covers_multiple = False
-        client.source_names = ()
+        client.source_names = ("test",)
         client.list_documents.return_value = [
-            Document(id="id-one", content="", title="Capital region"),
-            Document(id="id-two", content="", title="Nobel laureates"),
+            Document(id="id-one", content="", title="Capital region", source="test"),
+            Document(id="id-two", content="", title="Nobel laureates", source="test"),
         ]
         client.count_documents.return_value = 2
 
@@ -1347,7 +1401,9 @@ class TestDocumentSearchFilter:
                 assert len(list(modal.query(DocumentCheckbox))) == 2
 
                 client.list_documents.return_value = [
-                    Document(id="id-two", content="", title="Nobel laureates"),
+                    Document(
+                        id="id-two", content="", title="Nobel laureates", source="test"
+                    ),
                 ]
                 client.count_documents.return_value = 1
                 await modal.on_input_submitted(Input.Submitted(Input(), "Nobel"))

tests/conftest.py

@@ -21,6 +21,12 @@ embeddings:
 """)
 os.environ["HAIKU_RAG_CONFIG_PATH"] = str(_test_config_path)
 
+# telemetry.configure() passes send_to_logfire="if-token-present" explicitly,
+# which beats LOGFIRE_SEND_TO_LOGFIRE, so no token must resolve: drop the
+# environment variable and point credentials discovery at an empty directory.
+os.environ.pop("LOGFIRE_TOKEN", None)
+os.environ["LOGFIRE_CREDENTIALS_DIR"] = tempfile.mkdtemp()
+
 import pydantic_ai.models  # noqa: E402
 import pytest  # noqa: E402
 import yaml  # noqa: E402
@@ -102,7 +108,7 @@ def temp_yaml_config(tmp_path, monkeypatch):
                 "vector_dim": 2560,
             }
         },
-        "qa": {"model": {"provider": "ollama", "name": "gpt-oss"}},
+        "qa": {"model": {"provider": "ollama", "name": "qwen3.8"}},
     }
 
     with open(config_file, "w") as f:

tests/ingester/test_api.py

@@ -838,11 +838,11 @@ async def test_database_503_when_no_database_is_configured(state):
 
 @pytest.mark.asyncio
 async def test_the_report_follows_a_configured_uri(tmp_path, jobs, sync):
-    """A configured `lancedb.uri` places the database, so the report opens that
+    """A configured location places the database, so the report opens that
     and not the local default."""
     db_path = tmp_path / "configured.lancedb"
     await _seed_lancedb(db_path)
-    config = AppConfig(lancedb=LanceDBConfig(uri=str(db_path)))
+    config = AppConfig(lancedb=LanceDBConfig(databases={"configured": str(db_path)}))
     state = APIState(
         config=config,
         job_repo=jobs,

tests/ingester/test_cli.py

@@ -504,31 +504,58 @@ def test_cli_entry_point_exits_on_store_state_errors(monkeypatch, capsys, error)
 
 class TestPlacingTheIngesterDatabase:
     """The ingester writes wherever the configuration places the database, and
-    resolves that once. A path is an explicit override of a configured
-    `lancedb.uri`, so no local default stands in for one."""
+    resolves that once. `--db PATH` is the operator's explicit override and
+    constructs the scope directly."""
 
     @staticmethod
     def _app(config: AppConfig, db_path=None):
         from haiku.rag.ingester.app import IngesterApp
+        from haiku.rag.ingester.cli import _scope_for
 
-        return IngesterApp(config=config, db_path=db_path)
+        return IngesterApp(config=config, scope=_scope_for(db_path))
 
-    def test_a_configured_uri_becomes_the_scope(self, tmp_path):
-        config = AppConfig(lancedb=LanceDBConfig(uri="s3://bucket/prod.lancedb"))
+    def test_a_configured_location_becomes_the_scope(self, tmp_path):
+        config = AppConfig(
+            lancedb=LanceDBConfig(databases={"prod": "s3://bucket/prod.lancedb"})
+        )
 
         [ref] = self._app(config)._scope.databases
 
-        assert ref.uri == "s3://bucket/prod.lancedb"
+        assert ref.name == "prod"
+        assert ref.location == "s3://bucket/prod.lancedb"
         assert ref.db_path is None
 
-    def test_an_override_names_the_database(self, tmp_path):
-        config = AppConfig(lancedb=LanceDBConfig(uri="s3://bucket/prod.lancedb"))
+    def test_an_override_names_the_database_by_its_stem(self, tmp_path):
+        config = AppConfig(
+            lancedb=LanceDBConfig(databases={"prod": "s3://bucket/prod.lancedb"})
+        )
         override = tmp_path / "local.lancedb"
 
         [ref] = self._app(config, override)._scope.databases
 
-        assert ref.db_path == override
-        assert ref.uri == ""
+        assert ref.name == "local"
+        assert ref.location == override
+
+    def test_the_override_is_the_cli_s_alone(self, tmp_path):
+        """`IngesterApp` takes a resolved scope, so a Python caller has no path
+        to slip past the configuration; only the CLI constructs one."""
+        from haiku.rag.client.scope import DatabaseScope
+        from haiku.rag.ingester.cli import _scope_for
+
+        assert _scope_for(None) is None
+        assert _scope_for(tmp_path / "local.lancedb") == DatabaseScope.at(
+            tmp_path / "local.lancedb"
+        )
+
+    def test_a_path_without_a_stem_is_a_usage_error(self):
+        from pathlib import Path
+
+        import typer
+
+        from haiku.rag.ingester.cli import _scope_for
+
+        with pytest.raises(typer.BadParameter, match="no name"):
+            _scope_for(Path("/"))
 
     def test_one_configured_database_is_accepted(self, tmp_path):
         """A one-entry mapping names which database to write."""
@@ -567,6 +594,10 @@ class TestPlacingTheIngesterDatabase:
             f"    a: {tmp_path / 'a.lancedb'}\n"
             f"    b: {tmp_path / 'b.lancedb'}\n"
         )
+        import haiku.rag.config as config_module
+
+        # The CLI caches the loaded configuration process-wide.
+        monkeypatch.setattr(config_module, "_config", None)
         monkeypatch.setattr(sys, "argv", ["haiku-ingester", "run-batch"])
         monkeypatch.setenv("HAIKU_RAG_CONFIG_PATH", str(config_file))
 

tests/ingester/test_run_batch.py

@@ -14,6 +14,7 @@ import pytest
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.client.exceptions import UnsupportedSourceError
+from haiku.rag.client.scope import DatabaseScope
 from haiku.rag.config import (
     APIConfig,
     AppConfig,
@@ -121,7 +122,7 @@ async def test_run_batch_drains_upserts(tmp_path, use_client):
     use_client(client)
 
     report = await IngesterApp(
-        config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch()
 
     assert report.succeeded == 2
@@ -146,7 +147,7 @@ async def test_run_batch_reports_progress(tmp_path, use_client):
     progress = []
 
     report = await IngesterApp(
-        config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch(progress_callback=progress.append)
 
     assert report.succeeded == 2
@@ -171,7 +172,9 @@ async def test_run_batch_prunes_orphans(tmp_path, use_client):
     db_path = tmp_path / "db.lancedb"
 
     # First batch ingests both files and records sync_state for each.
-    first = await IngesterApp(config=config, db_path=db_path).run_batch()
+    first = await IngesterApp(
+        config=config, scope=DatabaseScope.at(db_path)
+    ).run_batch()
     assert first.succeeded == 2
     client.delete_document.assert_not_awaited()
 
@@ -179,7 +182,9 @@ async def test_run_batch_prunes_orphans(tmp_path, use_client):
     # sync_state but not on disk -> enqueues a DELETE for it.
     (tmp_path / "b.md").unlink()
 
-    second = await IngesterApp(config=config, db_path=db_path).run_batch()
+    second = await IngesterApp(
+        config=config, scope=DatabaseScope.at(db_path)
+    ).run_batch()
 
     # a.md is unchanged (same mtime) so it's not re-ingested; only the orphan
     # delete runs.
@@ -198,7 +203,7 @@ async def test_run_batch_reports_dead_on_permanent_failure(tmp_path, use_client)
     use_client(client)
 
     report = await IngesterApp(
-        config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch()
 
     assert report.succeeded == 0
@@ -217,7 +222,9 @@ async def test_run_batch_recovered_doc_is_not_counted_as_dead(tmp_path, use_clie
     failing = _mock_client()
     failing.create_document_from_source.side_effect = UnsupportedSourceError("nope")
     use_client(failing)
-    first = await IngesterApp(config=config, db_path=db_path).run_batch()
+    first = await IngesterApp(
+        config=config, scope=DatabaseScope.at(db_path)
+    ).run_batch()
     assert first.dead == 1
 
     healthy = _mock_client()
@@ -226,7 +233,9 @@ async def test_run_batch_recovered_doc_is_not_counted_as_dead(tmp_path, use_clie
     # records the revision in sync_state, so a plain re-run no longer retries an
     # unchanged file — recovery needs the content (mtime) to change.
     (tmp_path / "a.md").write_text("hello again")
-    second = await IngesterApp(config=config, db_path=db_path).run_batch()
+    second = await IngesterApp(
+        config=config, scope=DatabaseScope.at(db_path)
+    ).run_batch()
     assert second.dead == 0
     assert second.succeeded == 1
 
@@ -248,7 +257,7 @@ async def test_run_batch_reports_failed_sweep(
 
     with caplog.at_level("ERROR", logger="haiku.rag.ingester.pollers.base"):
         report = await IngesterApp(
-            config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+            config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
         ).run_batch()
 
     assert report.failed_sweeps == ["local"]
@@ -264,7 +273,7 @@ async def test_run_batch_empty_source_returns_immediately(tmp_path, use_client):
 
     report = await asyncio.wait_for(
         IngesterApp(
-            config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+            config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
         ).run_batch(),
         timeout=5.0,
     )
@@ -287,7 +296,9 @@ async def test_run_batch_dry_run_reports_manifest_without_mutating_queue(tmp_pat
     finally:
         await engine.dispose()
 
-    report = await IngesterApp(config=config, db_path=db_path).run_batch_dry_run()
+    report = await IngesterApp(
+        config=config, scope=DatabaseScope.at(db_path)
+    ).run_batch_dry_run()
 
     assert report.failed_sweeps == []
     assert report.manifest.version == 1
@@ -322,7 +333,7 @@ async def test_run_batch_from_manifest_drains_changes_without_sweeping(
     monkeypatch.setattr(PollerManager, "sweep_all", sweep_all)
 
     report = await IngesterApp(
-        config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch_from_manifest(
         _manifest(
             BatchChange(
@@ -350,7 +361,7 @@ async def test_run_batch_from_manifest_rejects_stale_upsert_revision(
     use_client(client)
 
     report = await IngesterApp(
-        config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch_from_manifest(
         _manifest(
             BatchChange(
@@ -378,7 +389,7 @@ async def test_run_batch_from_manifest_delete_uses_manifest_even_if_file_reappea
     use_client(client)
 
     report = await IngesterApp(
-        config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch_from_manifest(
         _manifest(
             BatchChange(
@@ -431,7 +442,7 @@ async def test_run_batch_from_manifest_resumes_same_manifest_work(tmp_path, use_
         await engine.dispose()
 
     report = await IngesterApp(
-        config=config, db_path=tmp_path / "db.lancedb"
+        config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb")
     ).run_batch_from_manifest(manifest)
 
     assert report.succeeded == 1
@@ -456,7 +467,7 @@ async def test_run_batch_from_manifest_rejects_non_manifest_pending_work(
 
     with pytest.raises(ValueError, match="non-manifest pending work"):
         await IngesterApp(
-            config=config, db_path=tmp_path / "db.lancedb"
+            config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb")
         ).run_batch_from_manifest(
             _manifest(
                 BatchChange(
@@ -505,7 +516,7 @@ async def test_run_batch_from_manifest_rejects_different_manifest_pending_work(
 
     with pytest.raises(ValueError, match="non-manifest pending work"):
         await IngesterApp(
-            config=config, db_path=tmp_path / "db.lancedb"
+            config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb")
         ).run_batch_from_manifest(manifest)
 
 
@@ -526,7 +537,7 @@ async def test_run_batch_from_manifest_rejects_unrelated_pending_work(
 
     with pytest.raises(ValueError, match="non-manifest pending work"):
         await IngesterApp(
-            config=config, db_path=tmp_path / "db.lancedb"
+            config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb")
         ).run_batch_from_manifest(
             _manifest(
                 BatchChange(
@@ -554,7 +565,7 @@ async def test_run_batch_from_manifest_rejects_duplicate_changes(tmp_path, use_c
 
     with pytest.raises(ValueError, match="duplicate change"):
         await IngesterApp(
-            config=_config(tmp_path), db_path=tmp_path / "db.lancedb"
+            config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
         ).run_batch_from_manifest(_manifest(change, change))
 
 
@@ -583,7 +594,9 @@ async def test_run_batch_aborts_when_all_workers_die(
 
     with caplog.at_level("ERROR", logger="haiku.rag.ingester.app"):
         report = await asyncio.wait_for(
-            IngesterApp(config=config, db_path=tmp_path / "db.lancedb").run_batch(),
+            IngesterApp(
+                config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb")
+            ).run_batch(),
             timeout=10.0,
         )
 
@@ -608,7 +621,7 @@ async def test_serve_starts_workers_pollers_and_shuts_down(tmp_path, use_client,
     use_client(_mock_client())
     config = _config(tmp_path)
     config.ingester.api = APIConfig(enabled=api, host="127.0.0.1", port=0)
-    app = IngesterApp(config=config, db_path=tmp_path / "db.lancedb")
+    app = IngesterApp(config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb"))
 
     task = asyncio.create_task(app.serve(api=api))
     try:
@@ -653,7 +666,7 @@ async def test_stop_pool_warns_when_shutdown_grace_elapses(tmp_path, caplog):
     """When a worker doesn't stop within the shutdown grace, _stop_pool logs a
     warning and still drains any pending cancel-cleanup releases."""
     config = _config(tmp_path, shutdown_grace_s=0.01)
-    app = IngesterApp(config=config, db_path=tmp_path / "db.lancedb")
+    app = IngesterApp(config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb"))
     pool = _SlowPool()
     app._pool = pool
 
@@ -694,7 +707,9 @@ async def test_run_batch_closes_sources_after_pool_stops(
 ):
     (tmp_path / "a.md").write_text("hello")
     use_client(_mock_client())
-    app = IngesterApp(config=_config(tmp_path), db_path=tmp_path / "db.lancedb")
+    app = IngesterApp(
+        config=_config(tmp_path), scope=DatabaseScope.at(tmp_path / "db.lancedb")
+    )
     order = _record_close_order(monkeypatch)
 
     await app.run_batch()
@@ -707,7 +722,7 @@ async def test_serve_closes_sources_after_pool_stops(tmp_path, use_client, monke
     use_client(_mock_client())
     config = _config(tmp_path)
     config.ingester.api = APIConfig(enabled=False)
-    app = IngesterApp(config=config, db_path=tmp_path / "db.lancedb")
+    app = IngesterApp(config=config, scope=DatabaseScope.at(tmp_path / "db.lancedb"))
     order = _record_close_order(monkeypatch)
 
     task = asyncio.create_task(app.serve(api=False))

tests/ingester/test_workers.py

@@ -1054,16 +1054,23 @@ async def test_breaker_isolates_sources(client, jobs, sync):
     for _ in range(10):
         pool._breaker_for("bad").record_failure()
 
+    async def _good_jobs_drained():
+        while True:
+            done = await jobs.list_jobs(status=JobStatus.SUCCEEDED, limit=50)
+            if len(done) == 3:
+                return done
+            await asyncio.sleep(0.02)
+
     await pool.start()
     try:
-        await asyncio.sleep(0.2)
-        succeeded = await jobs.list_jobs(status=JobStatus.SUCCEEDED, limit=50)
+        succeeded = await asyncio.wait_for(_good_jobs_drained(), timeout=5.0)
         queued = await jobs.list_jobs(status=JobStatus.QUEUED, limit=50)
     finally:
         await pool.stop()
 
     assert {j.uri for j in succeeded} == {"g0", "g1", "g2"}
     assert {j.uri for j in queued} == {"b0", "b1", "b2"}
+    assert [j.attempts for j in queued] == [0, 0, 0]
 
 
 @pytest.mark.asyncio