danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generate the multi-database corpus and build its databases

Browse source 
The acceptance dataset for lancedb.databases needs a corpus where the
expected database is known per question, which no existing dataset gives.
Three databases: northern and southern hold station reports on one schema,
equipment holds spec sheets that share the vocabulary and answer nothing.

Names are invented throughout. A real station lets the model answer from
priors, which would measure memorisation rather than retrieval.

Three near-name pairs and one shared entity carry the attribution cases.
Pair members are identical apart from the name and the numbers, since a
difference in instrument or technician would hand the model a free
discriminator. Elevations and years are unique across the corpus so a
number identifies one station, and document counts differ per database so
a count cannot be right by luck while attribution is wrong.

Questions and gold answers both derive from STATIONS and INSTRUMENTS, so
they cannot drift apart. Readings are derived with SHA-256 over database
and name: hash() is salted per process, and keying on the name alone gave
the two Station Auk reports identical tables.

evaluations run populates one database and refuses a configured set, so
the builder opens each database itself and the run is --skip-db. It
asserts no single chunk holds all twelve monthly readings, because
without that S3 collapses into a search question and the guarantee has to
survive a chunker change.
This commit is contained in:
Yiorgis Gozadinos 2026-08-26 15:03:57 +03:00
parent ba760b76c2
commit 7ae395a107
No known key found for this signature in database
3 changed files with 692 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

418
evaluations/evaluations/datasets/multidb.py Normal file
View file

@ -0,0 +1,418 @@
import hashlib
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from haiku.rag.client import HaikuRAG
from haiku.rag.config.models import AppConfig
NORTHERN = "northern"
SOUTHERN = "southern"
EQUIPMENT = "equipment"
MONTHS = (
"January",
"February",
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December",
)
@dataclass(frozen=True)
class Station:
"""One station report. Names are invented: a real station lets the model
answer from priors, which would measure memorisation instead of retrieval."""
name: str
database: str
elevation_m: int
commissioned: int
programme: str
instrument: str
technician: str
@property
def slug(self) -> str:
return self.name.lower().replace(" ", "-")
@property
def uri(self) -> str:
return f"station://{self.database}/{self.slug}"
@property
def title(self) -> str:
return f"Station {self.name} Report"
@property
def readings(self) -> tuple[int, ...]:
"""Twelve monthly wind-speed readings, derived from database and name so
they are stable across rebuilds and interpreters, and so the two Station
Auk reports differ. `hash()` is salted per process, so it cannot be used."""
digest = hashlib.sha256(f"{self.database}/{self.name}".encode()).digest()
return tuple(37 + digest[i] % 53 for i in range(12))
@property
def readings_total(self) -> int:
return sum(self.readings)
@dataclass(frozen=True)
class Instrument:
"""An equipment spec sheet: shares the stations' vocabulary (anemometer,
calibration, elevation) and answers none of the questions."""
model: str
kind: str
calibration_days: int
operating_ceiling_m: int
@property
def slug(self) -> str:
return self.model.lower().replace(" ", "-").replace("/", "-")
@property
def uri(self) -> str:
return f"equipment://{EQUIPMENT}/{self.slug}"
@property
def title(self) -> str:
return f"{self.model} Specification"
NORTHERN_PROGRAMME = "Northern Uplands Programme"
SOUTHERN_PROGRAMME = "Southern Ranges Programme"
# Three near-name pairs, each identical apart from the name and the numbers: the
# wrong database yields a wrong elevation, so B3 fails visibly and cited_sources
# names the culprit. Pair members share instrument and technician so only the name
# and the numbers distinguish them, which is what makes the reranker arm the harder
# one for that family.
#
# Station Auk exists in both databases with commissioning years 24 years apart:
# unscoped it should surface both, scoped it must yield exactly one.
#
# Elevations and years are unique across the corpus, so a number identifies one
# station. Document counts differ per database (9 / 8 / 6) so a count answer cannot
# be right by luck while attribution is wrong.
STATIONS: tuple[Station, ...] = (
Station(
"Kestrel",
NORTHERN,
1240,
1998,
NORTHERN_PROGRAMME,
"Vaisala WXT536",
"R. Aldiss",
),
Station(
"Petrel",
NORTHERN,
860,
1991,
NORTHERN_PROGRAMME,
"Gill WindSonic 4",
"M. Torrance",
),
Station(
"Skua", NORTHERN, 2145, 1989, NORTHERN_PROGRAMME, "Young 81000V", "S. Okonkwo"
),
Station(
"Auk", NORTHERN, 1105, 1987, NORTHERN_PROGRAMME, "Thies 4.3350", "R. Aldiss"
),
Station(
"Gannet",
NORTHERN,
640,
2006,
NORTHERN_PROGRAMME,
"Vaisala WXT536",
"M. Torrance",
),
Station(
"Tern",
NORTHERN,
1420,
2001,
NORTHERN_PROGRAMME,
"Gill WindSonic 4",
"S. Okonkwo",
),
Station(
"Guillemot",
NORTHERN,
1780,
1996,
NORTHERN_PROGRAMME,
"Young 81000V",
"R. Aldiss",
),
Station(
"Shearwater",
NORTHERN,
505,
2013,
NORTHERN_PROGRAMME,
"Thies 4.3350",
"M. Torrance",
),
Station(
"Fulmar",
NORTHERN,
1550,
1984,
NORTHERN_PROGRAMME,
"Campbell CSAT3B",
"S. Okonkwo",
),
Station(
"Kestrel Ridge",
SOUTHERN,
2310,
2004,
SOUTHERN_PROGRAMME,
"Vaisala WXT536",
"R. Aldiss",
),
Station(
"Petrel Point",
SOUTHERN,
1975,
1993,
SOUTHERN_PROGRAMME,
"Gill WindSonic 4",
"M. Torrance",
),
Station(
"Skua Bay",
SOUTHERN,
415,
1979,
SOUTHERN_PROGRAMME,
"Young 81000V",
"S. Okonkwo",
),
Station(
"Auk", SOUTHERN, 1770, 2011, SOUTHERN_PROGRAMME, "Thies 4.3350", "L. Feodorov"
),
Station(
"Albatross",
SOUTHERN,
2540,
1999,
SOUTHERN_PROGRAMME,
"Vaisala WXT536",
"P. Nakamura",
),
Station(
"Prion",
SOUTHERN,
2185,
1995,
SOUTHERN_PROGRAMME,
"Gill WindSonic 4",
"L. Feodorov",
),
Station(
"Sheathbill",
SOUTHERN,
930,
2016,
SOUTHERN_PROGRAMME,
"Young 81000V",
"P. Nakamura",
),
Station(
"Snowcap",
SOUTHERN,
2760,
2009,
SOUTHERN_PROGRAMME,
"Metek uSonic-3",
"L. Feodorov",
),
)
# The near-name pairs, northern member first. B3 draws its instances from these.
NEAR_NAME_PAIRS: tuple[tuple[str, str], ...] = (
("Kestrel", "Kestrel Ridge"),
("Petrel", "Petrel Point"),
("Skua", "Skua Bay"),
)
SHARED_STATION = "Auk"
INSTRUMENTS: tuple[Instrument, ...] = (
Instrument("Vaisala WXT536", "ultrasonic anemometer", 365, 3000),
Instrument("Gill WindSonic 4", "ultrasonic anemometer", 730, 2800),
Instrument("Young 81000V", "ultrasonic anemometer", 400, 3200),
Instrument("Thies 4.3350", "cup anemometer", 545, 2400),
Instrument("Campbell CSAT3B", "sonic anemometer", 300, 3500),
Instrument("Metek uSonic-3", "sonic anemometer", 450, 3100),
)
DATABASE_NAMES = (NORTHERN, SOUTHERN, EQUIPMENT)
def stations_in(database: str) -> tuple[Station, ...]:
return tuple(s for s in STATIONS if s.database == database)
def station(name: str, database: str) -> Station:
for candidate in STATIONS:
if candidate.name == name and candidate.database == database:
return candidate
raise KeyError(f"no station {name!r} in {database!r}")
def render_station_report(s: Station) -> str:
"""Four sections plus a twelve-row table, so `toc.json` has real structure,
`items.jsonl` reports a table, and the readings outlast one chunk."""
rows = "\n".join(
f"| {month} | {value} |"
for month, value in zip(MONTHS, s.readings, strict=True)
)
return f"""# {s.title}
## Overview
Station {s.name} sits at {s.elevation_m} metres and was commissioned in
{s.commissioned}. It belongs to the {s.programme} and reports hourly.
## Instruments
The primary sensor is a {s.instrument}. Calibration is verified against the
programme reference before each seasonal changeover.
## Measurements
Mean monthly wind speed, in tenths of a metre per second, for the reporting year:
| Month | Mean wind speed |
| --- | --- |
{rows}
## Maintenance
Maintenance is carried out by {s.technician}. The mast was last inspected during
the autumn visit; no corrosion was recorded and the guy tensions were within
tolerance.
"""
def render_spec_sheet(i: Instrument) -> str:
return f"""# {i.title}
## Overview
The {i.model} is a {i.kind} used across the station network. It is rated for
installation up to an elevation of {i.operating_ceiling_m} metres.
## Calibration
The recommended calibration interval is {i.calibration_days} days. Calibration
is performed against a reference anemometer under laboratory conditions.
## Notes
This sheet describes the instrument only. It records no station, no programme
and no measurement history.
"""
@dataclass(frozen=True)
class CorpusDocument:
database: str
uri: str
title: str
content: str
def corpus() -> tuple[CorpusDocument, ...]:
docs = [
CorpusDocument(s.database, s.uri, s.title, render_station_report(s))
for s in STATIONS
]
docs += [
CorpusDocument(EQUIPMENT, i.uri, i.title, render_spec_sheet(i))
for i in INSTRUMENTS
]
return tuple(docs)
def documents_for(database: str) -> tuple[CorpusDocument, ...]:
return tuple(d for d in corpus() if d.database == database)
class TableSplitError(AssertionError):
"""A single chunk holds every monthly reading, so `content.txt` is no longer
the only route to their total and the S3 case would test search instead."""
async def _assert_readings_outlast_one_chunk(client: HaikuRAG, s: Station) -> None:
doc = await client.get_document_by_uri(s.uri)
if doc is None or doc.id is None: # pragma: no cover - the builder just wrote it
raise TableSplitError(f"{s.uri} missing after import")
values = [str(v) for v in s.readings]
for chunk in await client.chunk_repository.get_by_document_id(doc.id):
if all(v in chunk.content for v in values):
raise TableSplitError(
f"one chunk of {s.uri} holds all {len(values)} monthly readings; "
"lower processing.chunk_size so the table splits"
)
async def build_databases(config: AppConfig) -> dict[str, int]:
"""Write each configured database from the generated corpus.
`evaluations run` populates one database and refuses a configured set, so
this dataset builds its own and the run is `--skip-db`.
"""
configured = set(config.lancedb.databases or {})
missing = set(DATABASE_NAMES) - configured
if missing:
raise ValueError(
f"lancedb.databases must place {sorted(DATABASE_NAMES)}; missing {sorted(missing)}"
)
written: dict[str, int] = {}
for name in DATABASE_NAMES:
async with HaikuRAG(config=config, sources=[name], create=True) as client:
for doc in documents_for(name):
await client.create_document(doc.content, uri=doc.uri, title=doc.title)
for s in stations_in(name):
await _assert_readings_outlast_one_chunk(client, s)
written[name] = len(documents_for(name))
return written
def iter_expected_totals() -> Iterator[tuple[Station, int]]:
for s in STATIONS:
yield s, s.readings_total
async def main() -> None: # pragma: no cover - operator entry point
import argparse
from haiku.rag.config import AppConfig, load_yaml_config
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, required=True)
args = parser.parse_args()
config = AppConfig.model_validate(load_yaml_config(args.config))
written = await build_databases(config)
for name, count in written.items():
print(f"{name}: {count} documents")
if __name__ == "__main__": # pragma: no cover - operator entry point
import asyncio
asyncio.run(main())

159
evaluations/evaluations/datasets/multidb_design.md Normal file
View file

@ -0,0 +1,159 @@
# The multi-database acceptance dataset
Acceptance gate for `lancedb.databases` (searching, asking and analyzing across
several databases at once), and a permanent regression dataset afterwards. The unit
tests pin mechanism; this dataset exists for three failure modes that are silent end
to end.
## The three failure modes
1. **Wrong attribution** — the right answer, cited to the wrong database. `cited_map`
scores URIs and would pass it.
2. **Lost evidence under fusion** — a fact in the second database never reaches the
model because the first filled the limit. Looks like a knowledge gap.
3. **Scope leakage** — a question scoped with `sources` draws on a database it
excluded.
`cited_sources` is already recorded per case, and `AnalysisState.executions` already
stores the model's own Python, so both attribution and sandbox-surface coverage are
measurable without production changes. What this dataset adds is a corpus where the
expected database is known per question.
## Corpus
Three databases: `northern` and `southern` hold station reports on the same schema,
`equipment` holds instrument spec sheets that share the vocabulary (anemometer,
calibration, elevation) and answer none of the questions. Counts differ — 9 / 8 / 6 —
so a count answer cannot be right by luck while attribution is wrong.
**Every name is invented.** A real station lets the model answer from priors, and the
eval would measure memorisation instead of retrieval. This is load-bearing: do not
replace these with real stations.
Each report has four sections (Overview / Instruments / Measurements / Maintenance)
plus a twelve-row monthly table under `Measurements`. Elevation and commissioning year
appear in **prose under Overview, never in the table**, so B1 and B3 stay chunk-level
retrieval questions instead of becoming table-reading questions. Elevations and years
are unique across the corpus, so a number identifies exactly one station.
Two kinds of deliberate collision:
- **Three near-name pairs** — Kestrel/Kestrel Ridge, Petrel/Petrel Point, Skua/Skua
Bay. Members of a pair are identical apart from the name and the numbers, sharing
instrument and technician, because a difference in either would hand the model a free
discriminator. This is what makes B3 *harder* with a reranker (see Arms).
- **One shared entity** — `Station Auk` in both databases, commissioned 1987 and 2011.
Unscoped it should surface both; scoped it must yield exactly one.
The corpus is generated from `STATIONS` and `INSTRUMENTS` in `multidb.py`, and the
expected answers are derived from the same definitions, so questions and gold answers
cannot drift apart. Monthly readings are derived with SHA-256 over
`{database}/{name}` — `hash()` is salted per process and would not be stable across
rebuilds, and keying on the name alone would give the two Station Auk reports identical
tables.
## Building
`evaluations run` populates one database and refuses a configured set, so this dataset
builds its own:
```
uv run python -m evaluations.datasets.multidb --config evaluations/configs/multidb.yaml
evaluations run multidb --config evaluations/configs/multidb.yaml --skip-db --skip-retrieval
```
The builder asserts, per station, that **no single chunk holds all twelve monthly
readings**, and fails the build if one does. Without that guarantee S3 collapses into a
search question, and the guarantee has to survive a chunker change — so it is asserted,
not assumed.
## Question families
B-family runs against both capabilities, S-family against analysis only.
| | family | proves |
|---|---|---|
| B1 | single-source fact | `cited_sources == ["northern"]` |
| B2 | cross-database join | fusion keeps the second fact; both cited |
| B3 | near-name distractor | precision — the twin must not be cited |
| B4 | B1 scoped with `sources` | scope honoured, zero leakage |
| B5 | shared entity, unscoped | surfaces and attributes both |
| B6 | absent station | grounded refusal |
| B7 | `sources=[]` | empty scope refuses rather than confabulating |
| S1 | documents per database | `list_documents()` |
| S2 | stations per programme mentioning a term | in-code `search()` |
| S3 | total of one station's monthly readings | whole-document surface |
| S4 | item and table counts | `items.jsonl` |
| S5 | section headings in order | `toc.json` |
| S6 | uri and database of a titled document | `metadata.json` |
**B3 and B4 carry 8-10 instances each**, because their gates are pass/fail and "zero
leakage" over three cases is not evidence of absence. **Half the B4 instances put the
better answer in the excluded database**, so that honouring scope costs the model the
better-matching chunk — otherwise the scoped-in database always holds the best content
and no-leakage is satisfied trivially.
**B7 is API-only.** The CLI has no `--sources` (it has `--database` for exactly one),
so an empty scope is unreachable from the command line.
### Why S3 is not a `content.txt` question
`extract_item_text` serialises a table item **to markdown**, and `items.jsonl` emits
`item.text` per item, so a table item's text is the whole twelve-row table.
`items.jsonl` is therefore an information superset of `content.txt`: same text,
itemised, plus structure and chunk ids. There is no natural question answerable only
from `content.txt`, and anything contrived enough to force it would measure the
plumbing rather than the capability.
So S3 asserts that the model reached **a whole-document surface — either `content.txt`
or `items.jsonl`** — and which one it chose is reported as a measurement. If
`content.txt` is never chosen, that is a finding about whether the file earns its place
in the VFS, the same class of question as `HAIKU_RAG_DISABLE_TOC`.
Surface coverage generally is **measured, not forced**: the model picks its route and
the run reports what it touched.
## Arms
- **RAG x {reranker, none}**, and the two arms cover *different* families rather than
the same family at two strengths. Over-fetch is reranker-gated
(`client/search.py`, `limit * _RERANK_OVERFETCH if client.reranker else limit`), so:
- **Fusion loss (B2) is a no-reranker property.** At `limit` 5 across three
databases, RRF fuses 15 candidates to 5 and the truncation bites.
- **Attribution under confusion (B3) is harder *with* the reranker**, because a
reranker scoring the union has no notion of source and is doing semantic matching
on exactly the near-identical pair built to confuse it. Without one, RRF keeps the
pair apart because each database contributes its own rank-1.
- **Analysis x reranker** for the S-family.
- Optionally analysis with `HAIKU_RAG_DISABLE_TOC=1`, which answers what that toggle
was added for.
### RRF ties resolve to configured order
In the RRF branch of `_fuse`, the sort key is the score alone. Python's sort is stable
and `reverse=True` preserves the relative order of equal elements, so equal-scoring
candidates resolve to insertion order, which is `source_names`, which is **the order
the databases appear in the configuration**. At `limit` 5 across three databases the
slot allocation is therefore decided by config order, not relevance.
Consequence for this dataset: **B2 instances randomise database order**, or the family
measures ordering instead of fusion. Consequence for operators: the first database
listed wins ties, and nothing else documents that.
## Gates and rates
**Two hard gates.** Any failure is a bug, not a rate:
- **Scope leakage = 0** — no B4 or B7 case cites a database outside its scope.
- **Attribution errors = 0** on B1 and B3, where exactly one database is correct.
**Rates**: judge pass split behaviour/structural, `cited_map`, and a surface-coverage
table built from the recorded code.
Scoring is **deterministic** everywhere except B6. The answers are known numbers, names
and counts, so exact and numeric matching score them without inheriting judge failure
modes — a red gate has to mean the code is broken, not that the judge spiralled. B1 and
B3 assert **gold present AND distractor absent**, because a hedge ("either 1240 m or
2310 m") passes a presence check while being exactly the failure B3 exists to catch.
Numerics are extracted and compared, never string-matched, since "1,240 metres",
"1240 m" and a full sentence are all legitimate.

115
evaluations/tests/test_multidb_corpus.py Normal file
View file

@ -0,0 +1,115 @@
import pytest
from evaluations.datasets.multidb import (
DATABASE_NAMES,
EQUIPMENT,
MONTHS,
NORTHERN,
SOUTHERN,
build_databases,
corpus,
documents_for,
render_station_report,
station,
stations_in,
)
from haiku.rag.config.models import AppConfig, LanceDBConfig
def test_document_counts_differ_per_database():
"""S1 asks how many documents a database holds. Equal counts would let a
wrong-attribution answer pass, so the three differ."""
counts = {name: len(documents_for(name)) for name in DATABASE_NAMES}
assert counts == {NORTHERN: 9, SOUTHERN: 8, EQUIPMENT: 6}
assert len(set(counts.values())) == len(counts)
assert len(corpus()) == sum(counts.values())
def test_elevations_and_years_identify_one_station():
"""Gold answers are numbers, so a number must not be ambiguous across the
corpus."""
from evaluations.datasets.multidb import STATIONS
assert len({s.elevation_m for s in STATIONS}) == len(STATIONS)
assert len({s.commissioned for s in STATIONS}) == len(STATIONS)
def test_near_name_pairs_differ_only_in_name_and_numbers():
"""B3's value is that the pair is otherwise identical; a difference in
instrument or technician would give the model a free discriminator."""
from evaluations.datasets.multidb import NEAR_NAME_PAIRS
for north_name, south_name in NEAR_NAME_PAIRS:
north, south = station(north_name, NORTHERN), station(south_name, SOUTHERN)
assert north.instrument == south.instrument
assert north.technician == south.technician
assert north.elevation_m != south.elevation_m
assert north.commissioned != south.commissioned
def test_near_name_stations_disagree_on_elevation():
"""B3 depends on the wrong database yielding a wrong number, so the pair must
never share an elevation."""
assert station("Kestrel", NORTHERN).elevation_m == 1240
assert station("Kestrel Ridge", SOUTHERN).elevation_m == 2310
def test_shared_entity_differs_between_databases():
"""Station Auk exists in both. B5 needs the two to be distinguishable, and S3
needs their readings to differ or the expected total is ambiguous."""
northern, southern = station("Auk", NORTHERN), station("Auk", SOUTHERN)
assert northern.commissioned != southern.commissioned
assert northern.readings != southern.readings
assert northern.uri != southern.uri
def test_readings_are_stable():
"""Gold answers are derived from these, so a change to the derivation silently
rewrites every expected total. Pinned deliberately."""
assert station("Kestrel", NORTHERN).readings_total == 689
assert station("Auk", NORTHERN).readings_total == 756
assert station("Auk", SOUTHERN).readings_total == 653
def test_station_report_has_four_sections_and_twelve_rows():
report = render_station_report(station("Kestrel", NORTHERN))
for heading in (
"## Overview",
"## Instruments",
"## Measurements",
"## Maintenance",
):
assert heading in report
for month in MONTHS:
assert f"| {month} |" in report
def test_equipment_shares_vocabulary_but_answers_nothing():
"""The dilution probe only works if the sheets compete on wording while
holding no station facts."""
sheets = documents_for(EQUIPMENT)
assert all("anemometer" in d.content for d in sheets)
assert all("calibration" in d.content.lower() for d in sheets)
station_names = {s.name for s in stations_in(NORTHERN)} | {
s.name for s in stations_in(SOUTHERN)
}
for sheet in sheets:
assert not any(f"Station {name}" in sheet.content for name in station_names)
assert "Programme" not in sheet.content
def test_every_station_is_reachable_by_name_and_database():
for s in (*stations_in(NORTHERN), *stations_in(SOUTHERN)):
assert station(s.name, s.database) is s
with pytest.raises(KeyError):
station("Kestrel", SOUTHERN)
async def test_build_refuses_a_config_missing_a_database():
"""Building into a config that does not place all three would write a corpus
the run cannot read."""
config = AppConfig(
lancedb=LanceDBConfig(databases={NORTHERN: "/tmp/n", SOUTHERN: "/tmp/s"})
)
with pytest.raises(ValueError, match="equipment"):
await build_databases(config)