From 7ae395a107e8e432470677d4075ec25193af426b Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 26 Aug 2026 15:03:57 +0300 Subject: [PATCH] Generate the multi-database corpus and build its databases The acceptance dataset for lancedb.databases needs a corpus where the expected database is known per question, which no existing dataset gives. Three databases: northern and southern hold station reports on one schema, equipment holds spec sheets that share the vocabulary and answer nothing. Names are invented throughout. A real station lets the model answer from priors, which would measure memorisation rather than retrieval. Three near-name pairs and one shared entity carry the attribution cases. Pair members are identical apart from the name and the numbers, since a difference in instrument or technician would hand the model a free discriminator. Elevations and years are unique across the corpus so a number identifies one station, and document counts differ per database so a count cannot be right by luck while attribution is wrong. Questions and gold answers both derive from STATIONS and INSTRUMENTS, so they cannot drift apart. Readings are derived with SHA-256 over database and name: hash() is salted per process, and keying on the name alone gave the two Station Auk reports identical tables. evaluations run populates one database and refuses a configured set, so the builder opens each database itself and the run is --skip-db. It asserts no single chunk holds all twelve monthly readings, because without that S3 collapses into a search question and the guarantee has to survive a chunker change. --- evaluations/evaluations/datasets/multidb.py | 418 ++++++++++++++++++ .../evaluations/datasets/multidb_design.md | 159 +++++++ evaluations/tests/test_multidb_corpus.py | 115 +++++ 3 files changed, 692 insertions(+) create mode 100644 evaluations/evaluations/datasets/multidb.py create mode 100644 evaluations/evaluations/datasets/multidb_design.md create mode 100644 evaluations/tests/test_multidb_corpus.py diff --git a/evaluations/evaluations/datasets/multidb.py b/evaluations/evaluations/datasets/multidb.py new file mode 100644 index 00000000..6f6c62d9 --- /dev/null +++ b/evaluations/evaluations/datasets/multidb.py @@ -0,0 +1,418 @@ +import hashlib +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +from haiku.rag.client import HaikuRAG +from haiku.rag.config.models import AppConfig + +NORTHERN = "northern" +SOUTHERN = "southern" +EQUIPMENT = "equipment" + +MONTHS = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) + + +@dataclass(frozen=True) +class Station: + """One station report. Names are invented: a real station lets the model + answer from priors, which would measure memorisation instead of retrieval.""" + + name: str + database: str + elevation_m: int + commissioned: int + programme: str + instrument: str + technician: str + + @property + def slug(self) -> str: + return self.name.lower().replace(" ", "-") + + @property + def uri(self) -> str: + return f"station://{self.database}/{self.slug}" + + @property + def title(self) -> str: + return f"Station {self.name} Report" + + @property + def readings(self) -> tuple[int, ...]: + """Twelve monthly wind-speed readings, derived from database and name so + they are stable across rebuilds and interpreters, and so the two Station + Auk reports differ. `hash()` is salted per process, so it cannot be used.""" + digest = hashlib.sha256(f"{self.database}/{self.name}".encode()).digest() + return tuple(37 + digest[i] % 53 for i in range(12)) + + @property + def readings_total(self) -> int: + return sum(self.readings) + + +@dataclass(frozen=True) +class Instrument: + """An equipment spec sheet: shares the stations' vocabulary (anemometer, + calibration, elevation) and answers none of the questions.""" + + model: str + kind: str + calibration_days: int + operating_ceiling_m: int + + @property + def slug(self) -> str: + return self.model.lower().replace(" ", "-").replace("/", "-") + + @property + def uri(self) -> str: + return f"equipment://{EQUIPMENT}/{self.slug}" + + @property + def title(self) -> str: + return f"{self.model} Specification" + + +NORTHERN_PROGRAMME = "Northern Uplands Programme" +SOUTHERN_PROGRAMME = "Southern Ranges Programme" + +# Three near-name pairs, each identical apart from the name and the numbers: the +# wrong database yields a wrong elevation, so B3 fails visibly and cited_sources +# names the culprit. Pair members share instrument and technician so only the name +# and the numbers distinguish them, which is what makes the reranker arm the harder +# one for that family. +# +# Station Auk exists in both databases with commissioning years 24 years apart: +# unscoped it should surface both, scoped it must yield exactly one. +# +# Elevations and years are unique across the corpus, so a number identifies one +# station. Document counts differ per database (9 / 8 / 6) so a count answer cannot +# be right by luck while attribution is wrong. +STATIONS: tuple[Station, ...] = ( + Station( + "Kestrel", + NORTHERN, + 1240, + 1998, + NORTHERN_PROGRAMME, + "Vaisala WXT536", + "R. Aldiss", + ), + Station( + "Petrel", + NORTHERN, + 860, + 1991, + NORTHERN_PROGRAMME, + "Gill WindSonic 4", + "M. Torrance", + ), + Station( + "Skua", NORTHERN, 2145, 1989, NORTHERN_PROGRAMME, "Young 81000V", "S. Okonkwo" + ), + Station( + "Auk", NORTHERN, 1105, 1987, NORTHERN_PROGRAMME, "Thies 4.3350", "R. Aldiss" + ), + Station( + "Gannet", + NORTHERN, + 640, + 2006, + NORTHERN_PROGRAMME, + "Vaisala WXT536", + "M. Torrance", + ), + Station( + "Tern", + NORTHERN, + 1420, + 2001, + NORTHERN_PROGRAMME, + "Gill WindSonic 4", + "S. Okonkwo", + ), + Station( + "Guillemot", + NORTHERN, + 1780, + 1996, + NORTHERN_PROGRAMME, + "Young 81000V", + "R. Aldiss", + ), + Station( + "Shearwater", + NORTHERN, + 505, + 2013, + NORTHERN_PROGRAMME, + "Thies 4.3350", + "M. Torrance", + ), + Station( + "Fulmar", + NORTHERN, + 1550, + 1984, + NORTHERN_PROGRAMME, + "Campbell CSAT3B", + "S. Okonkwo", + ), + Station( + "Kestrel Ridge", + SOUTHERN, + 2310, + 2004, + SOUTHERN_PROGRAMME, + "Vaisala WXT536", + "R. Aldiss", + ), + Station( + "Petrel Point", + SOUTHERN, + 1975, + 1993, + SOUTHERN_PROGRAMME, + "Gill WindSonic 4", + "M. Torrance", + ), + Station( + "Skua Bay", + SOUTHERN, + 415, + 1979, + SOUTHERN_PROGRAMME, + "Young 81000V", + "S. Okonkwo", + ), + Station( + "Auk", SOUTHERN, 1770, 2011, SOUTHERN_PROGRAMME, "Thies 4.3350", "L. Feodorov" + ), + Station( + "Albatross", + SOUTHERN, + 2540, + 1999, + SOUTHERN_PROGRAMME, + "Vaisala WXT536", + "P. Nakamura", + ), + Station( + "Prion", + SOUTHERN, + 2185, + 1995, + SOUTHERN_PROGRAMME, + "Gill WindSonic 4", + "L. Feodorov", + ), + Station( + "Sheathbill", + SOUTHERN, + 930, + 2016, + SOUTHERN_PROGRAMME, + "Young 81000V", + "P. Nakamura", + ), + Station( + "Snowcap", + SOUTHERN, + 2760, + 2009, + SOUTHERN_PROGRAMME, + "Metek uSonic-3", + "L. Feodorov", + ), +) + +# The near-name pairs, northern member first. B3 draws its instances from these. +NEAR_NAME_PAIRS: tuple[tuple[str, str], ...] = ( + ("Kestrel", "Kestrel Ridge"), + ("Petrel", "Petrel Point"), + ("Skua", "Skua Bay"), +) + +SHARED_STATION = "Auk" + +INSTRUMENTS: tuple[Instrument, ...] = ( + Instrument("Vaisala WXT536", "ultrasonic anemometer", 365, 3000), + Instrument("Gill WindSonic 4", "ultrasonic anemometer", 730, 2800), + Instrument("Young 81000V", "ultrasonic anemometer", 400, 3200), + Instrument("Thies 4.3350", "cup anemometer", 545, 2400), + Instrument("Campbell CSAT3B", "sonic anemometer", 300, 3500), + Instrument("Metek uSonic-3", "sonic anemometer", 450, 3100), +) + +DATABASE_NAMES = (NORTHERN, SOUTHERN, EQUIPMENT) + + +def stations_in(database: str) -> tuple[Station, ...]: + return tuple(s for s in STATIONS if s.database == database) + + +def station(name: str, database: str) -> Station: + for candidate in STATIONS: + if candidate.name == name and candidate.database == database: + return candidate + raise KeyError(f"no station {name!r} in {database!r}") + + +def render_station_report(s: Station) -> str: + """Four sections plus a twelve-row table, so `toc.json` has real structure, + `items.jsonl` reports a table, and the readings outlast one chunk.""" + rows = "\n".join( + f"| {month} | {value} |" + for month, value in zip(MONTHS, s.readings, strict=True) + ) + return f"""# {s.title} + +## Overview + +Station {s.name} sits at {s.elevation_m} metres and was commissioned in +{s.commissioned}. It belongs to the {s.programme} and reports hourly. + +## Instruments + +The primary sensor is a {s.instrument}. Calibration is verified against the +programme reference before each seasonal changeover. + +## Measurements + +Mean monthly wind speed, in tenths of a metre per second, for the reporting year: + +| Month | Mean wind speed | +| --- | --- | +{rows} + +## Maintenance + +Maintenance is carried out by {s.technician}. The mast was last inspected during +the autumn visit; no corrosion was recorded and the guy tensions were within +tolerance. +""" + + +def render_spec_sheet(i: Instrument) -> str: + return f"""# {i.title} + +## Overview + +The {i.model} is a {i.kind} used across the station network. It is rated for +installation up to an elevation of {i.operating_ceiling_m} metres. + +## Calibration + +The recommended calibration interval is {i.calibration_days} days. Calibration +is performed against a reference anemometer under laboratory conditions. + +## Notes + +This sheet describes the instrument only. It records no station, no programme +and no measurement history. +""" + + +@dataclass(frozen=True) +class CorpusDocument: + database: str + uri: str + title: str + content: str + + +def corpus() -> tuple[CorpusDocument, ...]: + docs = [ + CorpusDocument(s.database, s.uri, s.title, render_station_report(s)) + for s in STATIONS + ] + docs += [ + CorpusDocument(EQUIPMENT, i.uri, i.title, render_spec_sheet(i)) + for i in INSTRUMENTS + ] + return tuple(docs) + + +def documents_for(database: str) -> tuple[CorpusDocument, ...]: + return tuple(d for d in corpus() if d.database == database) + + +class TableSplitError(AssertionError): + """A single chunk holds every monthly reading, so `content.txt` is no longer + the only route to their total and the S3 case would test search instead.""" + + +async def _assert_readings_outlast_one_chunk(client: HaikuRAG, s: Station) -> None: + doc = await client.get_document_by_uri(s.uri) + if doc is None or doc.id is None: # pragma: no cover - the builder just wrote it + raise TableSplitError(f"{s.uri} missing after import") + values = [str(v) for v in s.readings] + for chunk in await client.chunk_repository.get_by_document_id(doc.id): + if all(v in chunk.content for v in values): + raise TableSplitError( + f"one chunk of {s.uri} holds all {len(values)} monthly readings; " + "lower processing.chunk_size so the table splits" + ) + + +async def build_databases(config: AppConfig) -> dict[str, int]: + """Write each configured database from the generated corpus. + + `evaluations run` populates one database and refuses a configured set, so + this dataset builds its own and the run is `--skip-db`. + """ + configured = set(config.lancedb.databases or {}) + missing = set(DATABASE_NAMES) - configured + if missing: + raise ValueError( + f"lancedb.databases must place {sorted(DATABASE_NAMES)}; missing {sorted(missing)}" + ) + + written: dict[str, int] = {} + for name in DATABASE_NAMES: + async with HaikuRAG(config=config, sources=[name], create=True) as client: + for doc in documents_for(name): + await client.create_document(doc.content, uri=doc.uri, title=doc.title) + for s in stations_in(name): + await _assert_readings_outlast_one_chunk(client, s) + written[name] = len(documents_for(name)) + return written + + +def iter_expected_totals() -> Iterator[tuple[Station, int]]: + for s in STATIONS: + yield s, s.readings_total + + +async def main() -> None: # pragma: no cover - operator entry point + import argparse + + from haiku.rag.config import AppConfig, load_yaml_config + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + args = parser.parse_args() + config = AppConfig.model_validate(load_yaml_config(args.config)) + written = await build_databases(config) + for name, count in written.items(): + print(f"{name}: {count} documents") + + +if __name__ == "__main__": # pragma: no cover - operator entry point + import asyncio + + asyncio.run(main()) diff --git a/evaluations/evaluations/datasets/multidb_design.md b/evaluations/evaluations/datasets/multidb_design.md new file mode 100644 index 00000000..34ef95f5 --- /dev/null +++ b/evaluations/evaluations/datasets/multidb_design.md @@ -0,0 +1,159 @@ +# The multi-database acceptance dataset + +Acceptance gate for `lancedb.databases` (searching, asking and analyzing across +several databases at once), and a permanent regression dataset afterwards. The unit +tests pin mechanism; this dataset exists for three failure modes that are silent end +to end. + +## The three failure modes + +1. **Wrong attribution** — the right answer, cited to the wrong database. `cited_map` + scores URIs and would pass it. +2. **Lost evidence under fusion** — a fact in the second database never reaches the + model because the first filled the limit. Looks like a knowledge gap. +3. **Scope leakage** — a question scoped with `sources` draws on a database it + excluded. + +`cited_sources` is already recorded per case, and `AnalysisState.executions` already +stores the model's own Python, so both attribution and sandbox-surface coverage are +measurable without production changes. What this dataset adds is a corpus where the +expected database is known per question. + +## Corpus + +Three databases: `northern` and `southern` hold station reports on the same schema, +`equipment` holds instrument spec sheets that share the vocabulary (anemometer, +calibration, elevation) and answer none of the questions. Counts differ — 9 / 8 / 6 — +so a count answer cannot be right by luck while attribution is wrong. + +**Every name is invented.** A real station lets the model answer from priors, and the +eval would measure memorisation instead of retrieval. This is load-bearing: do not +replace these with real stations. + +Each report has four sections (Overview / Instruments / Measurements / Maintenance) +plus a twelve-row monthly table under `Measurements`. Elevation and commissioning year +appear in **prose under Overview, never in the table**, so B1 and B3 stay chunk-level +retrieval questions instead of becoming table-reading questions. Elevations and years +are unique across the corpus, so a number identifies exactly one station. + +Two kinds of deliberate collision: + +- **Three near-name pairs** — Kestrel/Kestrel Ridge, Petrel/Petrel Point, Skua/Skua + Bay. Members of a pair are identical apart from the name and the numbers, sharing + instrument and technician, because a difference in either would hand the model a free + discriminator. This is what makes B3 *harder* with a reranker (see Arms). +- **One shared entity** — `Station Auk` in both databases, commissioned 1987 and 2011. + Unscoped it should surface both; scoped it must yield exactly one. + +The corpus is generated from `STATIONS` and `INSTRUMENTS` in `multidb.py`, and the +expected answers are derived from the same definitions, so questions and gold answers +cannot drift apart. Monthly readings are derived with SHA-256 over +`{database}/{name}` — `hash()` is salted per process and would not be stable across +rebuilds, and keying on the name alone would give the two Station Auk reports identical +tables. + +## Building + +`evaluations run` populates one database and refuses a configured set, so this dataset +builds its own: + +``` +uv run python -m evaluations.datasets.multidb --config evaluations/configs/multidb.yaml +evaluations run multidb --config evaluations/configs/multidb.yaml --skip-db --skip-retrieval +``` + +The builder asserts, per station, that **no single chunk holds all twelve monthly +readings**, and fails the build if one does. Without that guarantee S3 collapses into a +search question, and the guarantee has to survive a chunker change — so it is asserted, +not assumed. + +## Question families + +B-family runs against both capabilities, S-family against analysis only. + +| | family | proves | +|---|---|---| +| B1 | single-source fact | `cited_sources == ["northern"]` | +| B2 | cross-database join | fusion keeps the second fact; both cited | +| B3 | near-name distractor | precision — the twin must not be cited | +| B4 | B1 scoped with `sources` | scope honoured, zero leakage | +| B5 | shared entity, unscoped | surfaces and attributes both | +| B6 | absent station | grounded refusal | +| B7 | `sources=[]` | empty scope refuses rather than confabulating | +| S1 | documents per database | `list_documents()` | +| S2 | stations per programme mentioning a term | in-code `search()` | +| S3 | total of one station's monthly readings | whole-document surface | +| S4 | item and table counts | `items.jsonl` | +| S5 | section headings in order | `toc.json` | +| S6 | uri and database of a titled document | `metadata.json` | + +**B3 and B4 carry 8-10 instances each**, because their gates are pass/fail and "zero +leakage" over three cases is not evidence of absence. **Half the B4 instances put the +better answer in the excluded database**, so that honouring scope costs the model the +better-matching chunk — otherwise the scoped-in database always holds the best content +and no-leakage is satisfied trivially. + +**B7 is API-only.** The CLI has no `--sources` (it has `--database` for exactly one), +so an empty scope is unreachable from the command line. + +### Why S3 is not a `content.txt` question + +`extract_item_text` serialises a table item **to markdown**, and `items.jsonl` emits +`item.text` per item, so a table item's text is the whole twelve-row table. +`items.jsonl` is therefore an information superset of `content.txt`: same text, +itemised, plus structure and chunk ids. There is no natural question answerable only +from `content.txt`, and anything contrived enough to force it would measure the +plumbing rather than the capability. + +So S3 asserts that the model reached **a whole-document surface — either `content.txt` +or `items.jsonl`** — and which one it chose is reported as a measurement. If +`content.txt` is never chosen, that is a finding about whether the file earns its place +in the VFS, the same class of question as `HAIKU_RAG_DISABLE_TOC`. + +Surface coverage generally is **measured, not forced**: the model picks its route and +the run reports what it touched. + +## Arms + +- **RAG x {reranker, none}**, and the two arms cover *different* families rather than + the same family at two strengths. Over-fetch is reranker-gated + (`client/search.py`, `limit * _RERANK_OVERFETCH if client.reranker else limit`), so: + - **Fusion loss (B2) is a no-reranker property.** At `limit` 5 across three + databases, RRF fuses 15 candidates to 5 and the truncation bites. + - **Attribution under confusion (B3) is harder *with* the reranker**, because a + reranker scoring the union has no notion of source and is doing semantic matching + on exactly the near-identical pair built to confuse it. Without one, RRF keeps the + pair apart because each database contributes its own rank-1. +- **Analysis x reranker** for the S-family. +- Optionally analysis with `HAIKU_RAG_DISABLE_TOC=1`, which answers what that toggle + was added for. + +### RRF ties resolve to configured order + +In the RRF branch of `_fuse`, the sort key is the score alone. Python's sort is stable +and `reverse=True` preserves the relative order of equal elements, so equal-scoring +candidates resolve to insertion order, which is `source_names`, which is **the order +the databases appear in the configuration**. At `limit` 5 across three databases the +slot allocation is therefore decided by config order, not relevance. + +Consequence for this dataset: **B2 instances randomise database order**, or the family +measures ordering instead of fusion. Consequence for operators: the first database +listed wins ties, and nothing else documents that. + +## Gates and rates + +**Two hard gates.** Any failure is a bug, not a rate: + +- **Scope leakage = 0** — no B4 or B7 case cites a database outside its scope. +- **Attribution errors = 0** on B1 and B3, where exactly one database is correct. + +**Rates**: judge pass split behaviour/structural, `cited_map`, and a surface-coverage +table built from the recorded code. + +Scoring is **deterministic** everywhere except B6. The answers are known numbers, names +and counts, so exact and numeric matching score them without inheriting judge failure +modes — a red gate has to mean the code is broken, not that the judge spiralled. B1 and +B3 assert **gold present AND distractor absent**, because a hedge ("either 1240 m or +2310 m") passes a presence check while being exactly the failure B3 exists to catch. +Numerics are extracted and compared, never string-matched, since "1,240 metres", +"1240 m" and a full sentence are all legitimate. diff --git a/evaluations/tests/test_multidb_corpus.py b/evaluations/tests/test_multidb_corpus.py new file mode 100644 index 00000000..92f1b82e --- /dev/null +++ b/evaluations/tests/test_multidb_corpus.py @@ -0,0 +1,115 @@ +import pytest + +from evaluations.datasets.multidb import ( + DATABASE_NAMES, + EQUIPMENT, + MONTHS, + NORTHERN, + SOUTHERN, + build_databases, + corpus, + documents_for, + render_station_report, + station, + stations_in, +) +from haiku.rag.config.models import AppConfig, LanceDBConfig + + +def test_document_counts_differ_per_database(): + """S1 asks how many documents a database holds. Equal counts would let a + wrong-attribution answer pass, so the three differ.""" + counts = {name: len(documents_for(name)) for name in DATABASE_NAMES} + assert counts == {NORTHERN: 9, SOUTHERN: 8, EQUIPMENT: 6} + assert len(set(counts.values())) == len(counts) + assert len(corpus()) == sum(counts.values()) + + +def test_elevations_and_years_identify_one_station(): + """Gold answers are numbers, so a number must not be ambiguous across the + corpus.""" + from evaluations.datasets.multidb import STATIONS + + assert len({s.elevation_m for s in STATIONS}) == len(STATIONS) + assert len({s.commissioned for s in STATIONS}) == len(STATIONS) + + +def test_near_name_pairs_differ_only_in_name_and_numbers(): + """B3's value is that the pair is otherwise identical; a difference in + instrument or technician would give the model a free discriminator.""" + from evaluations.datasets.multidb import NEAR_NAME_PAIRS + + for north_name, south_name in NEAR_NAME_PAIRS: + north, south = station(north_name, NORTHERN), station(south_name, SOUTHERN) + assert north.instrument == south.instrument + assert north.technician == south.technician + assert north.elevation_m != south.elevation_m + assert north.commissioned != south.commissioned + + +def test_near_name_stations_disagree_on_elevation(): + """B3 depends on the wrong database yielding a wrong number, so the pair must + never share an elevation.""" + assert station("Kestrel", NORTHERN).elevation_m == 1240 + assert station("Kestrel Ridge", SOUTHERN).elevation_m == 2310 + + +def test_shared_entity_differs_between_databases(): + """Station Auk exists in both. B5 needs the two to be distinguishable, and S3 + needs their readings to differ or the expected total is ambiguous.""" + northern, southern = station("Auk", NORTHERN), station("Auk", SOUTHERN) + assert northern.commissioned != southern.commissioned + assert northern.readings != southern.readings + assert northern.uri != southern.uri + + +def test_readings_are_stable(): + """Gold answers are derived from these, so a change to the derivation silently + rewrites every expected total. Pinned deliberately.""" + assert station("Kestrel", NORTHERN).readings_total == 689 + assert station("Auk", NORTHERN).readings_total == 756 + assert station("Auk", SOUTHERN).readings_total == 653 + + +def test_station_report_has_four_sections_and_twelve_rows(): + report = render_station_report(station("Kestrel", NORTHERN)) + for heading in ( + "## Overview", + "## Instruments", + "## Measurements", + "## Maintenance", + ): + assert heading in report + for month in MONTHS: + assert f"| {month} |" in report + + +def test_equipment_shares_vocabulary_but_answers_nothing(): + """The dilution probe only works if the sheets compete on wording while + holding no station facts.""" + sheets = documents_for(EQUIPMENT) + assert all("anemometer" in d.content for d in sheets) + assert all("calibration" in d.content.lower() for d in sheets) + station_names = {s.name for s in stations_in(NORTHERN)} | { + s.name for s in stations_in(SOUTHERN) + } + for sheet in sheets: + assert not any(f"Station {name}" in sheet.content for name in station_names) + assert "Programme" not in sheet.content + + +def test_every_station_is_reachable_by_name_and_database(): + for s in (*stations_in(NORTHERN), *stations_in(SOUTHERN)): + assert station(s.name, s.database) is s + with pytest.raises(KeyError): + station("Kestrel", SOUTHERN) + + +async def test_build_refuses_a_config_missing_a_database(): + """Building into a config that does not place all three would write a corpus + the run cannot read.""" + config = AppConfig( + lancedb=LanceDBConfig(databases={NORTHERN: "/tmp/n", SOUTHERN: "/tmp/s"}) + ) + with pytest.raises(ValueError, match="equipment"): + await build_databases(config)