18e1127375
The two hard gates are pass/fail, not rates: any scope leak or attribution error is a bug. The report names the offending cases and prints a greppable GATES: PASSED / FAILED line, since this dataset is an acceptance gate rather than a number to watch drift on. The attribution gate covers only families where exactly one database is correct. B5 is answered by two, so citing both is right there and policing it would report a bug that is not one. Surface coverage comes from the model's own Python, now carried on the run result: counting executions said how often code ran but never which surface it reached. Coverage is measured, not required, so a surface nothing reaches is a finding about whether it earns its place in the VFS. answer_correct is recorded as a score rather than a boolean, since booleans become assertions and the run reads its accuracy headline from scores. The two gates stay boolean assertions, which is what they are. DatasetSpec grows a report_hook so this lives with the dataset instead of becoming a key check in the shared QA reporting. The reference-config invariant said a dataset with a deterministic evaluator must declare no judge. That is not true here: RefusalJudge runs on any case carrying an answerability label, which B6 and B7 do, so its sampling has to be pinned. The assertion now requires a pinned block wherever a judge runs and allows its absence only as the claim that none does.
127 lines
3.9 KiB
Python
127 lines
3.9 KiB
Python
from collections.abc import Callable, Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Literal
|
|
|
|
from datasets import Dataset
|
|
from pydantic import BaseModel, model_validator
|
|
from pydantic_evals import Case
|
|
from pydantic_evals.evaluators import Evaluator
|
|
|
|
|
|
class Turn(BaseModel):
|
|
speaker: Literal["user", "agent"]
|
|
text: str
|
|
|
|
|
|
class ConversationInput(BaseModel):
|
|
"""A conversation prefix plus the final user question (the last turn)."""
|
|
|
|
turns: list[Turn]
|
|
|
|
@model_validator(mode="after")
|
|
def _ends_with_user_turn(self) -> "ConversationInput":
|
|
if not self.turns or self.turns[-1].speaker != "user":
|
|
raise ValueError("conversation must end with a user turn")
|
|
return self
|
|
|
|
@property
|
|
def question(self) -> str:
|
|
return self.turns[-1].text
|
|
|
|
@property
|
|
def prefix(self) -> list[Turn]:
|
|
return self.turns[:-1]
|
|
|
|
@property
|
|
def transcript(self) -> str:
|
|
return "\n".join(f"{turn.speaker}: {turn.text}" for turn in self.turns)
|
|
|
|
|
|
class ScopedQuestion(BaseModel):
|
|
"""A question and the databases it may draw on.
|
|
|
|
The task function receives only a case's inputs, never its metadata, so a
|
|
per-case scope has to travel in the inputs. `sources=[]` covers no database
|
|
and `None` covers every one the client covers.
|
|
"""
|
|
|
|
question: str
|
|
sources: list[str] | None = None
|
|
|
|
|
|
@dataclass
|
|
class DocumentPayload:
|
|
uri: str
|
|
content: str | None = None
|
|
title: str | None = None
|
|
metadata: dict[str, Any] | None = None
|
|
format: str = "md"
|
|
source_path: Path | None = None
|
|
|
|
|
|
@dataclass
|
|
class RetrievalSample:
|
|
question: str
|
|
expected_uris: tuple[str, ...]
|
|
skip: bool = False
|
|
source_type: str | None = None
|
|
|
|
|
|
DocumentLoader = Callable[[], Dataset]
|
|
DocumentMapper = Callable[[Mapping[str, Any]], DocumentPayload | None]
|
|
RetrievalLoader = Callable[[], Dataset]
|
|
RetrievalMapper = Callable[[Mapping[str, Any]], RetrievalSample | None]
|
|
CaseBuilder = Callable[[int, Mapping[str, Any]], Case[Any, Any, dict[str, Any]]]
|
|
|
|
|
|
@dataclass
|
|
class DatasetSpec:
|
|
key: str
|
|
db_filename: str
|
|
document_loader: DocumentLoader
|
|
document_mapper: DocumentMapper
|
|
qa_loader: DocumentLoader
|
|
qa_case_builder: CaseBuilder
|
|
retrieval_loader: RetrievalLoader | None = None
|
|
retrieval_mapper: RetrievalMapper | None = None
|
|
retrieval_evaluators: list[Evaluator] | None = None
|
|
citation_evaluator: Evaluator | None = None
|
|
qa_evaluator: Evaluator | None = None
|
|
document_limit: int | None = None
|
|
retrieval_limit: int = 5
|
|
ingest_batch_size: int | None = None
|
|
live: bool = False
|
|
compaction: bool = False
|
|
experiment_metadata: dict[str, Any] | None = None
|
|
# Called with the report's cases after the run prints, for datasets that
|
|
# report something the shared summary cannot express (e.g. hard gates).
|
|
report_hook: Callable[[list[Any]], None] | None = None
|
|
|
|
def uses_configured_databases(
|
|
self, config, override_path: Path | None = None
|
|
) -> bool:
|
|
"""Whether `lancedb.databases` places the databases to evaluate over.
|
|
|
|
A path names one database and wins over the configuration, both when it
|
|
comes from `--db` and when the client resolves it. True for a mapping of
|
|
one, which is a configured database like any other and keeps its name.
|
|
"""
|
|
return bool(config.lancedb.databases) and override_path is None
|
|
|
|
def db_path(self, override_path: Path | None = None) -> Path:
|
|
"""Get the database path.
|
|
|
|
Args:
|
|
override_path: Optional path to override the default database location.
|
|
|
|
Returns:
|
|
The database path to use.
|
|
"""
|
|
if override_path is not None:
|
|
return override_path
|
|
|
|
from haiku.rag.utils import get_default_data_dir
|
|
|
|
data_dir = get_default_data_dir()
|
|
return data_dir / "evaluations" / "dbs" / self.db_filename
|