danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove unecessary Mean Reciprocal Rank metric

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-05-29 14:54:02 +03:00
parent 00f2a40a60
commit e653c49fce
No known key found for this signature in database
9 changed files with 16 additions and 190 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
docs/benchmarks.md
View file

@ -78,20 +78,13 @@ evaluations:
### Retrieval Metrics
**Mean Reciprocal Rank (MRR)** - Used when each query has exactly one relevant document.
- For each query, find the rank (position) of the first relevant document in top-K results
- Reciprocal rank = `1/rank` (e.g., rank 3 → 1/3 ≈ 0.333)
- If not found in top-K, score is 0
- MRR is the mean across all queries
- Range: 0 (never found) to 1 (always at rank 1)
**Mean Average Precision (MAP)** - Used when queries have multiple relevant documents.
**Mean Average Precision (MAP)** scores ranked retrieval results against the gold `expected_uris`.
- For each relevant document at position k, calculate precision@k = (relevant docs in top k) / k
- Average Precision (AP) = mean of these precision values / total relevant documents
- Average Precision (AP) = sum of these precision values / total relevant documents
- MAP is the mean of AP scores across all queries
- Range: 0 to 1. Rewards ranking relevant documents higher
- For single-doc queries this collapses to `1/rank` (i.e. reciprocal rank)
### QA Accuracy
@ -101,7 +94,7 @@ We picked `qwen3.6` over the previously-pinned `gpt-oss` after a 4-cell calibrat
### Citation Retrieval
Alongside QA accuracy, a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MRR / MAP math as raw retrieval. The score key is `cited_mrr` for single-doc datasets and `cited_map` for multi-doc. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case.
Alongside QA accuracy, a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MAP math as raw retrieval. The score key is `cited_map`. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case.
This is computed alongside QA accuracy from the same skill run, no extra invocations. The signal complements raw retrieval: where raw retrieval measures whether the retriever surfaced the gold document at any rank, citation retrieval measures whether the skill grounded its answer on it.

5
evaluations/README.md
View file

@ -52,9 +52,8 @@ evaluations run wix --target analysis-skill --skill-model ollama:gpt-oss
`--skill-model "provider:name"` overrides the skill model independently from
the judge (defaults to `qa.model`, or `analysis.model` when set for the
analysis-skill target). A citation retrieval metric (`cited_mrr` / `cited_map`)
is computed alongside QA accuracy from the URIs the skill registered via the
`cite` tool.
analysis-skill target). A citation retrieval metric (`cited_map`) is computed
alongside QA accuracy from the URIs the skill registered via the `cite` tool.
### Pre-built Databases

14
evaluations/evaluations/benchmark.py
View file

@ -19,9 +19,7 @@ from evaluations.datasets import DATASETS
from evaluations.evaluators import (
ANSWER_EQUIVALENCE_RUBRIC,
CitationMAPEvaluator,
CitationMRREvaluator,
MAPEvaluator,
MRREvaluator,
)
from evaluations.skill_runner import SkillFactory, run_skill_question
from haiku.rag.client import HaikuRAG
@ -30,11 +28,6 @@ from haiku.rag.config.models import ModelConfig
from haiku.rag.logging import configure_cli_logging
from haiku.rag.utils import get_model, parse_model_option
_CITATION_EVALUATORS: dict[type[Evaluator], type[Evaluator]] = {
MRREvaluator: CitationMRREvaluator,
MAPEvaluator: CitationMAPEvaluator,
}
Target = Literal["rag-skill", "analysis-skill"]
TARGETS: tuple[Target, ...] = ("rag-skill", "analysis-skill")
@ -306,10 +299,9 @@ def _skill_factory_for_target(target: Target) -> SkillFactory:
def _citation_evaluator_for(retrieval_evaluator: Evaluator | None) -> Evaluator | None:
"""Return the citation-scoring twin of the dataset's retrieval evaluator."""
if retrieval_evaluator is None:
return None
twin = _CITATION_EVALUATORS.get(type(retrieval_evaluator))
return twin() if twin is not None else None
if isinstance(retrieval_evaluator, MAPEvaluator):
return CitationMAPEvaluator()
return None
def _attach_relevant_uris(

8
evaluations/evaluations/evaluators/__init__.py
View file

@ -1,21 +1,15 @@
from evaluations.evaluators.citation import (
CitationMAPEvaluator,
CitationMRREvaluator,
)
from evaluations.evaluators.citation import CitationMAPEvaluator
from evaluations.evaluators.judge import (
ANSWER_EQUIVALENCE_RUBRIC,
LLMJudge,
LLMJudgeResponseSchema,
)
from evaluations.evaluators.map import MAPEvaluator
from evaluations.evaluators.mrr import MRREvaluator
__all__ = [
"ANSWER_EQUIVALENCE_RUBRIC",
"CitationMAPEvaluator",
"CitationMRREvaluator",
"LLMJudge",
"LLMJudgeResponseSchema",
"MAPEvaluator",
"MRREvaluator",
]

28
evaluations/evaluations/evaluators/citation.py
View file

@ -13,35 +13,13 @@ def _relevant_uris(ctx: EvaluatorContext) -> set[str]:
return set(ctx.metadata.get("relevant_uris", []))
@dataclass
class CitationMRREvaluator(Evaluator):
"""Reciprocal rank over the URIs the skill cited via the `cite` tool.
Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run
via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from
``ctx.metadata``. Returns ``1.0/rank`` of the first cited URI that is in
the relevant set, or ``0.0`` if none match.
Use for single-document datasets, mirroring :class:`MRREvaluator`.
"""
def get_default_evaluation_name(self) -> str:
return "cited_mrr"
def evaluate(self, ctx: EvaluatorContext) -> float:
relevant = _relevant_uris(ctx)
for rank, uri in enumerate(_cited_uris(ctx), start=1):
if uri in relevant:
return 1.0 / rank
return 0.0
@dataclass
class CitationMAPEvaluator(Evaluator):
"""Average precision over the URIs the skill cited via the `cite` tool.
Same input shape as :class:`CitationMRREvaluator`; use for multi-document
datasets, mirroring :class:`MAPEvaluator`.
Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run
via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from
``ctx.metadata``.
"""
def get_default_evaluation_name(self) -> str:

37
evaluations/evaluations/evaluators/mrr.py
View file

@ -1,37 +0,0 @@
from dataclasses import dataclass
from pydantic_evals.evaluators import Evaluator, EvaluatorContext
@dataclass
class MRREvaluator(Evaluator):
"""
Mean Reciprocal Rank evaluator for single-document retrieval.
MRR = 1/rank where rank is the position of the first relevant document.
Returns 0 if no relevant document is found.
Appropriate for retrieval tasks where each query has exactly one relevant document.
"""
def evaluate(self, ctx: EvaluatorContext) -> float:
"""
Calculate reciprocal rank for a single query.
Expected context:
- ctx.metadata['relevant_uris']: set/list of relevant document URIs
- ctx.output: list of retrieved document URIs (ordered by rank)
Returns:
float: 1/rank of first relevant doc, or 0.0 if not found
"""
if ctx.metadata is None:
return 0.0
relevant_uris = set(ctx.metadata.get("relevant_uris", []))
retrieved_uris = ctx.output
for rank, uri in enumerate(retrieved_uris, start=1):
if uri in relevant_uris:
return 1.0 / rank
return 0.0

11
evaluations/tests/test_benchmark.py
View file

@ -325,13 +325,6 @@ class TestRunQaBenchmarkSkillTarget:
class TestCitationEvaluatorWiring:
def test_returns_mrr_twin_for_mrr_evaluator(self) -> None:
from evaluations.benchmark import _citation_evaluator_for
from evaluations.evaluators import CitationMRREvaluator, MRREvaluator
result = _citation_evaluator_for(MRREvaluator())
assert isinstance(result, CitationMRREvaluator)
def test_returns_map_twin_for_map_evaluator(self) -> None:
from evaluations.benchmark import _citation_evaluator_for
from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator
@ -351,7 +344,7 @@ class TestAttachRelevantUris:
from evaluations.benchmark import _attach_relevant_uris
from evaluations.config import RetrievalSample
from evaluations.evaluators import MRREvaluator
from evaluations.evaluators import MAPEvaluator
cases: list[Case[str, str, dict]] = [
Case(name="c1", inputs="What is X?", expected_output="X is a thing"),
@ -382,7 +375,7 @@ class TestAttachRelevantUris:
retrieval_mapper=lambda d: RetrievalSample(
question=d["q"], expected_uris=d["uris"]
),
retrieval_evaluator=MRREvaluator(),
retrieval_evaluator=MAPEvaluator(),
)
_attach_relevant_uris(cases, spec, limit=None)

40
evaluations/tests/test_citation_evaluators.py
View file

@ -1,9 +1,6 @@
from unittest.mock import MagicMock
from evaluations.evaluators.citation import (
CitationMAPEvaluator,
CitationMRREvaluator,
)
from evaluations.evaluators.citation import CitationMAPEvaluator
def _ctx(cited: list[str], relevant: list[str]) -> MagicMock:
@ -13,41 +10,6 @@ def _ctx(cited: list[str], relevant: list[str]) -> MagicMock:
return ctx
class TestCitationMRREvaluator:
def setup_method(self) -> None:
self.evaluator = CitationMRREvaluator()
def test_first_citation_is_relevant(self) -> None:
assert self.evaluator.evaluate(_ctx(["a", "b"], ["a"])) == 1.0
def test_second_citation_is_relevant(self) -> None:
assert self.evaluator.evaluate(_ctx(["a", "b"], ["b"])) == 0.5
def test_no_citations(self) -> None:
assert self.evaluator.evaluate(_ctx([], ["a"])) == 0.0
def test_no_relevant(self) -> None:
assert self.evaluator.evaluate(_ctx(["a"], [])) == 0.0
def test_no_matches(self) -> None:
assert self.evaluator.evaluate(_ctx(["a", "b"], ["c"])) == 0.0
def test_metadata_none(self) -> None:
ctx = MagicMock()
ctx.metadata = None
ctx.attributes = {"cited_uris": ["a"]}
assert self.evaluator.evaluate(ctx) == 0.0
def test_attribute_missing(self) -> None:
ctx = MagicMock()
ctx.metadata = {"relevant_uris": ["a"]}
ctx.attributes = {}
assert self.evaluator.evaluate(ctx) == 0.0
def test_evaluation_name(self) -> None:
assert self.evaluator.get_default_evaluation_name() == "cited_mrr"
class TestCitationMAPEvaluator:
def setup_method(self) -> None:
self.evaluator = CitationMAPEvaluator()

48
evaluations/tests/test_evaluators.py
View file

@ -3,54 +3,6 @@ from unittest.mock import MagicMock
import pytest
from evaluations.evaluators.map import MAPEvaluator
from evaluations.evaluators.mrr import MRREvaluator
class TestMRREvaluator:
def setup_method(self) -> None:
self.evaluator = MRREvaluator()
def _make_ctx(
self, relevant_uris: list[str], retrieved_uris: list[str]
) -> MagicMock:
ctx = MagicMock()
ctx.metadata = {"relevant_uris": relevant_uris}
ctx.output = retrieved_uris
return ctx
def test_first_result_relevant(self) -> None:
ctx = self._make_ctx(["doc1"], ["doc1", "doc2", "doc3"])
assert self.evaluator.evaluate(ctx) == 1.0
def test_second_result_relevant(self) -> None:
ctx = self._make_ctx(["doc2"], ["doc1", "doc2", "doc3"])
assert self.evaluator.evaluate(ctx) == 0.5
def test_third_result_relevant(self) -> None:
ctx = self._make_ctx(["doc3"], ["doc1", "doc2", "doc3"])
assert self.evaluator.evaluate(ctx) == pytest.approx(1 / 3)
def test_no_relevant_found(self) -> None:
ctx = self._make_ctx(["doc_x"], ["doc1", "doc2", "doc3"])
assert self.evaluator.evaluate(ctx) == 0.0
def test_empty_retrieved(self) -> None:
ctx = self._make_ctx(["doc1"], [])
assert self.evaluator.evaluate(ctx) == 0.0
def test_multiple_relevant_returns_first_match(self) -> None:
ctx = self._make_ctx(["doc2", "doc3"], ["doc1", "doc2", "doc3"])
assert self.evaluator.evaluate(ctx) == 0.5
def test_none_metadata(self) -> None:
ctx = MagicMock()
ctx.metadata = None
ctx.output = ["doc1"]
assert self.evaluator.evaluate(ctx) == 0.0
def test_empty_relevant_uris(self) -> None:
ctx = self._make_ctx([], ["doc1", "doc2"])
assert self.evaluator.evaluate(ctx) == 0.0
class TestMAPEvaluator: