From e653c49fce11bf309063a4d31d7de3fb385fc988 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 29 May 2026 14:54:02 +0300 Subject: [PATCH] Remove unecessary Mean Reciprocal Rank metric --- docs/benchmarks.md | 15 ++---- evaluations/README.md | 5 +- evaluations/evaluations/benchmark.py | 14 ++---- .../evaluations/evaluators/__init__.py | 8 +--- .../evaluations/evaluators/citation.py | 28 ++--------- evaluations/evaluations/evaluators/mrr.py | 37 -------------- evaluations/tests/test_benchmark.py | 11 +---- evaluations/tests/test_citation_evaluators.py | 40 +--------------- evaluations/tests/test_evaluators.py | 48 ------------------- 9 files changed, 16 insertions(+), 190 deletions(-) delete mode 100644 evaluations/evaluations/evaluators/mrr.py diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 2ef0fc81..dbe61a23 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -78,20 +78,13 @@ evaluations: ### Retrieval Metrics -**Mean Reciprocal Rank (MRR)** - Used when each query has exactly one relevant document. - -- For each query, find the rank (position) of the first relevant document in top-K results -- Reciprocal rank = `1/rank` (e.g., rank 3 → 1/3 ≈ 0.333) -- If not found in top-K, score is 0 -- MRR is the mean across all queries -- Range: 0 (never found) to 1 (always at rank 1) - -**Mean Average Precision (MAP)** - Used when queries have multiple relevant documents. +**Mean Average Precision (MAP)** scores ranked retrieval results against the gold `expected_uris`. - For each relevant document at position k, calculate precision@k = (relevant docs in top k) / k -- Average Precision (AP) = mean of these precision values / total relevant documents +- Average Precision (AP) = sum of these precision values / total relevant documents - MAP is the mean of AP scores across all queries - Range: 0 to 1. Rewards ranking relevant documents higher +- For single-doc queries this collapses to `1/rank` (i.e. reciprocal rank) ### QA Accuracy @@ -101,7 +94,7 @@ We picked `qwen3.6` over the previously-pinned `gpt-oss` after a 4-cell calibrat ### Citation Retrieval -Alongside QA accuracy, a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MRR / MAP math as raw retrieval. The score key is `cited_mrr` for single-doc datasets and `cited_map` for multi-doc. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case. +Alongside QA accuracy, a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MAP math as raw retrieval. The score key is `cited_map`. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case. This is computed alongside QA accuracy from the same skill run, no extra invocations. The signal complements raw retrieval: where raw retrieval measures whether the retriever surfaced the gold document at any rank, citation retrieval measures whether the skill grounded its answer on it. diff --git a/evaluations/README.md b/evaluations/README.md index 112bf9cf..78e59d42 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -52,9 +52,8 @@ evaluations run wix --target analysis-skill --skill-model ollama:gpt-oss `--skill-model "provider:name"` overrides the skill model independently from the judge (defaults to `qa.model`, or `analysis.model` when set for the -analysis-skill target). A citation retrieval metric (`cited_mrr` / `cited_map`) -is computed alongside QA accuracy from the URIs the skill registered via the -`cite` tool. +analysis-skill target). A citation retrieval metric (`cited_map`) is computed +alongside QA accuracy from the URIs the skill registered via the `cite` tool. ### Pre-built Databases diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 254e0205..0f58de66 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -19,9 +19,7 @@ from evaluations.datasets import DATASETS from evaluations.evaluators import ( ANSWER_EQUIVALENCE_RUBRIC, CitationMAPEvaluator, - CitationMRREvaluator, MAPEvaluator, - MRREvaluator, ) from evaluations.skill_runner import SkillFactory, run_skill_question from haiku.rag.client import HaikuRAG @@ -30,11 +28,6 @@ from haiku.rag.config.models import ModelConfig from haiku.rag.logging import configure_cli_logging from haiku.rag.utils import get_model, parse_model_option -_CITATION_EVALUATORS: dict[type[Evaluator], type[Evaluator]] = { - MRREvaluator: CitationMRREvaluator, - MAPEvaluator: CitationMAPEvaluator, -} - Target = Literal["rag-skill", "analysis-skill"] TARGETS: tuple[Target, ...] = ("rag-skill", "analysis-skill") @@ -306,10 +299,9 @@ def _skill_factory_for_target(target: Target) -> SkillFactory: def _citation_evaluator_for(retrieval_evaluator: Evaluator | None) -> Evaluator | None: """Return the citation-scoring twin of the dataset's retrieval evaluator.""" - if retrieval_evaluator is None: - return None - twin = _CITATION_EVALUATORS.get(type(retrieval_evaluator)) - return twin() if twin is not None else None + if isinstance(retrieval_evaluator, MAPEvaluator): + return CitationMAPEvaluator() + return None def _attach_relevant_uris( diff --git a/evaluations/evaluations/evaluators/__init__.py b/evaluations/evaluations/evaluators/__init__.py index 6a855a4f..bd15abc0 100644 --- a/evaluations/evaluations/evaluators/__init__.py +++ b/evaluations/evaluations/evaluators/__init__.py @@ -1,21 +1,15 @@ -from evaluations.evaluators.citation import ( - CitationMAPEvaluator, - CitationMRREvaluator, -) +from evaluations.evaluators.citation import CitationMAPEvaluator from evaluations.evaluators.judge import ( ANSWER_EQUIVALENCE_RUBRIC, LLMJudge, LLMJudgeResponseSchema, ) from evaluations.evaluators.map import MAPEvaluator -from evaluations.evaluators.mrr import MRREvaluator __all__ = [ "ANSWER_EQUIVALENCE_RUBRIC", "CitationMAPEvaluator", - "CitationMRREvaluator", "LLMJudge", "LLMJudgeResponseSchema", "MAPEvaluator", - "MRREvaluator", ] diff --git a/evaluations/evaluations/evaluators/citation.py b/evaluations/evaluations/evaluators/citation.py index b17a1b61..b16e9df5 100644 --- a/evaluations/evaluations/evaluators/citation.py +++ b/evaluations/evaluations/evaluators/citation.py @@ -13,35 +13,13 @@ def _relevant_uris(ctx: EvaluatorContext) -> set[str]: return set(ctx.metadata.get("relevant_uris", [])) -@dataclass -class CitationMRREvaluator(Evaluator): - """Reciprocal rank over the URIs the skill cited via the `cite` tool. - - Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run - via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from - ``ctx.metadata``. Returns ``1.0/rank`` of the first cited URI that is in - the relevant set, or ``0.0`` if none match. - - Use for single-document datasets, mirroring :class:`MRREvaluator`. - """ - - def get_default_evaluation_name(self) -> str: - return "cited_mrr" - - def evaluate(self, ctx: EvaluatorContext) -> float: - relevant = _relevant_uris(ctx) - for rank, uri in enumerate(_cited_uris(ctx), start=1): - if uri in relevant: - return 1.0 / rank - return 0.0 - - @dataclass class CitationMAPEvaluator(Evaluator): """Average precision over the URIs the skill cited via the `cite` tool. - Same input shape as :class:`CitationMRREvaluator`; use for multi-document - datasets, mirroring :class:`MAPEvaluator`. + Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run + via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from + ``ctx.metadata``. """ def get_default_evaluation_name(self) -> str: diff --git a/evaluations/evaluations/evaluators/mrr.py b/evaluations/evaluations/evaluators/mrr.py deleted file mode 100644 index decbc93f..00000000 --- a/evaluations/evaluations/evaluators/mrr.py +++ /dev/null @@ -1,37 +0,0 @@ -from dataclasses import dataclass - -from pydantic_evals.evaluators import Evaluator, EvaluatorContext - - -@dataclass -class MRREvaluator(Evaluator): - """ - Mean Reciprocal Rank evaluator for single-document retrieval. - - MRR = 1/rank where rank is the position of the first relevant document. - Returns 0 if no relevant document is found. - - Appropriate for retrieval tasks where each query has exactly one relevant document. - """ - - def evaluate(self, ctx: EvaluatorContext) -> float: - """ - Calculate reciprocal rank for a single query. - - Expected context: - - ctx.metadata['relevant_uris']: set/list of relevant document URIs - - ctx.output: list of retrieved document URIs (ordered by rank) - - Returns: - float: 1/rank of first relevant doc, or 0.0 if not found - """ - if ctx.metadata is None: - return 0.0 - relevant_uris = set(ctx.metadata.get("relevant_uris", [])) - retrieved_uris = ctx.output - - for rank, uri in enumerate(retrieved_uris, start=1): - if uri in relevant_uris: - return 1.0 / rank - - return 0.0 diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 030ff492..84b93d32 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -325,13 +325,6 @@ class TestRunQaBenchmarkSkillTarget: class TestCitationEvaluatorWiring: - def test_returns_mrr_twin_for_mrr_evaluator(self) -> None: - from evaluations.benchmark import _citation_evaluator_for - from evaluations.evaluators import CitationMRREvaluator, MRREvaluator - - result = _citation_evaluator_for(MRREvaluator()) - assert isinstance(result, CitationMRREvaluator) - def test_returns_map_twin_for_map_evaluator(self) -> None: from evaluations.benchmark import _citation_evaluator_for from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator @@ -351,7 +344,7 @@ class TestAttachRelevantUris: from evaluations.benchmark import _attach_relevant_uris from evaluations.config import RetrievalSample - from evaluations.evaluators import MRREvaluator + from evaluations.evaluators import MAPEvaluator cases: list[Case[str, str, dict]] = [ Case(name="c1", inputs="What is X?", expected_output="X is a thing"), @@ -382,7 +375,7 @@ class TestAttachRelevantUris: retrieval_mapper=lambda d: RetrievalSample( question=d["q"], expected_uris=d["uris"] ), - retrieval_evaluator=MRREvaluator(), + retrieval_evaluator=MAPEvaluator(), ) _attach_relevant_uris(cases, spec, limit=None) diff --git a/evaluations/tests/test_citation_evaluators.py b/evaluations/tests/test_citation_evaluators.py index c049e7e7..31d0d6be 100644 --- a/evaluations/tests/test_citation_evaluators.py +++ b/evaluations/tests/test_citation_evaluators.py @@ -1,9 +1,6 @@ from unittest.mock import MagicMock -from evaluations.evaluators.citation import ( - CitationMAPEvaluator, - CitationMRREvaluator, -) +from evaluations.evaluators.citation import CitationMAPEvaluator def _ctx(cited: list[str], relevant: list[str]) -> MagicMock: @@ -13,41 +10,6 @@ def _ctx(cited: list[str], relevant: list[str]) -> MagicMock: return ctx -class TestCitationMRREvaluator: - def setup_method(self) -> None: - self.evaluator = CitationMRREvaluator() - - def test_first_citation_is_relevant(self) -> None: - assert self.evaluator.evaluate(_ctx(["a", "b"], ["a"])) == 1.0 - - def test_second_citation_is_relevant(self) -> None: - assert self.evaluator.evaluate(_ctx(["a", "b"], ["b"])) == 0.5 - - def test_no_citations(self) -> None: - assert self.evaluator.evaluate(_ctx([], ["a"])) == 0.0 - - def test_no_relevant(self) -> None: - assert self.evaluator.evaluate(_ctx(["a"], [])) == 0.0 - - def test_no_matches(self) -> None: - assert self.evaluator.evaluate(_ctx(["a", "b"], ["c"])) == 0.0 - - def test_metadata_none(self) -> None: - ctx = MagicMock() - ctx.metadata = None - ctx.attributes = {"cited_uris": ["a"]} - assert self.evaluator.evaluate(ctx) == 0.0 - - def test_attribute_missing(self) -> None: - ctx = MagicMock() - ctx.metadata = {"relevant_uris": ["a"]} - ctx.attributes = {} - assert self.evaluator.evaluate(ctx) == 0.0 - - def test_evaluation_name(self) -> None: - assert self.evaluator.get_default_evaluation_name() == "cited_mrr" - - class TestCitationMAPEvaluator: def setup_method(self) -> None: self.evaluator = CitationMAPEvaluator() diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py index e2e98ca4..68bc34a6 100644 --- a/evaluations/tests/test_evaluators.py +++ b/evaluations/tests/test_evaluators.py @@ -3,54 +3,6 @@ from unittest.mock import MagicMock import pytest from evaluations.evaluators.map import MAPEvaluator -from evaluations.evaluators.mrr import MRREvaluator - - -class TestMRREvaluator: - def setup_method(self) -> None: - self.evaluator = MRREvaluator() - - def _make_ctx( - self, relevant_uris: list[str], retrieved_uris: list[str] - ) -> MagicMock: - ctx = MagicMock() - ctx.metadata = {"relevant_uris": relevant_uris} - ctx.output = retrieved_uris - return ctx - - def test_first_result_relevant(self) -> None: - ctx = self._make_ctx(["doc1"], ["doc1", "doc2", "doc3"]) - assert self.evaluator.evaluate(ctx) == 1.0 - - def test_second_result_relevant(self) -> None: - ctx = self._make_ctx(["doc2"], ["doc1", "doc2", "doc3"]) - assert self.evaluator.evaluate(ctx) == 0.5 - - def test_third_result_relevant(self) -> None: - ctx = self._make_ctx(["doc3"], ["doc1", "doc2", "doc3"]) - assert self.evaluator.evaluate(ctx) == pytest.approx(1 / 3) - - def test_no_relevant_found(self) -> None: - ctx = self._make_ctx(["doc_x"], ["doc1", "doc2", "doc3"]) - assert self.evaluator.evaluate(ctx) == 0.0 - - def test_empty_retrieved(self) -> None: - ctx = self._make_ctx(["doc1"], []) - assert self.evaluator.evaluate(ctx) == 0.0 - - def test_multiple_relevant_returns_first_match(self) -> None: - ctx = self._make_ctx(["doc2", "doc3"], ["doc1", "doc2", "doc3"]) - assert self.evaluator.evaluate(ctx) == 0.5 - - def test_none_metadata(self) -> None: - ctx = MagicMock() - ctx.metadata = None - ctx.output = ["doc1"] - assert self.evaluator.evaluate(ctx) == 0.0 - - def test_empty_relevant_uris(self) -> None: - ctx = self._make_ctx([], ["doc1", "doc2"]) - assert self.evaluator.evaluate(ctx) == 0.0 class TestMAPEvaluator: