Remove unecessary Mean Reciprocal Rank metric

docs/benchmarks.md

@@ -78,20 +78,13 @@ evaluations:
 
 ### Retrieval Metrics
 
-**Mean Reciprocal Rank (MRR)** - Used when each query has exactly one relevant document.
+**Mean Average Precision (MAP)** scores ranked retrieval results against the gold `expected_uris`.
-
-- For each query, find the rank (position) of the first relevant document in top-K results
-- Reciprocal rank = `1/rank` (e.g., rank 3 → 1/3 ≈ 0.333)
-- If not found in top-K, score is 0
-- MRR is the mean across all queries
-- Range: 0 (never found) to 1 (always at rank 1)
-
-**Mean Average Precision (MAP)** - Used when queries have multiple relevant documents.
 
 - For each relevant document at position k, calculate precision@k = (relevant docs in top k) / k
-- Average Precision (AP) = mean of these precision values / total relevant documents
+- Average Precision (AP) = sum of these precision values / total relevant documents
 - MAP is the mean of AP scores across all queries
 - Range: 0 to 1. Rewards ranking relevant documents higher
+- For single-doc queries this collapses to `1/rank` (i.e. reciprocal rank)
 
 ### QA Accuracy
 
@@ -101,7 +94,7 @@ We picked `qwen3.6` over the previously-pinned `gpt-oss` after a 4-cell calibrat
 
 ### Citation Retrieval
 
-Alongside QA accuracy, a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MRR / MAP math as raw retrieval. The score key is `cited_mrr` for single-doc datasets and `cited_map` for multi-doc. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case.
+Alongside QA accuracy, a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MAP math as raw retrieval. The score key is `cited_map`. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case.
 
 This is computed alongside QA accuracy from the same skill run, no extra invocations. The signal complements raw retrieval: where raw retrieval measures whether the retriever surfaced the gold document at any rank, citation retrieval measures whether the skill grounded its answer on it.
 

evaluations/README.md

@@ -52,9 +52,8 @@ evaluations run wix --target analysis-skill --skill-model ollama:gpt-oss
 
 `--skill-model "provider:name"` overrides the skill model independently from
 the judge (defaults to `qa.model`, or `analysis.model` when set for the
-analysis-skill target). A citation retrieval metric (`cited_mrr` / `cited_map`)
+analysis-skill target). A citation retrieval metric (`cited_map`) is computed
-is computed alongside QA accuracy from the URIs the skill registered via the
+alongside QA accuracy from the URIs the skill registered via the `cite` tool.
-`cite` tool.
 
 ### Pre-built Databases
 

evaluations/evaluations/benchmark.py

@@ -19,9 +19,7 @@ from evaluations.datasets import DATASETS
 from evaluations.evaluators import (
     ANSWER_EQUIVALENCE_RUBRIC,
     CitationMAPEvaluator,
-    CitationMRREvaluator,
     MAPEvaluator,
-    MRREvaluator,
 )
 from evaluations.skill_runner import SkillFactory, run_skill_question
 from haiku.rag.client import HaikuRAG
@@ -30,11 +28,6 @@ from haiku.rag.config.models import ModelConfig
 from haiku.rag.logging import configure_cli_logging
 from haiku.rag.utils import get_model, parse_model_option
 
-_CITATION_EVALUATORS: dict[type[Evaluator], type[Evaluator]] = {
-    MRREvaluator: CitationMRREvaluator,
-    MAPEvaluator: CitationMAPEvaluator,
-}
-
 Target = Literal["rag-skill", "analysis-skill"]
 TARGETS: tuple[Target, ...] = ("rag-skill", "analysis-skill")
 
@@ -306,10 +299,9 @@ def _skill_factory_for_target(target: Target) -> SkillFactory:
 
 def _citation_evaluator_for(retrieval_evaluator: Evaluator | None) -> Evaluator | None:
     """Return the citation-scoring twin of the dataset's retrieval evaluator."""
-    if retrieval_evaluator is None:
+    if isinstance(retrieval_evaluator, MAPEvaluator):
-        return None
+        return CitationMAPEvaluator()
-    twin = _CITATION_EVALUATORS.get(type(retrieval_evaluator))
+    return None
-    return twin() if twin is not None else None
 
 
 def _attach_relevant_uris(

evaluations/evaluations/evaluators/__init__.py

@@ -1,21 +1,15 @@
-from evaluations.evaluators.citation import (
+from evaluations.evaluators.citation import CitationMAPEvaluator
-    CitationMAPEvaluator,
-    CitationMRREvaluator,
-)
 from evaluations.evaluators.judge import (
     ANSWER_EQUIVALENCE_RUBRIC,
     LLMJudge,
     LLMJudgeResponseSchema,
 )
 from evaluations.evaluators.map import MAPEvaluator
-from evaluations.evaluators.mrr import MRREvaluator
 
 __all__ = [
     "ANSWER_EQUIVALENCE_RUBRIC",
     "CitationMAPEvaluator",
-    "CitationMRREvaluator",
     "LLMJudge",
     "LLMJudgeResponseSchema",
     "MAPEvaluator",
-    "MRREvaluator",
 ]

evaluations/evaluations/evaluators/citation.py

@@ -13,35 +13,13 @@ def _relevant_uris(ctx: EvaluatorContext) -> set[str]:
     return set(ctx.metadata.get("relevant_uris", []))
 
 
-@dataclass
-class CitationMRREvaluator(Evaluator):
-    """Reciprocal rank over the URIs the skill cited via the `cite` tool.
-
-    Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run
-    via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from
-    ``ctx.metadata``. Returns ``1.0/rank`` of the first cited URI that is in
-    the relevant set, or ``0.0`` if none match.
-
-    Use for single-document datasets, mirroring :class:`MRREvaluator`.
-    """
-
-    def get_default_evaluation_name(self) -> str:
-        return "cited_mrr"
-
-    def evaluate(self, ctx: EvaluatorContext) -> float:
-        relevant = _relevant_uris(ctx)
-        for rank, uri in enumerate(_cited_uris(ctx), start=1):
-            if uri in relevant:
-                return 1.0 / rank
-        return 0.0
-
-
 @dataclass
 class CitationMAPEvaluator(Evaluator):
     """Average precision over the URIs the skill cited via the `cite` tool.
 
-    Same input shape as :class:`CitationMRREvaluator`; use for multi-document
+    Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run
-    datasets, mirroring :class:`MAPEvaluator`.
+    via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from
+    ``ctx.metadata``.
     """
 
     def get_default_evaluation_name(self) -> str:

evaluations/evaluations/evaluators/mrr.py

@@ -1,37 +0,0 @@
-from dataclasses import dataclass
-
-from pydantic_evals.evaluators import Evaluator, EvaluatorContext
-
-
-@dataclass
-class MRREvaluator(Evaluator):
-    """
-    Mean Reciprocal Rank evaluator for single-document retrieval.
-
-    MRR = 1/rank where rank is the position of the first relevant document.
-    Returns 0 if no relevant document is found.
-
-    Appropriate for retrieval tasks where each query has exactly one relevant document.
-    """
-
-    def evaluate(self, ctx: EvaluatorContext) -> float:
-        """
-        Calculate reciprocal rank for a single query.
-
-        Expected context:
-        - ctx.metadata['relevant_uris']: set/list of relevant document URIs
-        - ctx.output: list of retrieved document URIs (ordered by rank)
-
-        Returns:
-            float: 1/rank of first relevant doc, or 0.0 if not found
-        """
-        if ctx.metadata is None:
-            return 0.0
-        relevant_uris = set(ctx.metadata.get("relevant_uris", []))
-        retrieved_uris = ctx.output
-
-        for rank, uri in enumerate(retrieved_uris, start=1):
-            if uri in relevant_uris:
-                return 1.0 / rank
-
-        return 0.0

evaluations/tests/test_benchmark.py

@@ -325,13 +325,6 @@ class TestRunQaBenchmarkSkillTarget:
 
 
 class TestCitationEvaluatorWiring:
-    def test_returns_mrr_twin_for_mrr_evaluator(self) -> None:
-        from evaluations.benchmark import _citation_evaluator_for
-        from evaluations.evaluators import CitationMRREvaluator, MRREvaluator
-
-        result = _citation_evaluator_for(MRREvaluator())
-        assert isinstance(result, CitationMRREvaluator)
-
     def test_returns_map_twin_for_map_evaluator(self) -> None:
         from evaluations.benchmark import _citation_evaluator_for
         from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator
@@ -351,7 +344,7 @@ class TestAttachRelevantUris:
 
         from evaluations.benchmark import _attach_relevant_uris
         from evaluations.config import RetrievalSample
-        from evaluations.evaluators import MRREvaluator
+        from evaluations.evaluators import MAPEvaluator
 
         cases: list[Case[str, str, dict]] = [
             Case(name="c1", inputs="What is X?", expected_output="X is a thing"),
@@ -382,7 +375,7 @@ class TestAttachRelevantUris:
             retrieval_mapper=lambda d: RetrievalSample(
                 question=d["q"], expected_uris=d["uris"]
             ),
-            retrieval_evaluator=MRREvaluator(),
+            retrieval_evaluator=MAPEvaluator(),
         )
 
         _attach_relevant_uris(cases, spec, limit=None)

evaluations/tests/test_citation_evaluators.py

@@ -1,9 +1,6 @@
 from unittest.mock import MagicMock
 
-from evaluations.evaluators.citation import (
+from evaluations.evaluators.citation import CitationMAPEvaluator
-    CitationMAPEvaluator,
-    CitationMRREvaluator,
-)
 
 
 def _ctx(cited: list[str], relevant: list[str]) -> MagicMock:
@@ -13,41 +10,6 @@ def _ctx(cited: list[str], relevant: list[str]) -> MagicMock:
     return ctx
 
 
-class TestCitationMRREvaluator:
-    def setup_method(self) -> None:
-        self.evaluator = CitationMRREvaluator()
-
-    def test_first_citation_is_relevant(self) -> None:
-        assert self.evaluator.evaluate(_ctx(["a", "b"], ["a"])) == 1.0
-
-    def test_second_citation_is_relevant(self) -> None:
-        assert self.evaluator.evaluate(_ctx(["a", "b"], ["b"])) == 0.5
-
-    def test_no_citations(self) -> None:
-        assert self.evaluator.evaluate(_ctx([], ["a"])) == 0.0
-
-    def test_no_relevant(self) -> None:
-        assert self.evaluator.evaluate(_ctx(["a"], [])) == 0.0
-
-    def test_no_matches(self) -> None:
-        assert self.evaluator.evaluate(_ctx(["a", "b"], ["c"])) == 0.0
-
-    def test_metadata_none(self) -> None:
-        ctx = MagicMock()
-        ctx.metadata = None
-        ctx.attributes = {"cited_uris": ["a"]}
-        assert self.evaluator.evaluate(ctx) == 0.0
-
-    def test_attribute_missing(self) -> None:
-        ctx = MagicMock()
-        ctx.metadata = {"relevant_uris": ["a"]}
-        ctx.attributes = {}
-        assert self.evaluator.evaluate(ctx) == 0.0
-
-    def test_evaluation_name(self) -> None:
-        assert self.evaluator.get_default_evaluation_name() == "cited_mrr"
-
-
 class TestCitationMAPEvaluator:
     def setup_method(self) -> None:
         self.evaluator = CitationMAPEvaluator()

evaluations/tests/test_evaluators.py

@@ -3,54 +3,6 @@ from unittest.mock import MagicMock
 import pytest
 
 from evaluations.evaluators.map import MAPEvaluator
-from evaluations.evaluators.mrr import MRREvaluator
-
-
-class TestMRREvaluator:
-    def setup_method(self) -> None:
-        self.evaluator = MRREvaluator()
-
-    def _make_ctx(
-        self, relevant_uris: list[str], retrieved_uris: list[str]
-    ) -> MagicMock:
-        ctx = MagicMock()
-        ctx.metadata = {"relevant_uris": relevant_uris}
-        ctx.output = retrieved_uris
-        return ctx
-
-    def test_first_result_relevant(self) -> None:
-        ctx = self._make_ctx(["doc1"], ["doc1", "doc2", "doc3"])
-        assert self.evaluator.evaluate(ctx) == 1.0
-
-    def test_second_result_relevant(self) -> None:
-        ctx = self._make_ctx(["doc2"], ["doc1", "doc2", "doc3"])
-        assert self.evaluator.evaluate(ctx) == 0.5
-
-    def test_third_result_relevant(self) -> None:
-        ctx = self._make_ctx(["doc3"], ["doc1", "doc2", "doc3"])
-        assert self.evaluator.evaluate(ctx) == pytest.approx(1 / 3)
-
-    def test_no_relevant_found(self) -> None:
-        ctx = self._make_ctx(["doc_x"], ["doc1", "doc2", "doc3"])
-        assert self.evaluator.evaluate(ctx) == 0.0
-
-    def test_empty_retrieved(self) -> None:
-        ctx = self._make_ctx(["doc1"], [])
-        assert self.evaluator.evaluate(ctx) == 0.0
-
-    def test_multiple_relevant_returns_first_match(self) -> None:
-        ctx = self._make_ctx(["doc2", "doc3"], ["doc1", "doc2", "doc3"])
-        assert self.evaluator.evaluate(ctx) == 0.5
-
-    def test_none_metadata(self) -> None:
-        ctx = MagicMock()
-        ctx.metadata = None
-        ctx.output = ["doc1"]
-        assert self.evaluator.evaluate(ctx) == 0.0
-
-    def test_empty_relevant_uris(self) -> None:
-        ctx = self._make_ctx([], ["doc1", "doc2"])
-        assert self.evaluator.evaluate(ctx) == 0.0
 
 
 class TestMAPEvaluator: