danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Exclude fully unjudged conversations from the macro pass rate

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-08-17 11:19:59 +03:00
parent 587ba75a62
commit 73dfc62f33
No known key found for this signature in database
2 changed files with 49 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
evaluations/evaluations/benchmark.py
View file

@ -452,6 +452,10 @@ def _live_summary(report_cases, report_failures) -> dict[str, float | int] | Non
turns_total = sum(_score(case, "turns_total") for case in scored)
turns_judged = sum(_score(case, "turns_judged") for case in scored)
turns_passed = sum(_score(case, "turns_passed") for case in scored)
# A conversation with zero judged turns (its judge calls all failed)
# reports turn_pass_rate 0.0; averaging that in would count a judge
# outage as a failed conversation, against the exclusion policy.
judged = [case for case in scored if _score(case, "turns_judged")]
summary: dict[str, float | int] = {
"conversations": len(scored),
"conversations_attempted": len(report_cases) + len(report_failures),
@ -459,8 +463,10 @@ def _live_summary(report_cases, report_failures) -> dict[str, float | int] | Non
"turns_judged": turns_judged,
"turns_attempted": turns_total + failed_turns,
"micro_pass_rate": turns_passed / turns_judged if turns_judged else 0.0,
"macro_pass_rate": sum(_score(case, "turn_pass_rate") for case in scored)
/ len(scored),
"macro_pass_rate": sum(_score(case, "turn_pass_rate") for case in judged)
/ len(judged)
if judged
else 0.0,
}
cited = [case for case in scored if _score(case, "cited_map") is not None]

41
evaluations/tests/test_benchmark.py
View file

@ -359,6 +359,47 @@ class TestLiveSummary:
assert summary["turns_judged"] == 3
assert summary["turns_total"] == 4
def test_macro_rate_excludes_fully_unjudged_conversations(self) -> None:
"""A conversation whose every turn lost its judge reports
turn_pass_rate 0.0; treating that as a failed conversation would
contradict the exclusion policy. It must not enter the macro average."""
from evaluations.benchmark import _live_summary
cases = [
self._case(
{
"turn_pass_rate": 1.0,
"turns_passed": 2,
"turns_judged": 2,
"turns_total": 2,
"cited_eligible": 0,
"true_refusals": 0,
"false_refusals": 0,
"unanswerable_turns": 0,
}
),
self._case(
{
"turn_pass_rate": 0.0,
"turns_passed": 0,
"turns_judged": 0, # total judge outage for this conversation
"turns_total": 8,
"cited_eligible": 0,
"true_refusals": 0,
"false_refusals": 0,
"unanswerable_turns": 0,
}
),
]
summary = _live_summary(cases, [])
assert summary is not None
assert summary["macro_pass_rate"] == pytest.approx(1.0)
assert summary["micro_pass_rate"] == pytest.approx(1.0)
assert summary["turns_judged"] == 2
assert summary["turns_total"] == 10
def test_failed_conversations_do_not_affect_rates(self) -> None:
from evaluations.benchmark import _live_summary