Exclude fully unjudged conversations from the macro pass rate

evaluations/evaluations/benchmark.py

@@ -452,6 +452,10 @@ def _live_summary(report_cases, report_failures) -> dict[str, float | int] | Non
     turns_total = sum(_score(case, "turns_total") for case in scored)
     turns_judged = sum(_score(case, "turns_judged") for case in scored)
     turns_passed = sum(_score(case, "turns_passed") for case in scored)
+    # A conversation with zero judged turns (its judge calls all failed)
+    # reports turn_pass_rate 0.0; averaging that in would count a judge
+    # outage as a failed conversation, against the exclusion policy.
+    judged = [case for case in scored if _score(case, "turns_judged")]
     summary: dict[str, float | int] = {
         "conversations": len(scored),
         "conversations_attempted": len(report_cases) + len(report_failures),
@@ -459,8 +463,10 @@ def _live_summary(report_cases, report_failures) -> dict[str, float | int] | Non
         "turns_judged": turns_judged,
         "turns_attempted": turns_total + failed_turns,
         "micro_pass_rate": turns_passed / turns_judged if turns_judged else 0.0,
-        "macro_pass_rate": sum(_score(case, "turn_pass_rate") for case in scored)
+        "macro_pass_rate": sum(_score(case, "turn_pass_rate") for case in judged)
-        / len(scored),
+        / len(judged)
+        if judged
+        else 0.0,
     }
 
     cited = [case for case in scored if _score(case, "cited_map") is not None]

evaluations/tests/test_benchmark.py

@@ -359,6 +359,47 @@ class TestLiveSummary:
         assert summary["turns_judged"] == 3
         assert summary["turns_total"] == 4
 
+    def test_macro_rate_excludes_fully_unjudged_conversations(self) -> None:
+        """A conversation whose every turn lost its judge reports
+        turn_pass_rate 0.0; treating that as a failed conversation would
+        contradict the exclusion policy. It must not enter the macro average."""
+        from evaluations.benchmark import _live_summary
+
+        cases = [
+            self._case(
+                {
+                    "turn_pass_rate": 1.0,
+                    "turns_passed": 2,
+                    "turns_judged": 2,
+                    "turns_total": 2,
+                    "cited_eligible": 0,
+                    "true_refusals": 0,
+                    "false_refusals": 0,
+                    "unanswerable_turns": 0,
+                }
+            ),
+            self._case(
+                {
+                    "turn_pass_rate": 0.0,
+                    "turns_passed": 0,
+                    "turns_judged": 0,  # total judge outage for this conversation
+                    "turns_total": 8,
+                    "cited_eligible": 0,
+                    "true_refusals": 0,
+                    "false_refusals": 0,
+                    "unanswerable_turns": 0,
+                }
+            ),
+        ]
+
+        summary = _live_summary(cases, [])
+
+        assert summary is not None
+        assert summary["macro_pass_rate"] == pytest.approx(1.0)
+        assert summary["micro_pass_rate"] == pytest.approx(1.0)
+        assert summary["turns_judged"] == 2
+        assert summary["turns_total"] == 10
+
     def test_failed_conversations_do_not_affect_rates(self) -> None:
         from evaluations.benchmark import _live_summary