From 3375dbff71cac11b257d90a52559a4773838fb51 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 31 Aug 2026 17:22:07 +0300 Subject: [PATCH] Report pooled composition at passage level The pooled builder printed gold-bearing titles via pool_composition, which counts nothing meaningful when cloud and fiqa have one empty title each: it reported 17,086 gold-bearing of 40,000 where the passage-level truth is 1,800 gold and 38,200 distractors. The build itself was correct; only the line was wrong. Claude-Session: https://claude.ai/code/session_01WhudUtZm6qqiuv8Y1sbwSc --- evaluations/evaluations/datasets/mtrag_federated.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/evaluations/evaluations/datasets/mtrag_federated.py b/evaluations/evaluations/datasets/mtrag_federated.py index 51b30a70..81a9db5b 100644 --- a/evaluations/evaluations/datasets/mtrag_federated.py +++ b/evaluations/evaluations/datasets/mtrag_federated.py @@ -470,10 +470,13 @@ async def build_pooled_databases( by_domain: dict[str, int] = {} for row in pool: by_domain[row["domain"]] = by_domain.get(row["domain"], 0) + 1 - gold_side, distractors = pool_composition(pool, pooled_gold_ids()) + # Passage level, not `pool_composition`: that counts gold-bearing titles, + # which is meaningless here since cloud and fiqa have one empty title each. + gold = pooled_gold_ids() + gold_kept = sum(1 for row in pool if row["_id"] in gold) print( - f"pool: {len(pool)} passages, {gold_side} in gold-bearing titles, " - f"{distractors} distractors, by domain {by_domain}" + f"pool: {len(pool)} passages, {gold_kept} gold, " + f"{len(pool) - gold_kept} distractors, by domain {by_domain}" ) grouped = partition_pooled(pool, n, alpha, seed) written: dict[str, int] = {}