From ad06d6a4977c1b66f203645b225cde9124f85dd8 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 26 Aug 2026 15:40:29 +0300 Subject: [PATCH] Stop failing a correct answer for naming the other document Both S6 cases were scored wrong while answering correctly: the model gave the right uri and database, then added a note distinguishing the near-name document, and the forbidden-distractor rule rejected it for mentioning the twin's uri at all. Naming the other document as a different document is correct behaviour, and better than the minimum. Forbidding a distractor belongs on B3, where two elevations cannot both be the answer; here it punished disambiguation. S6 now requires the uri and then the database that holds it. --- evaluations/evaluations/datasets/multidb.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/evaluations/evaluations/datasets/multidb.py b/evaluations/evaluations/datasets/multidb.py index 7b772478..4a8437cb 100644 --- a/evaluations/evaluations/datasets/multidb.py +++ b/evaluations/evaluations/datasets/multidb.py @@ -707,18 +707,21 @@ def surface_rows() -> list[dict]: ) ) - # S6 — per-document metadata, with the twin's uri forbidden. - for north_name, south_name in NEAR_NAME_PAIRS[:2]: + # S6 — per-document metadata: the uri and then the database that holds it. + # + # The twin's uri is deliberately NOT forbidden here. Naming the other + # document as a different document is correct behaviour, and better than the + # minimum; forbidding it failed two answers that were right. Forbidding a + # distractor belongs on B3, where two elevations cannot both be the answer. + for north_name, _south_name in NEAR_NAME_PAIRS[:2]: s = station(north_name, NORTHERN) - twin = station(south_name, SOUTHERN) rows.append( _row( f"s6-{s.slug}", "S6", f"What is the uri of the document titled {s.title!r}, and which " "database holds it?", - expected_ordered=[s.uri], - forbidden_text=[twin.uri], + expected_ordered=[s.uri, s.database], ) ) return rows