From b62d6e920b200f3628d7a47d17a50378d48551af Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 30 Jul 2026 15:57:48 +0300 Subject: [PATCH] Drop budget_spent from eval attributes It was n_rejected_searches > 0 recorded next to the integer it derived from, and the name overclaimed: analysis_execute_code also raises ToolFailed when execute_count exceeds max_executions, which the flag never saw. Callers can compare the counters directly. --- CHANGELOG.md | 2 +- evaluations/evaluations/benchmark.py | 1 - evaluations/evaluations/capability_runner.py | 17 +++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e25dbba..e6059883 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Added -- `evaluations run` records `cited_chunk_ids`, `searched_uris`, `n_searches`, `n_search_calls`, `n_rejected_searches`, `n_failed_tools`, `n_executions`, `n_requests` and `budget_spent` as eval attributes alongside `cited_uris`. +- `evaluations run` records `cited_chunk_ids`, `searched_uris`, `n_searches`, `n_search_calls`, `n_rejected_searches`, `n_failed_tools`, `n_executions` and `n_requests` as eval attributes alongside `cited_uris`. ### Changed diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 51cef761..2541ca05 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -442,7 +442,6 @@ async def run_qa_benchmark( set_eval_attribute("n_failed_tools", result.n_failed_tools) set_eval_attribute("n_executions", result.n_executions) set_eval_attribute("n_requests", result.n_requests) - set_eval_attribute("budget_spent", result.budget_spent) return result.answer report = await _evaluate(answer_question) diff --git a/evaluations/evaluations/capability_runner.py b/evaluations/evaluations/capability_runner.py index 03da8773..7fd8e500 100644 --- a/evaluations/evaluations/capability_runner.py +++ b/evaluations/evaluations/capability_runner.py @@ -39,7 +39,6 @@ class CapabilityRunResult: n_rejected_searches: int = 0 n_failed_tools: int = 0 n_requests: int = 0 - budget_spent: bool = False def _count_tool_traffic( @@ -48,13 +47,16 @@ def _count_tool_traffic( """Count search calls, failed calls and model requests in a run. ``state.searches`` is keyed by query, so it collapses repeated queries and - never records a call the capability refused. Counting the message history - instead gives the real number of attempts. + never records a call the capability refused. The capability object cannot be + read instead: ``for_run`` hands the run a ``replace()`` copy, so the outer + instance's counters stay at zero. Counting the message history is the only + way to see the real number of attempts. - Failures are split by tool. Only the search tool fails for want of budget, - whereas the code tool raises ``ToolFailed`` for any error in model-written - Python, so counting every failure together would report a ``ZeroDivisionError`` - as budget exhaustion. + Search failures are counted apart because the search tool fails only when + its budget is spent. A failed code call is ambiguous — an exhausted + execution budget and any error in model-written Python both surface as + ``ToolFailed``, distinguishable in the history only by message text — so + ``n_failed_tools`` covers both without claiming to tell them apart. """ search_tool = f"{namespace}_search" search_calls = 0 @@ -162,5 +164,4 @@ async def run_capability_question( n_rejected_searches=n_rejected_searches, n_failed_tools=n_failed_tools, n_requests=n_requests, - budget_spent=n_rejected_searches > 0, )