From 5e10846292d29804847d872a7a2044e6f10d7c4d Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sun, 26 Jul 2026 18:56:00 +0300
Subject: [PATCH] Raise FRAMES input budget: qa max_tokens 8192, judge 16384

---
 evaluations/configs/frames.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/evaluations/configs/frames.yaml b/evaluations/configs/frames.yaml
index 7c20fad4..f9339aa9 100644
--- a/evaluations/configs/frames.yaml
+++ b/evaluations/configs/frames.yaml
@@ -28,7 +28,9 @@ qa:
     provider: openai
     name: gemma4-26b
     base_url: http://vllm:11432/v1
-    max_tokens: 49152
+    # vLLM reserves max_tokens out of max_model_len; a large value starves
+    # the input budget and 400s long agentic contexts.
+    max_tokens: 8192
 
 evaluations:
   judge:
@@ -36,4 +38,4 @@ evaluations:
     name: RedHatAI/Qwen3.6-35B-A3B-NVFP4
     base_url: http://vllm:11430/v1
     temperature: 0.0
-    max_tokens: 4096
+    max_tokens: 16384