From 9695e3c7f93c54fcb1360c4c596386876a15a874 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 14 Sep 2026 05:49:16 +0200 Subject: [PATCH] fix: an empty reply whose budget went to reasoning is retried with room to write Any call with a small max_tokens could come back blank from a reasoning model: finish_reason=length, content empty, the budget spent thinking. The shared LiteLLM call now retries that one signature once, with at least 6000 tokens and low reasoning effort, so no caller has to guess a budget. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- src/utils/ai.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils/ai.js b/src/utils/ai.js index 7cbc48ee..8c0b6f79 100644 --- a/src/utils/ai.js +++ b/src/utils/ai.js @@ -356,6 +356,15 @@ async function callBedrock(messages, model, temperature, maxTokens) { // ============================================================ // CALL LITELLM (OpenAI-compatible proxy) // ============================================================ +// A reasoning model spends part of max_tokens thinking before it writes. When +// the budget is small (a query rewrite at 80, a milestone note at 500) it can +// hit the limit with the whole budget spent on reasoning and nothing written: +// a 200, finish_reason=length, content ''. Every caller then saw an empty +// reply — the take-home sheet was the one the user noticed. The retry gives +// the same request room to finish and asks for low reasoning effort; it fires +// only for that signature, never for a genuinely empty answer. +var REASONING_RETRY_FLOOR = 6000; + async function callLiteLLM(messages, model, temperature, maxTokens, generation) { if (!litellmClient) throw new Error('LiteLLM not configured. Set LITELLM_API_BASE in .env'); @@ -375,6 +384,13 @@ async function callLiteLLM(messages, model, temperature, maxTokens, generation) var reasoningChars = String(choice.message.reasoning_content || choice.message.reasoning || '').length; console.warn('[ai] empty reply from ' + model + ': finish_reason=' + (choice.finish_reason || '?') + ' completion_tokens=' + (completion.usage && completion.usage.completion_tokens) + ' reasoning_chars=' + reasoningChars); + var starved = choice.finish_reason === 'length' && reasoningChars > 0 && !(generation && generation.reasoningRetried); + if (starved) { + var retryBudget = Math.max(REASONING_RETRY_FLOOR, (maxTokens || 0) * 4); + console.warn('[ai] reasoning consumed the budget; retrying ' + model + ' with max_tokens=' + retryBudget); + return callLiteLLM(messages, model, temperature, retryBudget, + Object.assign({}, generation || {}, { reasoningEffort: 'low', reasoningRetried: true })); + } } return { success: true,