diff --git a/src/utils/clinicalAnswer.js b/src/utils/clinicalAnswer.js index a12e46ce..82a092b0 100644 --- a/src/utils/clinicalAnswer.js +++ b/src/utils/clinicalAnswer.js @@ -33,6 +33,39 @@ function assistantGenerationOptions(overrides) { async function finalizeAssistantAnswer(ai, options) { options = options || {}; var answer = stripModelSourcesSection(String(ai && ai.content || '').trim()); + + // An empty answer is a failure, and it used to be the one failure that was + // never caught: shouldRegenerateTruncatedAnswer returns false for it — there + // is no dangling conjunction to detect in a zero-length string — so nothing + // regenerated it, nothing raised, and the empty turn was saved to the chat + // and rendered as an empty bubble. It then stayed in the history, where it + // degrades every answer after it. + // + // Asked for once more before giving up: a model returning nothing is usually + // transient. If it comes back empty again, that is said out loud rather than + // presented as an answer. + if (!answer && typeof options.callAI === 'function') { + console.warn('[clinical-assistant] the model returned an empty answer; asking once more', { + finishReason: ai && ai.finishReason, streamed: Boolean(options.streamed) }); + if (typeof options.onRegenerating === 'function') options.onRegenerating(); + var retried = await options.callAI(options.messages, Object.assign({}, options.generationOptions || {}, { + model: options.chatModel || undefined, + maxTokens: 5000 + })); + answer = stripModelSourcesSection(String(retried && retried.content || '').trim()); + if (retried) { + ai.model = retried.model || ai.model; + ai.provider = retried.provider || ai.provider; + ai.finishReason = retried.finishReason || ai.finishReason; + } + } + if (!answer) { + var empty = new Error('The model returned an empty answer. Try asking again, or rephrase the question.'); + empty.statusCode = 502; + empty.code = 'empty_answer'; + throw empty; + } + if (shouldRegenerateTruncatedAnswer(answer, ai && ai.finishReason) && typeof options.callAI === 'function') { console.warn('[clinical-assistant] answer looked truncated; regenerating final answer', { finishReason: ai && ai.finishReason, chars: answer.length, streamed: Boolean(options.streamed) }); if (typeof options.onRegenerating === 'function') options.onRegenerating(); @@ -69,6 +102,8 @@ function cleanDanglingSourceLeadIn(answer) { function shouldRegenerateTruncatedAnswer(answer, finishReason) { answer = String(answer || '').trim(); + // Empty is not "not truncated" — it is a different failure, and it is handled + // before this is ever called. Nothing here can judge a zero-length string. if (!answer) return false; if (finishReason === 'length') return true; if (/\b(the|a|an|and|or|but|with|without|for|to|of|in|on|as|by|from|because|therefore|however|some|many|most|few|several|additional|other|further|including|such|available)$/i.test(answer)) return true; diff --git a/test/empty-answer.test.js b/test/empty-answer.test.js new file mode 100644 index 00000000..2852fb76 --- /dev/null +++ b/test/empty-answer.test.js @@ -0,0 +1,70 @@ +// An empty answer from the model is a failure. It used to be the only failure +// that was never caught: the truncation check returns false for a zero-length +// string, so nothing regenerated it and nothing raised — the empty turn was +// saved into the chat and drawn as an empty bubble, then stayed in the history +// where it degrades every answer after it. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { finalizeAssistantAnswer } = require('../src/utils/clinicalAnswer'); + +function options(replies) { + const calls = []; + const queue = replies.slice(); + return { + calls, + messages: [{ role: 'user', content: 'q' }], + chatModel: 'synthetic', + generationOptions: {}, + callAI: async (messages, opts) => { + calls.push({ messages, opts }); + return { content: queue.length > 1 ? queue.shift() : queue[0], model: 'synthetic' }; + } + }; +} + +test('an empty answer is asked for once more rather than returned', async () => { + const o = options(['A real answer about croup.']); + const out = await finalizeAssistantAnswer({ content: '' }, o); + + assert.equal(o.calls.length, 1, 'one retry, not none and not a loop'); + assert.equal(out.answer, 'A real answer about croup.'); +}); + +test('an answer that is empty twice raises instead of being presented as one', async () => { + const o = options(['']); + await assert.rejects( + () => finalizeAssistantAnswer({ content: '' }, o), + err => { + assert.equal(err.code, 'empty_answer'); + assert.equal(err.statusCode, 502); + assert.match(err.message, /empty answer/i); + return true; + }); + assert.equal(o.calls.length, 1, 'it does not keep paying for the same empty reply'); +}); + +test('whitespace-only counts as empty — it renders as an empty bubble just the same', async () => { + const o = options([' \n\n ']); + await assert.rejects(() => finalizeAssistantAnswer({ content: '\n \t' }, o), + err => err.code === 'empty_answer'); +}); + +test('an answer that survives stripping is returned untouched and costs no retry', async () => { + const o = options(['should not be called']); + const out = await finalizeAssistantAnswer({ content: 'Croup is a viral illness.' }, o); + + assert.equal(out.answer, 'Croup is a viral illness.'); + assert.equal(o.calls.length, 0); +}); + +test('the caller is told to try again, because a repeat usually works', async () => { + const o = options(['']); + await assert.rejects(() => finalizeAssistantAnswer({ content: '' }, o), + err => /try asking again|rephrase/i.test(err.message)); +}); + +test('with no callAI to retry with, an empty answer still raises rather than returning empty', async () => { + await assert.rejects( + () => finalizeAssistantAnswer({ content: '' }, { messages: [] }), + err => err.code === 'empty_answer'); +});