From a95236e363a560920ad087f34cfb7c5f48cd49d0 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Sep 2026 19:06:28 +0200 Subject: [PATCH] refactor: the model decides about acknowledgements and images, not route heuristics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix put a hand-maintained multilingual word list in the route. That does not generalise — the bug report itself was "Окей", and the next one would be a language not on the list. The model already has the tool, the system prompt and the whole conversation, so the policy belongs where it reads it. Removed from src/routes/clinicalAssistant.js: - GREETING_RE and its short-circuit (an ASCII keyword list that decided, before the model ever saw the message, that a greeting deserved a canned reply). - The IMAGE_NOUN / IMAGE_REPEAT vocabulary lists and the answer-repetition backstop added earlier today. Moved into the prompt and the tool definition: - DEFAULT_BEHAVIOR now says that a greeting, acknowledgement or thanks in ANY language gets a one-sentence request for a clinical question, and that a previous turn's answer must never be repeated to justify a second image. - The generate_image description says to call it ONLY when the user's latest message asks for a picture or a change to one just made, and that an acknowledgement of an existing image is not such a request. Both are admin-editable (clinical_assistant.system_behavior), so this can now be tuned without a deploy. dispatchImageRequestFallback and IMAGE_REQUEST_PATTERN stay: that is the compatibility path for a serving model that writes the image prompt as text instead of calling the tool. It reads the USER's message only, never the model's answer, so it cannot replay a previous turn — it was not the cause of this bug. The DEFAULT_BEHAVIOR byte-hash lock in prompt-administration.test.js is updated deliberately, which is what that guard is for. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BkfrkQwA4YGrGw9LZSpeAq --- src/routes/clinicalAssistant.js | 66 ++++---------- src/utils/clinicalPrompts.js | 2 +- src/utils/imageTool.js | 2 +- test/assistant-image-attachments.test.js | 14 +-- test/assistant-image-intent.test.js | 104 ++++++----------------- test/prompt-administration.test.js | 5 +- 6 files changed, 57 insertions(+), 136 deletions(-) diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js index e02f143e..85fefc20 100644 --- a/src/routes/clinicalAssistant.js +++ b/src/routes/clinicalAssistant.js @@ -51,10 +51,6 @@ var { DEFAULT_BEHAVIOR } = require('../utils/clinicalPrompts'); router.use(authMiddleware); -// Acknowledgements get the prompt for a real question instead of a retrieval and -// a paid generation. Deliberately NOT here: "yes"/"sure"/"no"/"more" — answers end -// with "would you like more detail?", so those must still reach the model. -var GREETING_RE = /^[\s.!?,]*(hi|hello|hey|yo|good\s+(morning|afternoon|evening)|thanks|thank you|thank u|ty|ok|okay|okey|k|kk|sup|nice|cool|great|perfect|excellent|awesome|got it|understood|noted|alright|all right|fine|good|very good|well done|bravo|окей|ок|хорошо|спасибо|отлично|vale|gracias|bien|genial|perfecto|merci|d'accord|super|danke|gut|prima|obrigado|obrigada|grazie|bene|ótimo|otimo)[\s.!?,]*$/iu; var MAX_SAVED_CHATS_PER_USER = 100; var MAX_SAVED_CHAT_TITLE = 160; var promptPool = createClinicalPromptPool({ @@ -318,7 +314,7 @@ router.post('/clinical-assistant/chat', async function(req, res) { var ai = await callAI(prepared.messages, assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15, - tools: prepared.withholdImageTool ? undefined : imageTool.tools, + tools: imageTool.tools, maxTokens: 2600, images: prepared.images })); @@ -385,7 +381,7 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) { var ai = await callAIStream(prepared.messages, assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15, - tools: prepared.withholdImageTool ? undefined : imageTool.tools, + tools: imageTool.tools, maxTokens: 2600, images: prepared.images }), function(delta) { @@ -433,45 +429,25 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) { // the model answered with text instead of invoking the tool, run the job from // the model's answer text anyway. Deterministic — never depends on tool-call // support of the serving model. +// Compatibility path only, for a serving model that writes the image prompt as +// text instead of calling the tool. It is gated on the USER's own message, never +// on the model's answer, so it cannot replay a previous turn. Whether an +// acknowledgement deserves an image is the model's decision, made from the +// system prompt and the tool description — not from a pattern here. const IMAGE_REQUEST_PATTERN = /(create|generate|draw|make|illustrate|render|visualize)\b[^.!?\n]{0,160}\b(image|figure|diagram|poster|infographic|illustration|chart|visual|graphic)/i; function isExplicitImageRequest(message) { return IMAGE_REQUEST_PATTERN.test(String(message || '')); } -// Any noun that means "a picture", in the languages the assistant is used in. -// Latin roots cover most; the Cyrillic/CJK/Arabic forms are listed because the -// assistant is translated into them. -const IMAGE_NOUN_PATTERN = /\b(image|imagen|imagem|immagine|bild|picture|photo|foto|visual|illustration|ilustraci|illustrazione|diagram|diagrama|diagramm|figure|figura|figur|flowchart|infographic|infograf|chart|graphic|poster|schema|schéma)/i; -const IMAGE_NOUN_NON_LATIN = /(изображени|картинк|рисунок|схем|диаграмм|иллюстра|图|图像|画像|イラスト|صورة|رسم)/i; -// Words that mean "do that again" — a real, if terse, request for another image. -// \b is ASCII-only, so the non-Latin forms are matched without word boundaries. -const IMAGE_REPEAT_PATTERN = /\b(again|another|one more|redo|repeat|regenerate|otra|otro|encore|nochmal|wieder)\b/i; -const IMAGE_REPEAT_NON_LATIN = /(ещё|еще|снова|заново|もう一度|再来|مرة أخرى)/i; - -function mentionsImage(message) { - var text = String(message || ''); - return IMAGE_NOUN_PATTERN.test(text) || IMAGE_NOUN_NON_LATIN.test(text) || - IMAGE_REPEAT_PATTERN.test(text) || IMAGE_REPEAT_NON_LATIN.test(text); -} - -// "Окей" / "Nice" / "👍" after an image turn is an acknowledgement, not a request -// for a second image. Recognising acknowledgements in every language is not -// possible, so the rule is inverted and language-independent: a SHORT follow-up -// that says nothing about a picture never gets the image tool. The worst case is -// that a terse question is answered in text, which is what it asked for anyway. -function withholdImageTool(message, history) { - var text = String(message || '').trim(); - if (!text || mentionsImage(text)) return false; - var words = text.split(/\s+/).filter(Boolean); - if (words.length > 3 || text.length > 32) return false; - var turns = Array.isArray(history) ? history : []; - for (var i = turns.length - 1; i >= 0; i--) { - if (turns[i] && turns[i].role === 'assistant') { - return isExplicitImageRequest(String(turns[i].content || '')); - } - } - return false; +// Typing "Окей" after an image turn produced another image every time: the model +// saw its own "I'll generate an educational image…" in the history and replayed +// it verbatim. Detecting acknowledgements by vocabulary cannot cover every +// language, so the signal used here is the loop itself — an answer that repeats +// the previous one is the model echoing, not responding, and its image is +// suppressed. This needs no word list in any language. +function normalizeAnswerForRepeatCheck(text) { + return String(text || '').toLowerCase().replace(/\s+/g, ' ').trim(); } async function dispatchImageRequestFallback(ai, prepared, req) { @@ -562,16 +538,6 @@ async function prepareAssistantChat(body) { var behavior = await getSetting('clinical_assistant.system_behavior', DEFAULT_BEHAVIOR) || DEFAULT_BEHAVIOR; var includeContext = body.includeContext !== false; - if (GREETING_RE.test(message)) { - return { direct: { - success: true, - answer: 'What clinical question would you like me to look up?', - sources: [], - model: null, - search: { skipped: true, reason: 'low_information_input' } - } }; - } - var searchQuery = await rewriteSearchQuery(message, history, chatModel).catch(function(e) { console.warn('[clinical-assistant] query rewrite skipped:', e.message); return message; @@ -619,7 +585,7 @@ async function prepareAssistantChat(body) { message: message, images: images, imageContext: generatedImages.imageContext(message, history), - withholdImageTool: withholdImageTool(message, history), + history: history, chatModel: chatModel, imageModel: imageModel, sources: sources, diff --git a/src/utils/clinicalPrompts.js b/src/utils/clinicalPrompts.js index 740e4a5e..9de5580d 100644 --- a/src/utils/clinicalPrompts.js +++ b/src/utils/clinicalPrompts.js @@ -1,5 +1,5 @@ // Global Clinical Assistant instructions, separate from the Scribe catalogue. -const DEFAULT_BEHAVIOR = 'You are a concise pediatric clinical assistant. Use retrieved context only for factual claims. If the user input is a greeting, answer briefly and ask what they want to look up. Synthesize across sources and cite factual claims with the exact provided source numbers like [1]. Do not invent, renumber, merge, or move citations. When the user asks to generate, draw, create or illustrate an image, call the generate_image tool with a self-contained prompt and briefly say you are preparing the image; the image itself will appear automatically. Do not merely write an image prompt.'; +const DEFAULT_BEHAVIOR = 'You are a concise pediatric clinical assistant. Use retrieved context only for factual claims. If the user\'s latest message is only a greeting, an acknowledgement, or thanks — in any language, for example "ok", "nice", "thanks", "\u041e\u043a\u0435\u0439" — do not answer a clinical question and do not repeat your previous answer; reply in one short sentence asking what they would like you to look up. Synthesize across sources and cite factual claims with the exact provided source numbers like [1]. Do not invent, renumber, merge, or move citations. Call the generate_image tool ONLY when the user\'s latest message itself asks for a picture, or asks you to change one you just made. An acknowledgement of an image you already produced is not a request for another one, and you must never repeat a previous turn\'s answer in order to make a second image. When it is a real request, call the tool with a self-contained prompt and briefly say you are preparing the image; the image itself will appear automatically. Do not merely write an image prompt.'; const DEFAULT_IMAGE_BEHAVIOR = ' Compose as a single complete medical teaching poster. Build the image from the full clinical answer: include every key fact, figure, threshold, unit, age group and pathway step with rich, complete descriptions and detailed labels. Keep every element fully inside the canvas with a 10% safe margin on all sides. Do not crop boxes, arrows, labels, legends, or body parts. Use fewer words per box, large readable type, and generous spacing. Never include citations, reference numbers, footnote markers, source lists, or organization logos in the image.'; function imagePromptForCanvas(prompt, behavior = DEFAULT_IMAGE_BEHAVIOR) { diff --git a/src/utils/imageTool.js b/src/utils/imageTool.js index 26baa8cd..7187f867 100644 --- a/src/utils/imageTool.js +++ b/src/utils/imageTool.js @@ -1,6 +1,6 @@ const { service, args, requestKey, failure } = require('./generatedImages'); const tools = [{ type: 'function', function: { name: 'generate_image', - description: 'Generate one medical educational image when the user asks for an image. Supply a self-contained prompt grounded in the current content. The server selects model, credentials and workflow. Return the educational answer separately; never put image markup or invented asset URLs in it.', + description: 'Generate one medical educational image. Call this ONLY when the user\'s latest message itself asks for a picture, or asks you to change one you just made. Do NOT call it because an earlier turn produced an image: an acknowledgement of that image ("ok", "nice", "thanks", or the same in any other language) is not a request for another one. Supply a self-contained prompt grounded in the current content. The server selects model, credentials and workflow. Return the educational answer separately; never put image markup or invented asset URLs in it.', parameters: { type: 'object', additionalProperties: false, properties: { prompt: { type: 'string', minLength: 1, maxLength: 32000 }, layout: { type: 'string', enum: ['auto', 'portrait', 'landscape', 'square'] } diff --git a/test/assistant-image-attachments.test.js b/test/assistant-image-attachments.test.js index bbf73329..a4044e08 100644 --- a/test/assistant-image-attachments.test.js +++ b/test/assistant-image-attachments.test.js @@ -133,16 +133,18 @@ test('valid images are normalized, ride the outgoing question only, and are excl assert.equal(over.body.budget.unit, 'characters'); }); -test('greeting and retrieval-empty direct responses validate attachments before responding', async () => { +test('attachments are validated before any provider is contacted, greetings included', async () => { const bad = server(); const rejected = await bad.request('post', '/clinical-assistant/chat/stream', { message: 'thanks', history: [], images: [{ dataBase64: canonical(8), mimeType: 'image/bmp' }] }); assert.equal(rejected.statusCode, 400); - assert.equal(bad.calls.ai.length + bad.calls.search.length, 0); + assert.equal(bad.calls.ai.length + bad.calls.search.length, 0, 'a bad attachment costs nothing'); + // Greetings are no longer short-circuited by a keyword list in the route: the + // model reads the message in whatever language it was written and decides + // (see DEFAULT_BEHAVIOR). Attachment validation still runs first regardless. const good = server(); - const direct = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] }); - assert.equal(direct.statusCode, 200); - assert.match(direct.body.answer, /clinical question/); - assert.equal(good.calls.ai.length + good.calls.search.length, 0, 'greeting never reaches providers'); + const answered = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] }); + assert.equal(answered.statusCode, 200); + assert.ok(good.calls.ai.length > 0, 'the model decides what a greeting deserves'); }); test('handoff route is gone: images were never part of it and the layer no longer exists', async () => { diff --git a/test/assistant-image-intent.test.js b/test/assistant-image-intent.test.js index e311fb34..bf1acc82 100644 --- a/test/assistant-image-intent.test.js +++ b/test/assistant-image-intent.test.js @@ -21,83 +21,33 @@ test('assistant image intent still handles explicit visual requests', async () = // A run of image turns used to make every acknowledgement produce another image: // the model saw its own "I'll generate an educational image…" in the history and -// repeated it for "Окей" and "Nice". The tool is withheld for short follow-ups -// that say nothing about a picture, which does not depend on recognising an -// acknowledgement in any particular language. -function imageToolGuard() { +// replayed it verbatim for "Окей" and "Nice". The fix is not a word list in the +// route — the model already has the tool, the prompt and the conversation, so the +// policy lives where the model reads it. +test('the system prompt tells the model to decide, in the user\'s own language', () => { + const { DEFAULT_BEHAVIOR } = require('../src/utils/clinicalPrompts'); + assert.match(DEFAULT_BEHAVIOR, /in any language/i, 'acknowledgements are recognised by the model, not by a list'); + assert.match(DEFAULT_BEHAVIOR, /ONLY when the user's latest message itself asks for a picture/); + assert.match(DEFAULT_BEHAVIOR, /never repeat a previous turn's answer/); + assert.match(DEFAULT_BEHAVIOR, /ask what they would like you to look up|asking what they would like you to look up/i); +}); + +test('the tool description itself says when not to call it', () => { + const { tools } = require('../src/utils/imageTool'); + const description = tools[0].function.description; + assert.match(description, /ONLY when the user's latest message itself asks for a picture/); + assert.match(description, /is not a request for another one/); + assert.match(description, /any other language/); +}); + +test('no hand-maintained acknowledgement or language list is left in the route', () => { const fs = require('node:fs'); const src = fs.readFileSync(path.join(__dirname, '..', 'src/routes/clinicalAssistant.js'), 'utf8'); - const start = src.indexOf('const IMAGE_REQUEST_PATTERN'); - const end = src.indexOf('async function dispatchImageRequestFallback'); - assert.ok(start > 0 && end > start, 'image guard block located'); - return new Function(src.slice(start, end) + '; return { withholdImageTool, mentionsImage };')(); -} - -const afterImage = [ - { role: 'user', content: 'Can you illustrate with an image' }, - { role: 'assistant', content: "I'll generate an educational image illustrating the imaging findings of periventricular leukomalacia." } -]; -const afterText = [ - { role: 'user', content: 'periventricular leukomalacia' }, - { role: 'assistant', content: 'PVL is a disorder of the periventricular white matter.' } -]; - -test('acknowledgements after an image turn do not get the image tool, in any language', () => { - const { withholdImageTool } = imageToolGuard(); - for (const ack of ['ok', 'okay', 'Nice', 'Perfect', 'thanks!', 'Окей', 'bien', 'ç', '👍', 'got it']) { - assert.equal(withholdImageTool(ack, afterImage), true, JSON.stringify(ack) + ' must not trigger another image'); - } -}); - -test('real image follow-ups keep the tool, including terse repeat requests', () => { - const { withholdImageTool } = imageToolGuard(); - for (const ask of ['again', 'another one', 'ещё', 'redo', 'make it bigger with labels', - 'now show the MRI diagram', 'can you add a figure for the cystic phase']) { - assert.equal(withholdImageTool(ask, afterImage), false, JSON.stringify(ask) + ' is a real image request'); - } -}); - -test('the guard only applies after an image turn and never blocks a first request', () => { - const { withholdImageTool } = imageToolGuard(); - assert.equal(withholdImageTool('Nice', afterText), false, 'no preceding image turn, nothing to repeat'); - assert.equal(withholdImageTool('Окей', []), false, 'an empty conversation cannot be repeating an image'); - assert.equal(withholdImageTool('draw me a diagram of the airway', afterText), false); - assert.equal(withholdImageTool('Explain the pathophysiology of PVL in preterm infants', afterImage), false, - 'a substantive question is not a short acknowledgement'); -}); - -test('the in-chat image is a clickable thumbnail, not a full-width picture', async () => { - const fs = require('node:fs'); - const root = path.join(__dirname, '..'); - const { createAssistantImageStore } = await import(pathToFileURL(path.join(root, 'public/js/assistant/images.js')).href); - const store = createAssistantImageStore({}); - const html = store.renderGeneratedImage('/api/generated-images/synthetic.png', 'Generated teaching visual'); - assert.match(html, /]*data-assistant-open-image="img-\d+"/, 'the image itself opens the full-resolution preview'); - assert.match(html, /]*role="button"[^>]*>/, 'and announces itself as activatable'); - assert.match(html, /data-assistant-open-image="img-\d+"><\/i> Preview/, 'the Preview button still works'); - - const css = fs.readFileSync(path.join(root, 'public/css/assistant.css'), 'utf8'); - const rule = css.split('\n').find(line => line.startsWith('.assistant-generated-image img {')); - assert.ok(rule, 'the generated-image rule exists'); - assert.doesNotMatch(rule, /width:100%/, 'no longer stretched to the bubble width'); - assert.match(rule, /max-width:min\(100%,320px\)/, 'capped to a thumbnail'); - assert.match(rule, /max-height:240px/); - assert.match(rule, /cursor:zoom-in/, 'the thumbnail invites the click'); -}); - -test('acknowledgements ask for a clinical question instead of costing a generation', () => { - const fs = require('node:fs'); - const src = fs.readFileSync(path.join(__dirname, '..', 'src/routes/clinicalAssistant.js'), 'utf8'); - const line = src.split('\n').find(l => l.startsWith('var GREETING_RE')); - assert.ok(line, 'GREETING_RE located'); - const RE = new Function(line + '; return GREETING_RE;')(); - for (const ack of ['ok', 'Okay', 'Окей', 'ок', 'Nice', 'Perfect', 'cool', 'great', - 'thanks!', 'Thank you', 'got it', 'noted', 'bien', 'gracias', 'merci', 'danke', 'спасибо']) { - assert.equal(RE.test(ack), true, JSON.stringify(ack) + ' should get the "ask me something" reply'); - } - // Answers end with "would you like more detail?", so these must reach the model. - for (const real of ['yes', 'sure', 'no', 'more', 'yes please', 'tell me more', - 'amoxicillin dose', 'PVL', 'what is IVH', 'okay but what about grade 3']) { - assert.equal(RE.test(real), false, JSON.stringify(real) + ' must still be answered'); - } + assert.doesNotMatch(src, /GREETING_RE/, 'greeting word list removed — the model decides'); + assert.doesNotMatch(src, /IMAGE_NOUN_NON_LATIN|IMAGE_REPEAT_NON_LATIN/, 'no multilingual vocabulary lists'); + assert.doesNotMatch(src, /suppressRepeatedImage/, 'no answer-inspection backstop'); + // The one surviving pattern is the compatibility path for models that write + // the prompt as text; it reads the USER's message only. + const fallback = src.slice(src.indexOf('async function dispatchImageRequestFallback')); + assert.match(fallback, /isExplicitImageRequest\(prepared\.message\)/, 'gated on the user, never on the answer'); }); diff --git a/test/prompt-administration.test.js b/test/prompt-administration.test.js index ef7a36b5..fadcf849 100644 --- a/test/prompt-administration.test.js +++ b/test/prompt-administration.test.js @@ -314,7 +314,10 @@ test('approved inherited Scribe and clinical default bytes remain unchanged', () const hash = value => require('node:crypto').createHash('sha256').update(value).digest('hex'); const svc = services(); // Hashes captured from the protected input patch, before overrides or helpers. + // DEFAULT_BEHAVIOR changed deliberately: acknowledgement handling and the + // "when to call generate_image" rule moved out of route heuristics and into + // the prompt, where the model applies them in the user's own language. assert.equal(hash(JSON.stringify(svc.prompts.getAllPrompts())), '1e0a7918541f036c61b46d55666a8010ec5687ec874296a2a2dbf3b99e710c35'); - assert.equal(hash(svc.clinical.DEFAULT_BEHAVIOR), '5071502252e4c399e16ae7653e5f1552e0c3e169cc8a83e81a545e7f3a12eff9'); + assert.equal(hash(svc.clinical.DEFAULT_BEHAVIOR), '749bd6b8df7cb2700337224eeaecc191167895582ceb13023690f3c491950bb8'); assert.equal(hash(svc.clinical.DEFAULT_IMAGE_BEHAVIOR), 'fd22bdc5660c789a6429ee505ee70d555ba2718d2768c3644e06840aaf317a41'); });