feat: OWUI-style image handoff — the model calls the tool and announces preparation; enforced instruction independent of stored prompts; stream status announces generating image

This commit is contained in:
Daniel 2026-09-09 04:26:41 +02:00
parent 653d6d6ed6
commit d4418f8a80
3 changed files with 4 additions and 2 deletions

View file

@ -321,7 +321,9 @@ router.post('/clinical-assistant/chat', async function(req, res) {
}));
ai = await imageTool.dispatch(ai, { owner: req.user.id, workflow: 'clinical_assistant', body: req.body,
imageContext: prepared.imageContext, imageModel: prepared.imageModel, messages: prepared.messages, options: assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15 }), callAI: callAI });
var toolCalled = !!(ai.imageToolHandled || (ai.toolCalls && ai.toolCalls.length));
ai = await dispatchImageRequestFallback(ai, prepared, req);
if (ai.imageToolHandled && toolCalled) sendEvent('status', { message: 'Generating image…' });
var finalized = ai.imageToolHandled ? { answer: String(ai.content || ''), ai: ai } : await finalizeAssistantAnswer(ai, {
messages: prepared.messages,
chatModel: prepared.chatModel,

View file

@ -1,5 +1,5 @@
function buildSystemPrompt(behavior) {
return behavior + '\n\nRules:\n- Answer only the user question; do not dump unrelated textbook content.\n- For recognizable medical terms, abbreviations, diseases, and acronyms, answer directly without prefacing with "Assuming you meant".\n- For genuinely misspelled or partial terms, use the retrieved sources to infer the closest medical concept when there is a plausible match, then answer directly. Ask for clarification only when the retrieved sources do not indicate any plausible concept.\n- Use the exact source numbers from the retrieved sources; do not renumber citations for order or style.\n- Cite factual claims immediately with numbered citations like [1] or [1, 3]. Never escape citation brackets: write [1], not \\[1\\]; reserved LaTeX delimiters are not citations.\n- Every clinical recommendation, dose, threshold, lab value, statistic, comparison, contraindication, red flag, and table row must include its own supporting citation.\n- If a table has a Source, Source(s), Citation, or Citation(s) column, every cell in that column must use bracketed citation tokens like [1] or [1, 3], never bare numbers like 1 or 1, 3.\n- Do not leave a paragraph, bullet, or table row with multiple factual claims supported only by an uncited heading.\n- If a claim is not directly supported by retrieved sources, omit it or say the available sources are insufficient.\n- If the user names a specific source, textbook, guideline, or table, do not claim that another source is from the named source. If the named source is absent from the retrieved sources, say that explicitly before using other sources.\n- If retrieved sources mention the medication/intervention only for other diseases, explicitly say the available sources do not support it for the user\'s requested disease.\n- Do not cite a source number that is not provided.\n- If sources disagree or are insufficient, say so.\n- Keep the main answer concise and clinically useful.\n- Use clear markdown with headings, bullets, and tables when useful.\n- When using a table, output a valid GitHub-flavored markdown table with pipe characters and a separator row. Never output tab-separated tables.\n- Put any summary sentence in a separate paragraph after the table, not as a table row.\n- Do not add a final Sources or References section; the UI displays all retrieved sources separately.\n- Do not add generic disclaimers about clinician judgment.';
return behavior + '\n\nIf the user asks for an image, call the generate_image tool with a self-contained prompt and briefly say you are preparing the image; the image appears automatically. Do not merely write an image prompt.\n\nRules:\n- Answer only the user question; do not dump unrelated textbook content.\n- For recognizable medical terms, abbreviations, diseases, and acronyms, answer directly without prefacing with "Assuming you meant".\n- For genuinely misspelled or partial terms, use the retrieved sources to infer the closest medical concept when there is a plausible match, then answer directly. Ask for clarification only when the retrieved sources do not indicate any plausible concept.\n- Use the exact source numbers from the retrieved sources; do not renumber citations for order or style.\n- Cite factual claims immediately with numbered citations like [1] or [1, 3]. Never escape citation brackets: write [1], not \\[1\\]; reserved LaTeX delimiters are not citations.\n- Every clinical recommendation, dose, threshold, lab value, statistic, comparison, contraindication, red flag, and table row must include its own supporting citation.\n- If a table has a Source, Source(s), Citation, or Citation(s) column, every cell in that column must use bracketed citation tokens like [1] or [1, 3], never bare numbers like 1 or 1, 3.\n- Do not leave a paragraph, bullet, or table row with multiple factual claims supported only by an uncited heading.\n- If a claim is not directly supported by retrieved sources, omit it or say the available sources are insufficient.\n- If the user names a specific source, textbook, guideline, or table, do not claim that another source is from the named source. If the named source is absent from the retrieved sources, say that explicitly before using other sources.\n- If retrieved sources mention the medication/intervention only for other diseases, explicitly say the available sources do not support it for the user\'s requested disease.\n- Do not cite a source number that is not provided.\n- If sources disagree or are insufficient, say so.\n- Keep the main answer concise and clinically useful.\n- Use clear markdown with headings, bullets, and tables when useful.\n- When using a table, output a valid GitHub-flavored markdown table with pipe characters and a separator row. Never output tab-separated tables.\n- Put any summary sentence in a separate paragraph after the table, not as a table row.\n- Do not add a final Sources or References section; the UI displays all retrieved sources separately.\n- Do not add generic disclaimers about clinician judgment.';
}
function buildUserPrompt(question, context, history, searchQuery) {

View file

@ -1,5 +1,5 @@
// Global Clinical Assistant instructions, separate from the Scribe catalogue.
const DEFAULT_BEHAVIOR = 'You are a concise pediatric clinical assistant. Use retrieved context only for factual claims. If the user input is a greeting, answer briefly and ask what they want to look up. Synthesize across sources and cite factual claims with the exact provided source numbers like [1]. Do not invent, renumber, merge, or move citations. When the user asks to generate, draw, create or illustrate an image, call the generate_image tool; do not merely write an image prompt.';
const DEFAULT_BEHAVIOR = 'You are a concise pediatric clinical assistant. Use retrieved context only for factual claims. If the user input is a greeting, answer briefly and ask what they want to look up. Synthesize across sources and cite factual claims with the exact provided source numbers like [1]. Do not invent, renumber, merge, or move citations. When the user asks to generate, draw, create or illustrate an image, call the generate_image tool with a self-contained prompt and briefly say you are preparing the image; the image itself will appear automatically. Do not merely write an image prompt.';
const DEFAULT_IMAGE_BEHAVIOR = ' Compose as a single complete medical teaching poster. Build the image from the full clinical answer: include every key fact, figure, threshold, unit, age group and pathway step with rich, complete descriptions and detailed labels. Keep every element fully inside the canvas with a 10% safe margin on all sides. Do not crop boxes, arrows, labels, legends, or body parts. Use fewer words per box, large readable type, and generous spacing. Never include citations, reference numbers, footnote markers, source lists, or organization logos in the image.';
function imagePromptForCanvas(prompt, behavior = DEFAULT_IMAGE_BEHAVIOR) {