From 94210ac0a1b0bb142e02f5056238c79b002417d8 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 8 Sep 2026 15:53:05 +0200 Subject: [PATCH] feat: attach images to clinical search with strict validation and multimodal routing --- docs/CLINICAL_ASSISTANT.md | 11 + public/components/assistant.html | 3 + public/css/assistant.css | 8 + public/js/assistant/api.js | 16 + public/js/clinicalAssistant.js | 95 +++++- src/routes/clinicalAssistant.js | 17 +- src/utils/ai.js | 35 +- src/utils/clinicalConversation.js | 44 ++- test/assistant-image-attachments.test.js | 411 +++++++++++++++++++++++ 9 files changed, 630 insertions(+), 10 deletions(-) create mode 100644 test/assistant-image-attachments.test.js diff --git a/docs/CLINICAL_ASSISTANT.md b/docs/CLINICAL_ASSISTANT.md index 497e888..9b09f9f 100644 --- a/docs/CLINICAL_ASSISTANT.md +++ b/docs/CLINICAL_ASSISTANT.md @@ -72,6 +72,16 @@ show me the image/figure Clinical answer response caching is intentionally disabled. Redis can support prompt suggestions and operational metadata, but final answers should be generated from current retrieval context. +## Image Attachments + +Users can attach up to 4 images (PNG, JPEG, WebP) to an outgoing clinical question. Attachments are **input-only**: + +- They are validated client-side and authoritatively on the server (MIME allowlist, canonical base64, ≤ 5 MiB per image, ≤ 4 images, ≤ 10 MiB decoded total). Invalid input is rejected with 400 before any retrieval or provider call. +- They are sent **only** with the outgoing clinical question, never with handoff summaries, image generation, saved chats, downloads or transcripts. +- The conversation budget counts text only: images are excluded from the UTF-16 code-unit count. The server still validates every request. +- Only OpenAI-compatible providers (LiteLLM, OpenRouter, Azure) receive them as multimodal content parts (`text` + `image_url` data URIs) on the latest user message; the system/retrieval/history structure is unchanged. Legacy direct adapters (Bedrock/Vertex) refuse with a clear 400 before contacting the provider. +- Attachments clear on a successful send and on New chat; a rejected send keeps them for correction. + ## Settings Important settings include: @@ -93,5 +103,6 @@ Add or update tests when changing: - named-source provenance behavior, - table rendering, - image intent routing, +- image attachment validation and multimodal payload shape, - MCP result normalization, - model discovery or settings behavior. diff --git a/public/components/assistant.html b/public/components/assistant.html index 0efc363..b3a2a6c 100644 --- a/public/components/assistant.html +++ b/public/components/assistant.html @@ -44,8 +44,11 @@

Loading conversation limit; history and draft are counted in UTF-16 code units. The server validates each request.

+
+ + diff --git a/public/css/assistant.css b/public/css/assistant.css index 79ecdab..efac079 100644 --- a/public/css/assistant.css +++ b/public/css/assistant.css @@ -56,6 +56,14 @@ .assistant-cite { display:inline-flex; align-items:center; justify-content:center; min-width:18px; height:18px; padding:0 6px; margin:0 1px; border-radius:999px; background:var(--purple-light); color:var(--purple); font-size:10px; font-weight:800; text-decoration:none; vertical-align:baseline; border:1px solid rgba(124,58,237,.18); text-transform:uppercase; letter-spacing:.03em; } .assistant-cite:hover { background:var(--purple); color:white; text-decoration:none; } .assistant-composer { border-top:1px solid var(--g200); padding:12px; background:white; display:grid; gap:8px; } +.assistant-attachments { display:flex; flex-wrap:wrap; gap:8px; } +.assistant-attachments[hidden] { display:none; } +.assistant-attachment { position:relative; width:64px; height:64px; border:1px solid var(--g200); border-radius:8px; overflow:hidden; background:var(--g50); } +.assistant-attachment img { width:100%; height:100%; object-fit:cover; display:block; } +.assistant-attachment-remove { position:absolute; top:3px; right:3px; width:18px; height:18px; border:none; border-radius:50%; background:rgba(0,0,0,.65); color:#fff; font-size:10px; line-height:18px; cursor:pointer; padding:0; } +.assistant-attachment-remove:hover { background:rgba(0,0,0,.85); } +.assistant-attach { display:inline-flex; align-items:center; gap:6px; font-size:12px; color:var(--g600); cursor:pointer; padding:6px 10px; border:1px dashed var(--g300); border-radius:8px; } +.assistant-attach:hover { border-color:var(--blue); color:var(--blue); } .assistant-composer textarea, .assistant-side textarea { width:100%; border:1.5px solid var(--g300); border-radius:10px; padding:10px 12px; resize:vertical; font-family:inherit; font-size:13px; outline:none; } .assistant-composer textarea:focus, .assistant-side textarea:focus { border-color:var(--blue); box-shadow:0 0 0 3px var(--blue-light); } .assistant-composer-footer { display:flex; justify-content:space-between; align-items:center; gap:10px; } diff --git a/public/js/assistant/api.js b/public/js/assistant/api.js index a597d0f..bd79843 100644 --- a/public/js/assistant/api.js +++ b/public/js/assistant/api.js @@ -103,3 +103,19 @@ export function deleteSavedAssistantChat(id) { credentials: 'same-origin' }).then(function(r) { return r.json(); }); } + +// Image attachments are input-only: they ride the outgoing clinical question +// payload and are never persisted or sent with handoff. The server enforces +// the same limits authoritatively; these mirror them for early feedback. +export var assistantAttachmentLimits = Object.freeze({ + mimeTypes: ['image/png', 'image/jpeg', 'image/webp'], + maxImages: 4, + maxImageBytes: 5 * 1024 * 1024, + maxTotalBytes: 10 * 1024 * 1024 +}); + +export function assistantAttachmentPayload(attachments) { + return (Array.isArray(attachments) ? attachments : []).map(function(attachment) { + return { dataBase64: attachment.dataBase64, mimeType: attachment.mimeType }; + }); +} diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js index babc3dd..48f2af7 100644 --- a/public/js/clinicalAssistant.js +++ b/public/js/clinicalAssistant.js @@ -20,10 +20,13 @@ import { openAssistantStream, startAssistantImageJob, saveAssistantChat, - requestAssistantHandoff + requestAssistantHandoff, + assistantAttachmentLimits, + assistantAttachmentPayload } from './assistant/api.js'; var initialized = false; var messages = []; + var attachments = []; // Input-only images: never persisted, never sent with handoff. var lastAnswer = ''; var lastSources = []; var mermaidReady = false; @@ -63,6 +66,7 @@ import { var exportBtn = document.getElementById('btn-assistant-export-pdf'); var imageBtn = document.getElementById('btn-assistant-image'); var imageClearBtn = document.getElementById('btn-assistant-image-clear'); + var attachInput = document.getElementById('assistant-attach-input'); var input = document.getElementById('assistant-input'); if (form) form.addEventListener('submit', onAsk); @@ -90,6 +94,7 @@ import { if (exportBtn) exportBtn.addEventListener('click', exportAnswerPdf); if (imageBtn) imageBtn.addEventListener('click', generateImage); if (imageClearBtn) imageClearBtn.addEventListener('click', clearGeneratedImage); + if (attachInput) attachInput.addEventListener('change', onAttachFiles); document.addEventListener('click', onAssistantDocumentClick); if (input) input.addEventListener('keydown', function (e) { if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') onAsk(e); @@ -162,16 +167,23 @@ import { var row = appendMessage('user', text); if (row && loading && loading.parentNode) loading.parentNode.insertBefore(row, loading); if (input) input.value = ''; + // Attached images are input-only and clear on a successful send. + attachments = []; + renderAttachments(); exporter.invalidate(); updateConversationBudget(); }; - return streamAssistantResponse({ + var payload = { message: text, idempotencyKey: crypto.randomUUID(), history: history, includeContext: !includeContext || includeContext.checked - }, loading, request) + }; + // Images ride the outgoing clinical question only, never handoff or saves. + if (attachments.length) payload.images = assistantAttachmentPayload(attachments); + + return streamAssistantResponse(payload, loading, request) .catch(function (err) { if (request.cancelled) return; setBusy(false, 'Error', true); @@ -386,6 +398,73 @@ import { return row; } + function onAttachFiles(e) { + var input = e && e.target; + var files = Array.prototype.slice.call(input && input.files ? input.files : []); + files.forEach(addAttachmentFile); + if (input) input.value = ''; // Allow re-selecting the same file later. + } + + function addAttachmentFile(file) { + var limits = assistantAttachmentLimits; + if (!file) return; + if (limits.mimeTypes.indexOf(file.type) === -1) { + if (typeof showToast === 'function') showToast('Only PNG, JPEG and WebP images can be attached; "' + String(file.name || 'attachment') + '" was ignored.', 'error'); + return; + } + if (typeof file.size === 'number' && file.size > limits.maxImageBytes) { + if (typeof showToast === 'function') showToast('Each image is limited to 5 MiB; "' + String(file.name || 'attachment') + '" was ignored.', 'error'); + return; + } + if (attachments.length >= limits.maxImages) { + if (typeof showToast === 'function') showToast('A maximum of 4 images can be attached to one question.', 'error'); + return; + } + var totalBytes = attachments.reduce(function(sum, a) { return sum + (a.size || 0); }, 0); + if (typeof file.size === 'number' && totalBytes + file.size > limits.maxTotalBytes) { + if (typeof showToast === 'function') showToast('Attached images are limited to 10 MiB in total.', 'error'); + return; + } + var reader = new FileReader(); + reader.onload = function() { + var dataUrl = String(reader.result || ''); + var comma = dataUrl.indexOf(','); + if (comma === -1 || dataUrl.indexOf(';base64,') === -1) { + if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error'); + return; + } + attachments.push({ dataBase64: dataUrl.slice(comma + 1), mimeType: file.type, size: file.size, src: dataUrl }); + renderAttachments(); + }; + reader.onerror = function() { + if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error'); + }; + reader.readAsDataURL(file); + } + + function renderAttachments() { + var wrap = document.getElementById('assistant-attachments'); + if (!wrap) return; + wrap.innerHTML = ''; + attachments.forEach(function(attachment, index) { + var item = document.createElement('div'); + item.className = 'assistant-attachment'; + var img = document.createElement('img'); + img.src = attachment.src || ('data:' + attachment.mimeType + ';base64,' + attachment.dataBase64); + img.alt = 'Attached image ' + (index + 1); + var remove = document.createElement('button'); + remove.type = 'button'; + remove.className = 'assistant-attachment-remove'; + remove.setAttribute('data-assistant-remove-attachment', String(index)); + remove.setAttribute('aria-label', 'Remove attached image ' + (index + 1)); + remove.textContent = '✕'; + item.appendChild(img); + item.appendChild(remove); + wrap.appendChild(item); + }); + wrap.hidden = !attachments.length; + } + function renderSuggestionButtons(suggestions) { var wrap = document.createElement('div'); wrap.className = 'assistant-suggestion-buttons'; @@ -522,6 +601,12 @@ import { if (bubble && Array.isArray(bubble.assistantSources)) renderSources(bubble.assistantSources); return; // The native anchor navigates to the matching source in the refreshed panel. } + var removeAttachment = e.target.closest('[data-assistant-remove-attachment]'); + if (removeAttachment) { + attachments.splice(Number(removeAttachment.getAttribute('data-assistant-remove-attachment')), 1); + renderAttachments(); + return; + } var loadBtn = e.target.closest('[data-assistant-load-chat]'); if (loadBtn) { e.preventDefault(); @@ -574,6 +659,8 @@ import { if (event && messages.length && !window.confirm('Start a new chat? Save or download this conversation first if you want to keep it.')) return; if (activeAssistantRequest) cancelAssistantSearch(); messages = []; + attachments = []; + renderAttachments(); lastAnswer = ''; lastSources = []; lastGeneratedImageSrc = ''; @@ -931,6 +1018,7 @@ import { var send = document.getElementById('btn-assistant-send'); var cancel = document.getElementById('btn-assistant-cancel'); var input = document.getElementById('assistant-input'); + var attachInput = document.getElementById('assistant-attach-input'); if (status) { status.classList.toggle('busy', !!isBusy); status.classList.toggle('error', !!isError); @@ -948,6 +1036,7 @@ import { cancel.style.display = canCancel ? 'inline-flex' : 'none'; } if (input) input.disabled = !!isBusy; + if (attachInput) attachInput.disabled = !!isBusy; } function sanitize(html) { return window.DOMPurify ? window.DOMPurify.sanitize(html, { ADD_ATTR: ['target'] }) : escapeHtml(html); } diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js index b749095..aedbd52 100644 --- a/src/routes/clinicalAssistant.js +++ b/src/routes/clinicalAssistant.js @@ -40,7 +40,7 @@ var { finalizeAssistantAnswer } = require('../utils/clinicalAnswer'); -var { conversationBudget, checkConversation, savedChatPayload } = require('../utils/clinicalConversation'); +var { conversationBudget, checkConversation, validateAttachments, savedChatPayload } = require('../utils/clinicalConversation'); var { DEFAULT_BEHAVIOR } = require('../utils/clinicalPrompts'); @@ -193,7 +193,8 @@ router.post('/clinical-assistant/chat', async function(req, res) { model: prepared.chatModel || undefined, temperature: 0.15, tools: imageTool.tools, - maxTokens: 2600 + maxTokens: 2600, + images: prepared.images })); ai = await imageTool.dispatch(ai, { owner: req.user.id, workflow: 'clinical_assistant', body: req.body, imageContext: prepared.imageContext, messages: prepared.messages, options: assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15 }), callAI: callAI }); @@ -201,7 +202,7 @@ router.post('/clinical-assistant/chat', async function(req, res) { messages: prepared.messages, chatModel: prepared.chatModel, callAI: callAI, - generationOptions: assistantGenerationOptions({ temperature: 0.15 }) + generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images }) }); var answer = finalized.answer; ai = finalized.ai; @@ -258,7 +259,8 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) { model: prepared.chatModel || undefined, temperature: 0.15, tools: imageTool.tools, - maxTokens: 2600 + maxTokens: 2600, + images: prepared.images }), function(delta) { sendEvent('token', { token: delta }); }); @@ -269,7 +271,7 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) { messages: prepared.messages, chatModel: prepared.chatModel, callAI: callAI, - generationOptions: assistantGenerationOptions({ temperature: 0.15 }), + generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images }), streamed: true, onRegenerating: function() { sendEvent('status', { message: 'Completing answer...' }); } }); @@ -354,6 +356,10 @@ async function prepareAssistantChat(body) { var checked = checkConversation(body.history, body.message, await getConversationLimit()); var message = checked.message; var history = checked.history; + // Image attachments are validated before any retrieval or provider call. + // The conversation budget counts the text only; images are excluded from + // the UTF-16 count and are never persisted in saved chats or transcripts. + var images = validateAttachments(body.images); var chatModel = await getSetting('clinical_assistant.chat_model', '') || await getSetting('models.default', ''); var searchLimit = clampInt(await getSetting('clinical_assistant.search_limit', '8'), 3, 20, 8); @@ -416,6 +422,7 @@ async function prepareAssistantChat(body) { var context = formatSourcesForPrompt(sources); return { message: message, + images: images, imageContext: generatedImages.imageContext(message, history), chatModel: chatModel, sources: sources, diff --git a/src/utils/ai.js b/src/utils/ai.js index a5560c5..66acd6a 100644 --- a/src/utils/ai.js +++ b/src/utils/ai.js @@ -139,6 +139,35 @@ if (activeProvider === 'openrouter' && !openrouter) { console.log('🤖 Active AI provider:', activeProvider); +// ============================================================ +// IMAGE ATTACHMENTS — multimodal content for the latest user message. +// Only OpenAI-compatible providers accept content-part messages. The legacy +// direct adapters (Bedrock/Vertex) cannot take image parts, so requests with +// images are refused with a clear 400 before any provider contact. +// ============================================================ +var MULTIMODAL_PROVIDERS = ['litellm', 'openrouter', 'azure']; + +function applyImageAttachments(messages, images) { + if (!Array.isArray(images) || !images.length) return messages; + if (MULTIMODAL_PROVIDERS.indexOf(activeProvider) === -1) { + var unsupported = new Error('Image attachments require an OpenAI-compatible provider (LiteLLM, OpenRouter, or Azure). The active provider cannot accept image input, so no request was sent.'); + unsupported.statusCode = 400; + unsupported.code = 'IMAGES_UNSUPPORTED_PROVIDER'; + throw unsupported; + } + var out = messages.slice(); + var index = out.length - 1; + while (index >= 0 && !(out[index] && out[index].role === 'user')) index--; + if (index < 0) return out; // No user message to attach to. + var message = out[index]; + var content = Array.isArray(message.content) ? message.content.slice() : [{ type: 'text', text: String(message.content || '') }]; + images.forEach(function (image) { + content.push({ type: 'image_url', image_url: { url: 'data:' + image.mimeType + ';base64,' + image.dataBase64 } }); + }); + out[index] = Object.assign({}, message, { content: content }); + return out; +} + // Preserve unrecognized provider statuses; only explicit successful stops are complete. function normalizeFinishReason(reason) { if (typeof reason !== 'string') return reason ?? null; @@ -455,6 +484,8 @@ async function callAIStream(messages, options, onToken) { await assertModelAllowed(model, options); } + messages = applyImageAttachments(messages, options.images); + var client = null; var provider = null; if (activeProvider === 'litellm' && litellmClient) { @@ -527,6 +558,8 @@ async function callAI(messages, options) { await assertModelAllowed(model, options); } + messages = applyImageAttachments(messages, options.images); + try { var result; @@ -731,4 +764,4 @@ async function discoverModels() { return discovered; } -module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient }; +module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient, applyImageAttachments }; diff --git a/src/utils/clinicalConversation.js b/src/utils/clinicalConversation.js index dd4bd13..852c406 100644 --- a/src/utils/clinicalConversation.js +++ b/src/utils/clinicalConversation.js @@ -1,4 +1,7 @@ // Conversation text uses an exact character budget, not an estimated model token limit. +// Image attachments are excluded from this budget: UTF-16 code units count the +// text only. Attachments are validated separately below and are input-only — +// they are never persisted in saved chats or transcripts. const DEFAULT_CONVERSATION_CHARS = 120000; const MAX_SAVED_CHAT_BYTES = 8 * 1024 * 1024; @@ -94,6 +97,45 @@ function savedJobs(imageJobs) { return imageJobs.map(job => ({ jobId: job.jobId })); } +// Image attachments accompany the outgoing clinical question only. +// Strict limits: PNG/JPEG/WebP, canonical base64, <= 4 images, +// <= 5 MiB decoded per image, <= 10 MiB decoded total. Invalid input is +// rejected before any retrieval or provider call. The conversation budget +// above counts the text only; image bytes never enter the UTF-16 count. +const ATTACHMENT_MIME_TYPES = ['image/png', 'image/jpeg', 'image/webp']; +const MAX_ATTACHMENT_IMAGES = 4; +const MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024; +const MAX_ATTACHMENT_TOTAL_BYTES = 10 * 1024 * 1024; +const CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; + +function validateAttachments(images) { + if (images === undefined || images === null) return []; + if (!Array.isArray(images)) throw failure('Image attachments must be a list.', 400, 'INVALID_ATTACHMENTS'); + if (images.length > MAX_ATTACHMENT_IMAGES) throw failure('A maximum of 4 images can be attached to one question.', 400, 'INVALID_ATTACHMENTS'); + let total = 0; + return images.map(function (image) { + if (!image || typeof image !== 'object' || Array.isArray(image) || + typeof image.dataBase64 !== 'string' || typeof image.mimeType !== 'string') { + throw failure('Each image attachment needs a base64 payload and a MIME type.', 400, 'INVALID_ATTACHMENTS'); + } + if (!ATTACHMENT_MIME_TYPES.includes(image.mimeType)) { + throw failure('Only PNG, JPEG and WebP image attachments are allowed.', 400, 'INVALID_ATTACHMENTS'); + } + if (image.dataBase64.length % 4 !== 0 || !CANONICAL_BASE64.test(image.dataBase64)) { + throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS'); + } + const decoded = Buffer.from(image.dataBase64, 'base64'); + if (decoded.toString('base64') !== image.dataBase64) { + throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS'); + } + if (decoded.length === 0) throw failure('Image attachments must not be empty.', 400, 'INVALID_ATTACHMENTS'); + if (decoded.length > MAX_ATTACHMENT_BYTES) throw failure('Each image attachment is limited to 5 MiB.', 400, 'INVALID_ATTACHMENTS'); + total += decoded.length; + if (total > MAX_ATTACHMENT_TOTAL_BYTES) throw failure('Image attachments are limited to 10 MiB in total.', 400, 'INVALID_ATTACHMENTS'); + return { dataBase64: image.dataBase64, mimeType: image.mimeType }; + }); +} + function savedChatPayload(body) { const messages = validateMessages(body.messages).map(function(message, index) { const original = body.messages[index]; @@ -128,4 +170,4 @@ function savedChatPayload(body) { return payload; } -module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, savedChatPayload }; +module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, validateAttachments, savedChatPayload }; diff --git a/test/assistant-image-attachments.test.js b/test/assistant-image-attachments.test.js new file mode 100644 index 0000000..4afbb2f --- /dev/null +++ b/test/assistant-image-attachments.test.js @@ -0,0 +1,411 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const express = require('express'); +const { JSDOM } = require('jsdom'); +const { pathToFileURL } = require('node:url'); +const policy = require('../src/utils/clinicalConversation'); +const answer = require('../src/utils/clinicalAnswer'); +const root = path.join(__dirname, '..'); +const read = file => fs.readFileSync(path.join(root, file), 'utf8'); +const quiet = { log() {}, warn() {}, error() {}, info() {} }; +const b64 = bytes => Buffer.from(bytes).toString('base64'); +const canonical = (length, fill = 7) => Buffer.alloc(length, fill).toString('base64'); +const png = { dataBase64: canonical(16), mimeType: 'image/png' }; +const jpeg = { dataBase64: canonical(32), mimeType: 'image/jpeg' }; +const webp = { dataBase64: canonical(48), mimeType: 'image/webp' }; + +function server(options = {}) { + const calls = { ai: [], search: [], writes: [], health: [] }; + let saved; + const db = { + async getSetting(key) { + if (options.dbError) throw Error('private diagnostic'); + if (key.endsWith('conversation_chars')) return options.legacyLimit ?? 'broken'; + return options.model || null; + }, + async get(sql, params) { + if (sql.includes('COUNT')) return { cnt: 0 }; + return saved && params[1] === 7 ? { id: 1, title: saved[1], payload: saved[2] } : null; + }, + async run(sql, params) { calls.writes.push(sql); saved = params; return { lastInsertRowid: 1 }; } + }; + const ai = async (messages, settings) => { + calls.ai.push({ messages, settings }); + return { content: 'Complete supported answer. [1]', finishReason: 'stop', model: 'synthetic' }; + }; + const source = { number: 1, title: 'Synthetic source', excerpt: 'Synthetic reference.', page: 9 }; + const mocks = { + express, axios: { async post(url, payload) { calls.images.push(payload); return { data: { data: [{ b64_json: 'c3ludGhldGlj' }] } }; } }, crypto: require('node:crypto'), '../db/database': db, + '../middleware/auth': { authMiddleware() {} }, '../utils/ai': options.ai || { callAI: ai, callAIStream: ai }, + '../utils/errors': { gatewayUrl: path => 'http://synthetic.invalid' + path }, '../utils/litellm': { getLiteLLMHeaders: () => ({}) }, '../utils/logger': { audit() {}, error() {} }, + '../utils/crypto': { encryptString: value => 'encrypted:' + value, decryptString: value => value.replace(/^encrypted:/, '') }, + '../utils/redis': { async getJson() { return null; }, async setJson() {} }, '../utils/clinicalPromptPool': { createClinicalPromptPool: () => ({}) }, + '../utils/generatedImages': require('../src/utils/generatedImages'), + '../utils/generatedImageLinks': require('../src/utils/generatedImageLinks'), + '../utils/imageTool': require('../src/utils/imageTool'), + '../utils/clinicalPrompts': require('../src/utils/clinicalPrompts'), + '../utils/clinicalMcpClient': { + async semanticSearch(query) { calls.search.push(query); return {}; }, async getMcpHealth() { calls.health.push('health'); return {}; } + }, + '../utils/clinicalRetrieval': { + cleanSourceExcerpt: require('../src/utils/clinicalRetrieval').cleanSourceExcerpt, + normalizeMcpSearchResponse: () => options.noSources ? [] : [source], + normalizeMcpMultimodalResponse: () => [], dedupeSources: value => value, + isVisualSourceQuery: () => false, classifyAndRerankMultimodalResults: async () => [] + }, + '../utils/clinicalAnswer': answer, '../utils/clinicalConversation': policy + }; + const module = { exports: {} }; + vm.runInNewContext(read('src/routes/clinicalAssistant.js'), { + module, exports: module.exports, console: quiet, Buffer, Map, + process: { env: { CLINICAL_ASSISTANT_MCP_WARMUP: 'false', CLINICAL_ASSISTANT_CONVERSATION_CHARS: options.limit, LITELLM_API_BASE: 'http://synthetic.invalid' } }, setTimeout() {}, + require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; } + }); + async function request(method, endpoint, body, userId = 7) { + const handler = module.exports.stack.find(layer => layer.route && layer.route.path === endpoint && layer.route.methods[method]).route.stack.find(layer => layer.method === method).handle; + const res = { + statusCode: 200, headers: {}, events: '', body: null, + status(code) { this.statusCode = code; return this; }, json(body) { this.body = body; return this; }, + setHeader(k, v) { this.headers[k] = v; }, write(text) { this.events += text; }, end() {}, flushHeaders() {} + }; + await handler({ body, params: { id: 1 }, user: { id: userId } }, res); + return res; + } + return { calls, request }; +} + +test('attachment policy validates MIME, canonical base64, per-image/count/total limits without provider contact', async () => { + const oversize = { dataBase64: canonical(5 * 1024 * 1024 + 1), mimeType: 'image/png' }; + const big = { dataBase64: canonical(4 * 1024 * 1024), mimeType: 'image/jpeg' }; + const invalid = [ + { images: [{ ...png, mimeType: 'image/svg+xml' }] }, + { images: [{ ...png, mimeType: 'image/gif' }] }, + { images: [{ ...png, mimeType: 'application/octet-stream' }] }, + { images: [{ ...png, dataBase64: 'not base64!!!' }] }, + { images: [{ ...png, dataBase64: 'SGVsbG8' }] }, + { images: [{ ...png, dataBase64: 'AB==' }] }, // non-canonical trailing bits + { images: [{ ...png, dataBase64: '' }] }, + { images: [{ ...png, dataBase64: 42 }] }, + { images: [{ mimeType: 'image/png' }] }, + { images: [{ dataBase64: png.dataBase64 }] }, + { images: [null] }, { images: ['png'] }, { images: [{}] }, + { images: 'not-a-list' }, + { images: [png, jpeg, webp, png, jpeg] }, + { images: [oversize] }, + { images: [big, big, big] } + ]; + for (const body of invalid) { + for (const endpoint of ['/clinical-assistant/chat', '/clinical-assistant/chat/stream']) { + const app = server(); + const result = await app.request('post', endpoint, { message: 'What is the recommended dose of acetaminophen for a febrile three year old?', history: [], ...body }); + assert.equal(result.statusCode, 400, JSON.stringify(body.images) + ' on ' + endpoint); + assert.equal(result.body.code, 'INVALID_ATTACHMENTS'); + assert.equal(app.calls.ai.length + app.calls.search.length, 0, 'no provider or retrieval call on ' + endpoint); + } + } +}); + +test('valid images are normalized, ride the outgoing question only, and are excluded from the UTF-16 text budget', async () => { + const app = server({ limit: '1000' }); + const images = [{ ...png, extra: 'ignored' }, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }]; + const question = 'What about monitoring?'; + const result = await app.request('post', '/clinical-assistant/chat', { message: question, history: [{ role: 'user', content: 'x'.repeat(500) }], images }); + assert.equal(result.statusCode, 200); + assert.equal(app.calls.search.length, 1); + assert.equal(app.calls.ai.length, 2, 'rewrite plus generation'); + const generation = app.calls.ai.at(-1); + assert.deepEqual(generation.settings.images, [png, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }], 'normalized, extra fields dropped'); + assert.equal(typeof generation.messages[1].content, 'string', 'the route keeps prompt text; ai.js builds multimodal parts'); + assert.ok(generation.messages[1].content.includes(question)); + assert.equal(app.calls.ai[0].settings.images, undefined, 'the retrieval rewrite is text-only'); + + const atLimit = server({ limit: '1000' }); + const budgetBody = { message: 'x', history: [{ role: 'user', content: 'x'.repeat(999) }], images: [png, jpeg, webp, png] }; + assert.equal((await atLimit.request('post', '/clinical-assistant/chat', budgetBody)).statusCode, 200, 'images are excluded from the UTF-16 budget'); + assert.equal((await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' })).statusCode, 413, 'the text boundary is unchanged by images'); + const over = await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' }); + assert.equal(over.body.budget.unit, 'characters'); +}); + +test('greeting and retrieval-empty direct responses validate attachments before responding', async () => { + const bad = server(); + const rejected = await bad.request('post', '/clinical-assistant/chat/stream', { message: 'thanks', history: [], images: [{ dataBase64: canonical(8), mimeType: 'image/bmp' }] }); + assert.equal(rejected.statusCode, 400); + assert.equal(bad.calls.ai.length + bad.calls.search.length, 0); + const good = server(); + const direct = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] }); + assert.equal(direct.statusCode, 200); + assert.match(direct.body.answer, /clinical question/); + assert.equal(good.calls.ai.length + good.calls.search.length, 0, 'greeting never reaches providers'); +}); + +test('handoff ignores image attachments and never forwards them', async () => { + const app = server(); + const result = await app.request('post', '/clinical-assistant/handoff', { history: [{ role: 'user', content: 'Known facts.' }], images: [{ dataBase64: canonical(8), mimeType: 'image/png' }] }); + assert.equal(result.statusCode, 200); + assert.equal(app.calls.ai.length, 1); + assert.ok(app.calls.ai[0].messages[1].content.includes('Known facts.')); + assert.equal(app.calls.ai[0].settings.images, undefined); + assert.doesNotMatch(JSON.stringify(app.calls.ai[0].messages), /base64/); +}); + +function openaiAdapter() { + const requests = []; + class OpenAI { + constructor() { + this.chat = { completions: { create: async r => { + requests.push(r); + if (r.stream) return [{ choices: [{ delta: { content: 'streamed answer' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }]; + return { choices: [{ message: { content: 'Synthetic multimodal answer.' }, finish_reason: 'stop' }], usage: null }; + } } }; + } + } + const mocks = { + openai: { OpenAI }, + './models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['synthetic-vision']); } }, + './generationOptions': require('../src/utils/generationOptions'), + './logger': { apiCall() {}, error() {} }, + '../db/database': { async getSetting() { return null; } } + }; + const module = { exports: {} }; + vm.runInNewContext(read('src/utils/ai.js'), { + module, exports: module.exports, console: quiet, TextDecoder, + process: { env: { LITELLM_API_BASE: 'http://synthetic.invalid', LITELLM_API_KEY: 'sk-synthetic' } }, + require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; } + }); + return { requests, ai: module.exports }; +} + +test('LiteLLM generation builds multimodal parts only on the latest user message and keeps tools/temperature/streaming', async () => { + const tools = [{ type: 'function', function: { name: 'generate_image', parameters: { type: 'object' } } }]; + const { requests, ai } = openaiAdapter(); + const messages = [ + { role: 'system', content: 'System behavior.' }, + { role: 'user', content: 'Earlier turn.' }, + { role: 'assistant', content: 'Earlier answer.' }, + { role: 'user', content: 'Latest question.' } + ]; + const result = await ai.callAI(messages, { model: 'synthetic-vision', images: [png, jpeg], tools, temperature: 0.15, maxTokens: 2600 }); + assert.equal(result.content, 'Synthetic multimodal answer.'); + const request = requests[0]; + assert.equal(request.model, 'synthetic-vision'); + assert.equal(request.temperature, 0.15); + assert.equal(request.max_tokens, 2600); + assert.deepEqual(request.tools, tools); + assert.equal(request.tool_choice, 'auto'); + assert.equal(request.parallel_tool_calls, false); + assert.deepEqual(request.messages[0], { role: 'system', content: 'System behavior.' }); + assert.deepEqual(request.messages[1], { role: 'user', content: 'Earlier turn.' }); + assert.deepEqual(request.messages[2], { role: 'assistant', content: 'Earlier answer.' }); + assert.equal(request.messages[3].role, 'user'); + assert.equal(request.messages[3].content[0].type, 'text'); + assert.equal(request.messages[3].content[0].text, 'Latest question.'); + assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[1])), { type: 'image_url', image_url: { url: 'data:image/png;base64,' + png.dataBase64 } }); + assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[2])), { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,' + jpeg.dataBase64 } }); + + const streamed = await ai.callAIStream(messages, { model: 'synthetic-vision', images: [png], tools, temperature: 0.15, maxTokens: 2600 }, () => {}); + assert.equal(streamed.content, 'streamed answer'); + const streamRequest = requests[1]; + assert.equal(streamRequest.stream, true); + assert.deepEqual(streamRequest.tools, tools); + assert.equal(streamRequest.messages[3].content.length, 2); + assert.deepEqual(JSON.parse(JSON.stringify(streamRequest.messages[3].content[1].image_url)), { url: 'data:image/png;base64,' + png.dataBase64 }); + + const noImages = await ai.callAI(messages, { model: 'synthetic-vision' }); + assert.equal(noImages.content, 'Synthetic multimodal answer.'); + assert.deepEqual(requests[2].messages[3], { role: 'user', content: 'Latest question.' }); +}); + +test('legacy direct provider adapters refuse image attachments with 400 before contacting the provider', async () => { + const sdkCalls = []; + class InvokeModelCommand { constructor(input) { this.input = input; } } + class ConverseCommand { constructor(input) { this.input = input; } } + const mocks = { + openai: { OpenAI: class { constructor() { throw Error('Unexpected OpenAI client'); } } }, + './models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['anthropic.claude-synthetic', 'amazon/nova-lite', 'google/gemini-2.5-flash']); } }, + './generationOptions': require('../src/utils/generationOptions'), + './logger': { apiCall() {}, error() {} }, + '../db/database': { async getSetting() { return null; } }, + '@aws-sdk/client-bedrock-runtime': { + InvokeModelCommand, ConverseCommand, + BedrockRuntimeClient: class { async send(command) { sdkCalls.push(command); return { body: Buffer.from('{}') }; } } + }, + '@google-cloud/vertexai': { + VertexAI: class { getGenerativeModel() { return { async generateContent(request) { sdkCalls.push(request); return { response: { candidates: [] } }; } }; } } + } + }; + const module = { exports: {} }; + vm.runInNewContext(read('src/utils/ai.js'), { + module, exports: module.exports, console: quiet, TextDecoder, + process: { env: { AI_PROVIDER: 'bedrock', AWS_BEDROCK_REGION: 'synthetic' } }, + require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; } + }); + const messages = [{ role: 'user', content: 'Question with image.' }]; + await assert.rejects(module.exports.callAI(messages, { model: 'anthropic.claude-synthetic', images: [png] }), error => { + assert.equal(error.statusCode, 400); + assert.equal(error.code, 'IMAGES_UNSUPPORTED_PROVIDER'); + assert.match(error.message, /OpenAI-compatible provider/); + return true; + }); + await assert.rejects(module.exports.callAIStream(messages, { model: 'anthropic.claude-synthetic', images: [png] }, () => {}), error => error.statusCode === 400); + assert.equal(sdkCalls.length, 0, 'refused before any provider contact'); + + const vertexModule = { exports: {} }; + vm.runInNewContext(read('src/utils/ai.js'), { + module: vertexModule, exports: vertexModule.exports, console: quiet, TextDecoder, + process: { env: { AI_PROVIDER: 'vertex', GOOGLE_VERTEX_PROJECT: 'synthetic' } }, + require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; } + }); + await assert.rejects(vertexModule.exports.callAI(messages, { model: 'google/gemini-2.5-flash', images: [png] }), error => error.statusCode === 400); + assert.equal(sdkCalls.length, 0); +}); + +function browserUI(options = {}) { + const dom = new JSDOM('
' + read('public/components/assistant.html') + '
', { + url: 'https://example.test/', runScripts: 'outside-only' + }); + const calls = { stream: [], handoff: [], save: [] }; + const toasts = []; + const escapeHtml = text => String(text).replace(/&/g, '&').replace(//g, '>'); + const context = { + window: dom.window, document: dom.window.document, navigator: dom.window.navigator, + console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require('node:crypto').webcrypto, + FileReader: dom.window.FileReader, File: dom.window.File, + setTimeout() {}, showToast(text, kind) { toasts.push([String(text), kind || '']); }, escapeHtml, escapeAttr: escapeHtml, + renderAssistantMarkdown: text => escapeHtml(text), renderSourcesList: () => '', EMPTY_PROMPT_SETS: [[]], + createAssistantExporter: () => ({ invalidate() {}, exportAnswerPdf() {} }), + createAssistantImageStore: () => ({ renderGeneratedImage: src => '', clear() {} }), + isImageRequest: () => false, + fetchAssistantStatus: async () => ({ success: true, conversationChars: options.limit || 120000 }), + fetchAssistantExamples: async () => ({}), fetchSavedAssistantChats: async () => ({ success: true, chats: [] }), + openAssistantStream: async payload => { + calls.stream.push(payload); + if (options.stream) return options.stream(payload); + return new Response('event: done\ndata: ' + JSON.stringify({ success: true, answer: 'Complete response.', sources: [] }) + '\n\n'); + }, + fetchAssistantChat: async payload => ({ success: true, answer: 'Fallback answer.' }), + saveAssistantChat: async payload => { calls.save.push(payload); return { success: true }; }, + requestAssistantHandoff: async history => { calls.handoff.push(history); return { success: true, summary: 'Explicit handoff.' }; } + }; + vm.createContext(context); + vm.runInContext(read('public/js/clinicalAssistant.js').replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, ''), context); + context.bindEvents(); + context.conversationChars = options.limit || 120000; + return { context, document: dom.window.document, calls, toasts, dom }; +} + +async function realApiHelpers() { + const module = await import(pathToFileURL(path.join(root, 'public/js/assistant/api.js')).href); + return module; +} + +function attachFile(ui, name, type, bytes) { + const input = ui.document.getElementById('assistant-attach-input'); + const file = new ui.dom.window.File([bytes], name, { type }); + Object.defineProperty(input, 'files', { value: [file], configurable: true }); + input.dispatchEvent(new ui.dom.window.Event('change')); +} + +async function tick() { for (let i = 0; i < 25; i++) await new Promise(resolve => setImmediate(resolve)); } + +test('UI attaches only PNG/JPEG/WebP with thumbnails, removal and client-side limits', async () => { + const api = await realApiHelpers(); + const ui = browserUI(); + ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits; + ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload; + const bytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]); + attachFile(ui, 'photo.png', 'image/png', bytes); + attachFile(ui, 'scan.jpeg', 'image/jpeg', bytes); + await tick(); + const wrap = ui.document.getElementById('assistant-attachments'); + assert.equal(wrap.hidden, false); + assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2); + assert.ok(wrap.querySelector('img').getAttribute('src').startsWith('data:image/png;base64,')); + + attachFile(ui, 'notes.txt', 'text/plain', new Uint8Array([1, 2, 3])); + await tick(); + assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2); + assert.ok(ui.toasts.some(([text]) => /PNG, JPEG and WebP/.test(text))); + + attachFile(ui, 'third.webp', 'image/webp', bytes); + attachFile(ui, 'fourth.png', 'image/png', bytes); + await tick(); + assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4); + attachFile(ui, 'fifth.png', 'image/png', bytes); + await tick(); + assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4); + assert.ok(ui.toasts.some(([text]) => /maximum of 4/.test(text))); + + attachFile(ui, 'huge.png', 'image/png', new Uint8Array(5 * 1024 * 1024 + 1)); + await tick(); + assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4); + assert.ok(ui.toasts.some(([text]) => /5 MiB/.test(text))); + + wrap.querySelector('[data-assistant-remove-attachment="0"]').click(); + await tick(); + assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 3); + assert.equal(ui.context.attachments.length, 3); + ui.dom.window.close(); +}); + +test('UI sends images only with the question, clears them on success, and keeps them out of handoff, saves and the budget label', async () => { + const api = await realApiHelpers(); + const ui = browserUI({ limit: 1000 }); + ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits; + ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload; + const bytes = new Uint8Array([1, 2, 3, 4]); + attachFile(ui, 'photo.png', 'image/png', bytes); + await tick(); + const expected = [{ dataBase64: Buffer.from(bytes).toString('base64'), mimeType: 'image/png' }]; + + ui.context.restoreSavedChat({ messages: [{ role: 'user', content: 'x'.repeat(999) }], lastAnswer: '' }); + const input = ui.document.getElementById('assistant-input'); + input.value = 'x'; + input.dispatchEvent(new ui.dom.window.Event('input')); + const label = ui.document.getElementById('assistant-context-budget'); + assert.equal(label.textContent, '1,000 / 1,000 conversation characters (UTF-16 code units).', 'budget label stays text-only'); + assert.doesNotMatch(label.textContent, /image/i); + + // Handoff and saved chats never carry attachments. + await ui.context.requestHandoff(); + assert.equal(ui.calls.handoff.length, 1); + assert.equal(ui.calls.handoff[0].images, undefined); + assert.equal(JSON.stringify(ui.calls.handoff[0]).includes('dataBase64'), false); + await ui.context.saveCurrentChat(); + assert.equal(ui.calls.save[0].images, undefined); + assert.equal(JSON.stringify(ui.calls.save[0]).includes('dataBase64'), false); + + await ui.context.onAsk(); + await tick(); + assert.deepEqual(JSON.parse(JSON.stringify(ui.calls.stream[0].images)), expected); + assert.equal(ui.context.attachments.length, 0, 'cleared on successful send'); + assert.equal(ui.document.getElementById('assistant-attachments').hidden, true); + assert.equal(ui.context.messages.length, 3, 'question and answer appended'); + + // New chat clears any later attachments. + attachFile(ui, 'again.png', 'image/png', bytes); + await tick(); + assert.equal(ui.context.attachments.length, 1); + await ui.context.clearConversation(); + assert.equal(ui.context.attachments.length, 0); + assert.equal(ui.document.getElementById('assistant-attachments').hidden, true); + + attachFile(ui, 'kept.png', 'image/png', bytes); + await tick(); + assert.equal(ui.context.attachments.length, 1); + const failing = browserUI({ stream: () => new Response(JSON.stringify({ error: 'Image rejected' }), { status: 400 }) }); + failing.context.assistantAttachmentLimits = api.assistantAttachmentLimits; + failing.context.assistantAttachmentPayload = api.assistantAttachmentPayload; + attachFile(failing, 'kept.png', 'image/png', bytes); + await tick(); + failing.document.getElementById('assistant-input').value = 'Question with attachment'; + await failing.context.onAsk(); + await tick(); + assert.equal(failing.context.attachments.length, 1, 'rejected send keeps the attachments for correction'); + assert.equal(failing.document.getElementById('assistant-attachments').hidden, false); + ui.dom.window.close(); + failing.dom.window.close(); +});