const { test } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const vm = require('node:vm'); const express = require('express'); const { JSDOM } = require('jsdom'); const { pathToFileURL } = require('node:url'); const policy = require('../src/utils/clinicalConversation'); const answer = require('../src/utils/clinicalAnswer'); const root = path.join(__dirname, '..'); const read = file => fs.readFileSync(path.join(root, file), 'utf8'); const quiet = { log() {}, warn() {}, error() {}, info() {} }; const b64 = bytes => Buffer.from(bytes).toString('base64'); const canonical = (length, fill = 7) => Buffer.alloc(length, fill).toString('base64'); const png = { dataBase64: canonical(16), mimeType: 'image/png' }; const jpeg = { dataBase64: canonical(32), mimeType: 'image/jpeg' }; const webp = { dataBase64: canonical(48), mimeType: 'image/webp' }; function server(options = {}) { const calls = { ai: [], search: [], writes: [], health: [] }; let saved; const db = { async getSetting(key) { if (options.dbError) throw Error('private diagnostic'); if (key.endsWith('conversation_chars')) return options.legacyLimit ?? 'broken'; return options.model || null; }, async get(sql, params) { if (sql.includes('COUNT')) return { cnt: 0 }; return saved && params[1] === 7 ? { id: 1, title: saved[1], payload: saved[2] } : null; }, async run(sql, params) { calls.writes.push(sql); saved = params; return { lastInsertRowid: 1 }; } }; const ai = async (messages, settings) => { calls.ai.push({ messages, settings }); return { content: 'Complete supported answer. [1]', finishReason: 'stop', model: 'synthetic' }; }; const source = { number: 1, title: 'Synthetic source', excerpt: 'Synthetic reference.', page: 9 }; const mocks = { express, axios: { async post(url, payload) { if (String(url).includes('/translate')) return { data: { translatedText: 'Synthetic translation.' } }; calls.images.push(payload); return { data: { data: [{ b64_json: 'c3ludGhldGlj' }] } }; } }, crypto: require('node:crypto'), '../db/database': db, '../middleware/auth': { authMiddleware() {} }, './auth': { __sendEmail: async () => false }, '../utils/ai': options.ai || { callAI: ai, callAIStream: ai }, '../utils/errors': { gatewayUrl: path => 'http://synthetic.invalid' + path }, '../utils/litellm': { getLiteLLMHeaders: () => ({}) }, '../utils/logger': { audit() {}, error() {} }, '../utils/crypto': { encryptString: value => 'encrypted:' + value, decryptString: value => value.replace(/^encrypted:/, '') }, '../utils/redis': { async getJson() { return null; }, async setJson() {} }, '../utils/clinicalPromptPool': { createClinicalPromptPool: () => ({}) }, '../utils/generatedImages': require('../src/utils/generatedImages'), '../utils/generatedImageLinks': require('../src/utils/generatedImageLinks'), '../utils/imageTool': require('../src/utils/imageTool'), '../utils/clinicalPrompts': require('../src/utils/clinicalPrompts'), '../utils/clinicalMcpClient': { async semanticSearch(query) { calls.search.push(query); return {}; }, async getMcpHealth() { calls.health.push('health'); return {}; } }, '../utils/clinicalRetrieval': { cleanSourceExcerpt: require('../src/utils/clinicalRetrieval').cleanSourceExcerpt, normalizeMcpSearchResponse: () => options.noSources ? [] : [source], normalizeMcpMultimodalResponse: () => [], dedupeSources: value => value, isVisualSourceQuery: () => false, classifyAndRerankMultimodalResults: async () => [] }, '../utils/clinicalAnswer': answer, '../utils/clinicalConversation': policy, '../utils/clinicalTranslation': require('../src/utils/clinicalTranslation'), '../utils/patientTakehome': require('../src/utils/patientTakehome') }; const module = { exports: {} }; vm.runInNewContext(read('src/routes/clinicalAssistant.js'), { module, exports: module.exports, console: quiet, Buffer, Map, process: { env: { CLINICAL_ASSISTANT_MCP_WARMUP: 'false', CLINICAL_ASSISTANT_CONVERSATION_CHARS: options.limit, LITELLM_API_BASE: 'http://synthetic.invalid' } }, setTimeout() {}, require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; } }); async function request(method, endpoint, body, userId = 7) { const handler = module.exports.stack.find(layer => layer.route && layer.route.path === endpoint && layer.route.methods[method]).route.stack.find(layer => layer.method === method).handle; const res = { statusCode: 200, headers: {}, events: '', body: null, status(code) { this.statusCode = code; return this; }, json(body) { this.body = body; return this; }, setHeader(k, v) { this.headers[k] = v; }, write(text) { this.events += text; }, end() {}, flushHeaders() {} }; await handler({ body, params: { id: 1 }, user: { id: userId } }, res); return res; } return { calls, request }; } test('attachment policy validates MIME, canonical base64, per-image/count/total limits without provider contact', async () => { const oversize = { dataBase64: canonical(5 * 1024 * 1024 + 1), mimeType: 'image/png' }; const big = { dataBase64: canonical(4 * 1024 * 1024), mimeType: 'image/jpeg' }; const invalid = [ { images: [{ ...png, mimeType: 'image/svg+xml' }] }, { images: [{ ...png, mimeType: 'image/gif' }] }, { images: [{ ...png, mimeType: 'application/octet-stream' }] }, { images: [{ ...png, dataBase64: 'not base64!!!' }] }, { images: [{ ...png, dataBase64: 'SGVsbG8' }] }, { images: [{ ...png, dataBase64: 'AB==' }] }, // non-canonical trailing bits { images: [{ ...png, dataBase64: '' }] }, { images: [{ ...png, dataBase64: 42 }] }, { images: [{ mimeType: 'image/png' }] }, { images: [{ dataBase64: png.dataBase64 }] }, { images: [null] }, { images: ['png'] }, { images: [{}] }, { images: 'not-a-list' }, { images: [png, jpeg, webp, png, jpeg] }, { images: [oversize] }, { images: [big, big, big] } ]; for (const body of invalid) { for (const endpoint of ['/clinical-assistant/chat', '/clinical-assistant/chat/stream']) { const app = server(); const result = await app.request('post', endpoint, { message: 'What is the recommended dose of acetaminophen for a febrile three year old?', history: [], ...body }); assert.equal(result.statusCode, 400, JSON.stringify(body.images) + ' on ' + endpoint); assert.equal(result.body.code, 'INVALID_ATTACHMENTS'); assert.equal(app.calls.ai.length + app.calls.search.length, 0, 'no provider or retrieval call on ' + endpoint); } } }); test('valid images are normalized, ride the outgoing question only, and are excluded from the UTF-16 text budget', async () => { const app = server({ limit: '1000' }); const images = [{ ...png, extra: 'ignored' }, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }]; const question = 'What about monitoring?'; const result = await app.request('post', '/clinical-assistant/chat', { message: question, history: [{ role: 'user', content: 'x'.repeat(500) }], images }); assert.equal(result.statusCode, 200); assert.equal(app.calls.search.length, 1); assert.equal(app.calls.ai.length, 2, 'rewrite plus generation'); const generation = app.calls.ai.at(-1); assert.deepEqual(generation.settings.images, [png, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }], 'normalized, extra fields dropped'); assert.equal(typeof generation.messages[1].content, 'string', 'the route keeps prompt text; ai.js builds multimodal parts'); assert.ok(generation.messages[1].content.includes(question)); assert.equal(app.calls.ai[0].settings.images, undefined, 'the retrieval rewrite is text-only'); const atLimit = server({ limit: '1000' }); const budgetBody = { message: 'x', history: [{ role: 'user', content: 'x'.repeat(999) }], images: [png, jpeg, webp, png] }; assert.equal((await atLimit.request('post', '/clinical-assistant/chat', budgetBody)).statusCode, 200, 'images are excluded from the UTF-16 budget'); assert.equal((await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' })).statusCode, 413, 'the text boundary is unchanged by images'); const over = await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' }); assert.equal(over.body.budget.unit, 'characters'); }); test('greeting and retrieval-empty direct responses validate attachments before responding', async () => { const bad = server(); const rejected = await bad.request('post', '/clinical-assistant/chat/stream', { message: 'thanks', history: [], images: [{ dataBase64: canonical(8), mimeType: 'image/bmp' }] }); assert.equal(rejected.statusCode, 400); assert.equal(bad.calls.ai.length + bad.calls.search.length, 0); const good = server(); const direct = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] }); assert.equal(direct.statusCode, 200); assert.match(direct.body.answer, /clinical question/); assert.equal(good.calls.ai.length + good.calls.search.length, 0, 'greeting never reaches providers'); }); test('handoff route is gone: images were never part of it and the layer no longer exists', async () => { const app = server(); await assert.rejects(app.request('post', '/clinical-assistant/handoff', { history: [{ role: 'user', content: 'Known facts.' }], images: [{ dataBase64: canonical(8), mimeType: 'image/png' }] }), /route/); assert.equal(app.calls.ai.length + app.calls.search.length, 0); }); function openaiAdapter() { const requests = []; class OpenAI { constructor() { this.chat = { completions: { create: async r => { requests.push(r); if (r.stream) return [{ choices: [{ delta: { content: 'streamed answer' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }]; return { choices: [{ message: { content: 'Synthetic multimodal answer.' }, finish_reason: 'stop' }], usage: null }; } } }; } } const mocks = { openai: { OpenAI }, './models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['synthetic-vision']); } }, './generationOptions': require('../src/utils/generationOptions'), './logger': { apiCall() {}, error() {} }, '../db/database': { async getSetting() { return null; } } }; const module = { exports: {} }; vm.runInNewContext(read('src/utils/ai.js'), { module, exports: module.exports, console: quiet, TextDecoder, process: { env: { LITELLM_API_BASE: 'http://synthetic.invalid', LITELLM_API_KEY: 'sk-synthetic' } }, require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; } }); return { requests, ai: module.exports }; } test('LiteLLM generation builds multimodal parts only on the latest user message and keeps tools/temperature/streaming', async () => { const tools = [{ type: 'function', function: { name: 'generate_image', parameters: { type: 'object' } } }]; const { requests, ai } = openaiAdapter(); const messages = [ { role: 'system', content: 'System behavior.' }, { role: 'user', content: 'Earlier turn.' }, { role: 'assistant', content: 'Earlier answer.' }, { role: 'user', content: 'Latest question.' } ]; const result = await ai.callAI(messages, { model: 'synthetic-vision', images: [png, jpeg], tools, temperature: 0.15, maxTokens: 2600 }); assert.equal(result.content, 'Synthetic multimodal answer.'); const request = requests[0]; assert.equal(request.model, 'synthetic-vision'); assert.equal(request.temperature, 0.15); assert.equal(request.max_tokens, 2600); assert.deepEqual(request.tools, tools); assert.equal(request.tool_choice, 'auto'); assert.equal(request.parallel_tool_calls, false); assert.deepEqual(request.messages[0], { role: 'system', content: 'System behavior.' }); assert.deepEqual(request.messages[1], { role: 'user', content: 'Earlier turn.' }); assert.deepEqual(request.messages[2], { role: 'assistant', content: 'Earlier answer.' }); assert.equal(request.messages[3].role, 'user'); assert.equal(request.messages[3].content[0].type, 'text'); assert.equal(request.messages[3].content[0].text, 'Latest question.'); assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[1])), { type: 'image_url', image_url: { url: 'data:image/png;base64,' + png.dataBase64 } }); assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[2])), { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,' + jpeg.dataBase64 } }); const streamed = await ai.callAIStream(messages, { model: 'synthetic-vision', images: [png], tools, temperature: 0.15, maxTokens: 2600 }, () => {}); assert.equal(streamed.content, 'streamed answer'); const streamRequest = requests[1]; assert.equal(streamRequest.stream, true); assert.deepEqual(streamRequest.tools, tools); assert.equal(streamRequest.messages[3].content.length, 2); assert.deepEqual(JSON.parse(JSON.stringify(streamRequest.messages[3].content[1].image_url)), { url: 'data:image/png;base64,' + png.dataBase64 }); const noImages = await ai.callAI(messages, { model: 'synthetic-vision' }); assert.equal(noImages.content, 'Synthetic multimodal answer.'); assert.deepEqual(requests[2].messages[3], { role: 'user', content: 'Latest question.' }); }); test('legacy direct provider adapters refuse image attachments with 400 before contacting the provider', async () => { const sdkCalls = []; class InvokeModelCommand { constructor(input) { this.input = input; } } class ConverseCommand { constructor(input) { this.input = input; } } const mocks = { openai: { OpenAI: class { constructor() { throw Error('Unexpected OpenAI client'); } } }, './models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['anthropic.claude-synthetic', 'amazon/nova-lite', 'google/gemini-2.5-flash']); } }, './generationOptions': require('../src/utils/generationOptions'), './logger': { apiCall() {}, error() {} }, '../db/database': { async getSetting() { return null; } }, '@aws-sdk/client-bedrock-runtime': { InvokeModelCommand, ConverseCommand, BedrockRuntimeClient: class { async send(command) { sdkCalls.push(command); return { body: Buffer.from('{}') }; } } }, '@google-cloud/vertexai': { VertexAI: class { getGenerativeModel() { return { async generateContent(request) { sdkCalls.push(request); return { response: { candidates: [] } }; } }; } } } }; const module = { exports: {} }; vm.runInNewContext(read('src/utils/ai.js'), { module, exports: module.exports, console: quiet, TextDecoder, process: { env: { AI_PROVIDER: 'bedrock', AWS_BEDROCK_REGION: 'synthetic' } }, require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; } }); const messages = [{ role: 'user', content: 'Question with image.' }]; await assert.rejects(module.exports.callAI(messages, { model: 'anthropic.claude-synthetic', images: [png] }), error => { assert.equal(error.statusCode, 400); assert.equal(error.code, 'IMAGES_UNSUPPORTED_PROVIDER'); assert.match(error.message, /OpenAI-compatible provider/); return true; }); await assert.rejects(module.exports.callAIStream(messages, { model: 'anthropic.claude-synthetic', images: [png] }, () => {}), error => error.statusCode === 400); assert.equal(sdkCalls.length, 0, 'refused before any provider contact'); const vertexModule = { exports: {} }; vm.runInNewContext(read('src/utils/ai.js'), { module: vertexModule, exports: vertexModule.exports, console: quiet, TextDecoder, process: { env: { AI_PROVIDER: 'vertex', GOOGLE_VERTEX_PROJECT: 'synthetic' } }, require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; } }); await assert.rejects(vertexModule.exports.callAI(messages, { model: 'google/gemini-2.5-flash', images: [png] }), error => error.statusCode === 400); assert.equal(sdkCalls.length, 0); }); function browserUI(options = {}) { const dom = new JSDOM('
' + read('public/components/assistant.html') + '
', { url: 'https://example.test/', runScripts: 'outside-only' }); const calls = { stream: [], save: [] }; const toasts = []; const escapeHtml = text => String(text).replace(/&/g, '&').replace(//g, '>'); const context = { window: dom.window, document: dom.window.document, navigator: dom.window.navigator, console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require('node:crypto').webcrypto, FileReader: dom.window.FileReader, File: dom.window.File, setTimeout() {}, showToast(text, kind) { toasts.push([String(text), kind || '']); }, escapeHtml, escapeAttr: escapeHtml, renderAssistantMarkdown: text => escapeHtml(text), renderSourcesList: () => '', EMPTY_PROMPT_SETS: [[]], createAssistantExporter: () => ({ invalidate() {}, exportAnswerPdf() {} }), createAssistantImageStore: () => ({ renderGeneratedImage: src => '', clear() {} }), isImageRequest: () => false, fetchAssistantStatus: async () => ({ success: true, conversationChars: options.limit || 120000 }), fetchAssistantExamples: async () => ({}), fetchSavedAssistantChats: async () => ({ success: true, chats: [] }), openAssistantStream: async payload => { calls.stream.push(payload); if (options.stream) return options.stream(payload); return new Response('event: done\ndata: ' + JSON.stringify({ success: true, answer: 'Complete response.', sources: [] }) + '\n\n'); }, fetchAssistantChat: async payload => ({ success: true, answer: 'Fallback answer.' }), saveAssistantChat: async payload => { calls.save.push(payload); return { success: true }; } }; vm.createContext(context); vm.runInContext(read('public/js/clinicalAssistant.js').replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, ''), context); context.bindEvents(); context.conversationChars = options.limit || 120000; return { context, document: dom.window.document, calls, toasts, dom }; } async function realApiHelpers() { const module = await import(pathToFileURL(path.join(root, 'public/js/assistant/api.js')).href); return module; } function attachFile(ui, name, type, bytes) { const input = ui.document.getElementById('assistant-attach-input'); const file = new ui.dom.window.File([bytes], name, { type }); Object.defineProperty(input, 'files', { value: [file], configurable: true }); input.dispatchEvent(new ui.dom.window.Event('change')); } async function tick() { for (let i = 0; i < 25; i++) await new Promise(resolve => setImmediate(resolve)); } test('UI attaches only PNG/JPEG/WebP with thumbnails, removal and client-side limits', async () => { const api = await realApiHelpers(); const ui = browserUI(); ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits; ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload; const bytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]); attachFile(ui, 'photo.png', 'image/png', bytes); attachFile(ui, 'scan.jpeg', 'image/jpeg', bytes); await tick(); const wrap = ui.document.getElementById('assistant-attachments'); assert.equal(wrap.hidden, false); assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2); assert.ok(wrap.querySelector('img').getAttribute('src').startsWith('data:image/png;base64,')); attachFile(ui, 'notes.txt', 'text/plain', new Uint8Array([1, 2, 3])); await tick(); assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2); assert.ok(ui.toasts.some(([text]) => /PNG, JPEG and WebP/.test(text))); attachFile(ui, 'third.webp', 'image/webp', bytes); attachFile(ui, 'fourth.png', 'image/png', bytes); await tick(); assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4); attachFile(ui, 'fifth.png', 'image/png', bytes); await tick(); assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4); assert.ok(ui.toasts.some(([text]) => /maximum of 4/.test(text))); attachFile(ui, 'huge.png', 'image/png', new Uint8Array(5 * 1024 * 1024 + 1)); await tick(); assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4); assert.ok(ui.toasts.some(([text]) => /5 MiB/.test(text))); wrap.querySelector('[data-assistant-remove-attachment="0"]').click(); await tick(); assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 3); assert.equal(ui.context.attachments.length, 3); ui.dom.window.close(); }); test('UI sends images only with the question, clears them on success, persists them in saved chats, and keeps the budget label text-only', async () => { const api = await realApiHelpers(); const ui = browserUI({ limit: 1000 }); ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits; ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload; const bytes = new Uint8Array([1, 2, 3, 4]); attachFile(ui, 'photo.png', 'image/png', bytes); await tick(); const expected = [{ dataBase64: Buffer.from(bytes).toString('base64'), mimeType: 'image/png' }]; ui.context.restoreSavedChat({ messages: [{ role: 'user', content: 'x'.repeat(999) }], lastAnswer: '' }); const input = ui.document.getElementById('assistant-input'); input.value = 'x'; input.dispatchEvent(new ui.dom.window.Event('input')); assert.equal(ui.document.getElementById('assistant-context-budget'), null, 'the constant character counter is gone; only the approaching-limit warning remains'); // Pending attachments stay input-only until the question is sent. ui.context.performAutosave(); assert.equal(ui.calls.save[0].images, undefined); assert.equal(JSON.stringify(ui.calls.save[0]).includes('dataBase64'), false); await ui.context.onAsk(); await tick(); assert.deepEqual(JSON.parse(JSON.stringify(ui.calls.stream[0].images)), expected); assert.equal(ui.context.attachments.length, 0, 'cleared on successful send'); assert.equal(ui.document.getElementById('assistant-attachments').hidden, true); assert.equal(ui.context.messages.length, 3, 'question and answer appended'); // The sent question now persists its attachments with the saved chat. ui.context.performAutosave(); assert.ok(JSON.stringify(ui.calls.save.at(-1)).includes('dataBase64')); const savedUser = ui.calls.save.at(-1).messages.filter(m => m.role === 'user').at(-1); assert.deepEqual(JSON.parse(JSON.stringify(savedUser.attachments)), [{ dataBase64: expected[0].dataBase64, mimeType: 'image/png', name: 'photo.png' }]); // New chat clears any later attachments. attachFile(ui, 'again.png', 'image/png', bytes); await tick(); assert.equal(ui.context.attachments.length, 1); await ui.context.clearConversation(); assert.equal(ui.context.attachments.length, 0); assert.equal(ui.document.getElementById('assistant-attachments').hidden, true); attachFile(ui, 'kept.png', 'image/png', bytes); await tick(); assert.equal(ui.context.attachments.length, 1); const failing = browserUI({ stream: () => new Response(JSON.stringify({ error: 'Image rejected' }), { status: 400 }) }); failing.context.assistantAttachmentLimits = api.assistantAttachmentLimits; failing.context.assistantAttachmentPayload = api.assistantAttachmentPayload; attachFile(failing, 'kept.png', 'image/png', bytes); await tick(); failing.document.getElementById('assistant-input').value = 'Question with attachment'; await failing.context.onAsk(); await tick(); assert.equal(failing.context.attachments.length, 1, 'rejected send keeps the attachments for correction'); assert.equal(failing.document.getElementById('assistant-attachments').hidden, false); ui.dom.window.close(); failing.dom.window.close(); });