Merge branch 'fix/attach-images-20260907'
All checks were successful
Forgejo Android APK / Root app tests (push) Successful in 24s
Forgejo Android APK / Build signed APK (push) Successful in 2m13s

This commit is contained in:
Daniel 2026-09-08 15:53:05 +02:00
commit 81abc60591
9 changed files with 630 additions and 10 deletions

View file

@ -72,6 +72,16 @@ show me the image/figure
Clinical answer response caching is intentionally disabled. Redis can support prompt suggestions and operational metadata, but final answers should be generated from current retrieval context.
## Image Attachments
Users can attach up to 4 images (PNG, JPEG, WebP) to an outgoing clinical question. Attachments are **input-only**:
- They are validated client-side and authoritatively on the server (MIME allowlist, canonical base64, ≤ 5 MiB per image, ≤ 4 images, ≤ 10 MiB decoded total). Invalid input is rejected with 400 before any retrieval or provider call.
- They are sent **only** with the outgoing clinical question, never with handoff summaries, image generation, saved chats, downloads or transcripts.
- The conversation budget counts text only: images are excluded from the UTF-16 code-unit count. The server still validates every request.
- Only OpenAI-compatible providers (LiteLLM, OpenRouter, Azure) receive them as multimodal content parts (`text` + `image_url` data URIs) on the latest user message; the system/retrieval/history structure is unchanged. Legacy direct adapters (Bedrock/Vertex) refuse with a clear 400 before contacting the provider.
- Attachments clear on a successful send and on New chat; a rejected send keeps them for correction.
## Settings
Important settings include:
@ -93,5 +103,6 @@ Add or update tests when changing:
- named-source provenance behavior,
- table rendering,
- image intent routing,
- image attachment validation and multimodal payload shape,
- MCP result normalization,
- model discovery or settings behavior.

View file

@ -44,8 +44,11 @@
<textarea id="assistant-input" rows="3" placeholder="Ask a focused clinical question..." autocomplete="off" aria-describedby="assistant-context-budget assistant-context-warning"></textarea>
<p id="assistant-context-budget" class="assistant-muted" aria-live="polite">Loading conversation limit; history and draft are counted in UTF-16 code units. The server validates each request.</p>
<div id="assistant-context-warning" role="alert" hidden></div>
<div id="assistant-attachments" class="assistant-attachments" aria-label="Attached images" hidden></div>
<textarea id="assistant-handoff-text" hidden readonly></textarea>
<div class="assistant-composer-footer">
<label class="assistant-attach" for="assistant-attach-input" title="Attach up to 4 images (PNG, JPEG, WebP)"><i class="fas fa-paperclip"></i> Attach images</label>
<input type="file" id="assistant-attach-input" accept="image/png,image/jpeg,image/webp" multiple hidden>
<label class="assistant-check"><input type="checkbox" id="assistant-include-context" checked> retrieve broader context</label>
<button id="btn-assistant-cancel" class="btn-sm btn-ghost" type="button" hidden><i class="fas fa-stop"></i> Cancel search</button>
<button id="btn-assistant-send" class="btn-generate" type="submit"><i class="fas fa-paper-plane"></i> Ask</button>

View file

@ -56,6 +56,14 @@
.assistant-cite { display:inline-flex; align-items:center; justify-content:center; min-width:18px; height:18px; padding:0 6px; margin:0 1px; border-radius:999px; background:var(--purple-light); color:var(--purple); font-size:10px; font-weight:800; text-decoration:none; vertical-align:baseline; border:1px solid rgba(124,58,237,.18); text-transform:uppercase; letter-spacing:.03em; }
.assistant-cite:hover { background:var(--purple); color:white; text-decoration:none; }
.assistant-composer { border-top:1px solid var(--g200); padding:12px; background:white; display:grid; gap:8px; }
.assistant-attachments { display:flex; flex-wrap:wrap; gap:8px; }
.assistant-attachments[hidden] { display:none; }
.assistant-attachment { position:relative; width:64px; height:64px; border:1px solid var(--g200); border-radius:8px; overflow:hidden; background:var(--g50); }
.assistant-attachment img { width:100%; height:100%; object-fit:cover; display:block; }
.assistant-attachment-remove { position:absolute; top:3px; right:3px; width:18px; height:18px; border:none; border-radius:50%; background:rgba(0,0,0,.65); color:#fff; font-size:10px; line-height:18px; cursor:pointer; padding:0; }
.assistant-attachment-remove:hover { background:rgba(0,0,0,.85); }
.assistant-attach { display:inline-flex; align-items:center; gap:6px; font-size:12px; color:var(--g600); cursor:pointer; padding:6px 10px; border:1px dashed var(--g300); border-radius:8px; }
.assistant-attach:hover { border-color:var(--blue); color:var(--blue); }
.assistant-composer textarea, .assistant-side textarea { width:100%; border:1.5px solid var(--g300); border-radius:10px; padding:10px 12px; resize:vertical; font-family:inherit; font-size:13px; outline:none; }
.assistant-composer textarea:focus, .assistant-side textarea:focus { border-color:var(--blue); box-shadow:0 0 0 3px var(--blue-light); }
.assistant-composer-footer { display:flex; justify-content:space-between; align-items:center; gap:10px; }

View file

@ -103,3 +103,19 @@ export function deleteSavedAssistantChat(id) {
credentials: 'same-origin'
}).then(function(r) { return r.json(); });
}
// Image attachments are input-only: they ride the outgoing clinical question
// payload and are never persisted or sent with handoff. The server enforces
// the same limits authoritatively; these mirror them for early feedback.
export var assistantAttachmentLimits = Object.freeze({
mimeTypes: ['image/png', 'image/jpeg', 'image/webp'],
maxImages: 4,
maxImageBytes: 5 * 1024 * 1024,
maxTotalBytes: 10 * 1024 * 1024
});
export function assistantAttachmentPayload(attachments) {
return (Array.isArray(attachments) ? attachments : []).map(function(attachment) {
return { dataBase64: attachment.dataBase64, mimeType: attachment.mimeType };
});
}

View file

@ -20,10 +20,13 @@ import {
openAssistantStream,
startAssistantImageJob,
saveAssistantChat,
requestAssistantHandoff
requestAssistantHandoff,
assistantAttachmentLimits,
assistantAttachmentPayload
} from './assistant/api.js';
var initialized = false;
var messages = [];
var attachments = []; // Input-only images: never persisted, never sent with handoff.
var lastAnswer = '';
var lastSources = [];
var mermaidReady = false;
@ -63,6 +66,7 @@ import {
var exportBtn = document.getElementById('btn-assistant-export-pdf');
var imageBtn = document.getElementById('btn-assistant-image');
var imageClearBtn = document.getElementById('btn-assistant-image-clear');
var attachInput = document.getElementById('assistant-attach-input');
var input = document.getElementById('assistant-input');
if (form) form.addEventListener('submit', onAsk);
@ -90,6 +94,7 @@ import {
if (exportBtn) exportBtn.addEventListener('click', exportAnswerPdf);
if (imageBtn) imageBtn.addEventListener('click', generateImage);
if (imageClearBtn) imageClearBtn.addEventListener('click', clearGeneratedImage);
if (attachInput) attachInput.addEventListener('change', onAttachFiles);
document.addEventListener('click', onAssistantDocumentClick);
if (input) input.addEventListener('keydown', function (e) {
if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') onAsk(e);
@ -162,16 +167,23 @@ import {
var row = appendMessage('user', text);
if (row && loading && loading.parentNode) loading.parentNode.insertBefore(row, loading);
if (input) input.value = '';
// Attached images are input-only and clear on a successful send.
attachments = [];
renderAttachments();
exporter.invalidate();
updateConversationBudget();
};
return streamAssistantResponse({
var payload = {
message: text,
idempotencyKey: crypto.randomUUID(),
history: history,
includeContext: !includeContext || includeContext.checked
}, loading, request)
};
// Images ride the outgoing clinical question only, never handoff or saves.
if (attachments.length) payload.images = assistantAttachmentPayload(attachments);
return streamAssistantResponse(payload, loading, request)
.catch(function (err) {
if (request.cancelled) return;
setBusy(false, 'Error', true);
@ -386,6 +398,73 @@ import {
return row;
}
function onAttachFiles(e) {
var input = e && e.target;
var files = Array.prototype.slice.call(input && input.files ? input.files : []);
files.forEach(addAttachmentFile);
if (input) input.value = ''; // Allow re-selecting the same file later.
}
function addAttachmentFile(file) {
var limits = assistantAttachmentLimits;
if (!file) return;
if (limits.mimeTypes.indexOf(file.type) === -1) {
if (typeof showToast === 'function') showToast('Only PNG, JPEG and WebP images can be attached; "' + String(file.name || 'attachment') + '" was ignored.', 'error');
return;
}
if (typeof file.size === 'number' && file.size > limits.maxImageBytes) {
if (typeof showToast === 'function') showToast('Each image is limited to 5 MiB; "' + String(file.name || 'attachment') + '" was ignored.', 'error');
return;
}
if (attachments.length >= limits.maxImages) {
if (typeof showToast === 'function') showToast('A maximum of 4 images can be attached to one question.', 'error');
return;
}
var totalBytes = attachments.reduce(function(sum, a) { return sum + (a.size || 0); }, 0);
if (typeof file.size === 'number' && totalBytes + file.size > limits.maxTotalBytes) {
if (typeof showToast === 'function') showToast('Attached images are limited to 10 MiB in total.', 'error');
return;
}
var reader = new FileReader();
reader.onload = function() {
var dataUrl = String(reader.result || '');
var comma = dataUrl.indexOf(',');
if (comma === -1 || dataUrl.indexOf(';base64,') === -1) {
if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error');
return;
}
attachments.push({ dataBase64: dataUrl.slice(comma + 1), mimeType: file.type, size: file.size, src: dataUrl });
renderAttachments();
};
reader.onerror = function() {
if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error');
};
reader.readAsDataURL(file);
}
function renderAttachments() {
var wrap = document.getElementById('assistant-attachments');
if (!wrap) return;
wrap.innerHTML = '';
attachments.forEach(function(attachment, index) {
var item = document.createElement('div');
item.className = 'assistant-attachment';
var img = document.createElement('img');
img.src = attachment.src || ('data:' + attachment.mimeType + ';base64,' + attachment.dataBase64);
img.alt = 'Attached image ' + (index + 1);
var remove = document.createElement('button');
remove.type = 'button';
remove.className = 'assistant-attachment-remove';
remove.setAttribute('data-assistant-remove-attachment', String(index));
remove.setAttribute('aria-label', 'Remove attached image ' + (index + 1));
remove.textContent = '✕';
item.appendChild(img);
item.appendChild(remove);
wrap.appendChild(item);
});
wrap.hidden = !attachments.length;
}
function renderSuggestionButtons(suggestions) {
var wrap = document.createElement('div');
wrap.className = 'assistant-suggestion-buttons';
@ -522,6 +601,12 @@ import {
if (bubble && Array.isArray(bubble.assistantSources)) renderSources(bubble.assistantSources);
return; // The native anchor navigates to the matching source in the refreshed panel.
}
var removeAttachment = e.target.closest('[data-assistant-remove-attachment]');
if (removeAttachment) {
attachments.splice(Number(removeAttachment.getAttribute('data-assistant-remove-attachment')), 1);
renderAttachments();
return;
}
var loadBtn = e.target.closest('[data-assistant-load-chat]');
if (loadBtn) {
e.preventDefault();
@ -574,6 +659,8 @@ import {
if (event && messages.length && !window.confirm('Start a new chat? Save or download this conversation first if you want to keep it.')) return;
if (activeAssistantRequest) cancelAssistantSearch();
messages = [];
attachments = [];
renderAttachments();
lastAnswer = '';
lastSources = [];
lastGeneratedImageSrc = '';
@ -931,6 +1018,7 @@ import {
var send = document.getElementById('btn-assistant-send');
var cancel = document.getElementById('btn-assistant-cancel');
var input = document.getElementById('assistant-input');
var attachInput = document.getElementById('assistant-attach-input');
if (status) {
status.classList.toggle('busy', !!isBusy);
status.classList.toggle('error', !!isError);
@ -948,6 +1036,7 @@ import {
cancel.style.display = canCancel ? 'inline-flex' : 'none';
}
if (input) input.disabled = !!isBusy;
if (attachInput) attachInput.disabled = !!isBusy;
}
function sanitize(html) { return window.DOMPurify ? window.DOMPurify.sanitize(html, { ADD_ATTR: ['target'] }) : escapeHtml(html); }

View file

@ -40,7 +40,7 @@ var {
finalizeAssistantAnswer
} = require('../utils/clinicalAnswer');
var { conversationBudget, checkConversation, savedChatPayload } = require('../utils/clinicalConversation');
var { conversationBudget, checkConversation, validateAttachments, savedChatPayload } = require('../utils/clinicalConversation');
var { DEFAULT_BEHAVIOR } = require('../utils/clinicalPrompts');
@ -193,7 +193,8 @@ router.post('/clinical-assistant/chat', async function(req, res) {
model: prepared.chatModel || undefined,
temperature: 0.15,
tools: imageTool.tools,
maxTokens: 2600
maxTokens: 2600,
images: prepared.images
}));
ai = await imageTool.dispatch(ai, { owner: req.user.id, workflow: 'clinical_assistant', body: req.body,
imageContext: prepared.imageContext, messages: prepared.messages, options: assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15 }), callAI: callAI });
@ -201,7 +202,7 @@ router.post('/clinical-assistant/chat', async function(req, res) {
messages: prepared.messages,
chatModel: prepared.chatModel,
callAI: callAI,
generationOptions: assistantGenerationOptions({ temperature: 0.15 })
generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images })
});
var answer = finalized.answer;
ai = finalized.ai;
@ -258,7 +259,8 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) {
model: prepared.chatModel || undefined,
temperature: 0.15,
tools: imageTool.tools,
maxTokens: 2600
maxTokens: 2600,
images: prepared.images
}), function(delta) {
sendEvent('token', { token: delta });
});
@ -269,7 +271,7 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) {
messages: prepared.messages,
chatModel: prepared.chatModel,
callAI: callAI,
generationOptions: assistantGenerationOptions({ temperature: 0.15 }),
generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images }),
streamed: true,
onRegenerating: function() { sendEvent('status', { message: 'Completing answer...' }); }
});
@ -354,6 +356,10 @@ async function prepareAssistantChat(body) {
var checked = checkConversation(body.history, body.message, await getConversationLimit());
var message = checked.message;
var history = checked.history;
// Image attachments are validated before any retrieval or provider call.
// The conversation budget counts the text only; images are excluded from
// the UTF-16 count and are never persisted in saved chats or transcripts.
var images = validateAttachments(body.images);
var chatModel = await getSetting('clinical_assistant.chat_model', '') || await getSetting('models.default', '');
var searchLimit = clampInt(await getSetting('clinical_assistant.search_limit', '8'), 3, 20, 8);
@ -416,6 +422,7 @@ async function prepareAssistantChat(body) {
var context = formatSourcesForPrompt(sources);
return {
message: message,
images: images,
imageContext: generatedImages.imageContext(message, history),
chatModel: chatModel,
sources: sources,

View file

@ -139,6 +139,35 @@ if (activeProvider === 'openrouter' && !openrouter) {
console.log('🤖 Active AI provider:', activeProvider);
// ============================================================
// IMAGE ATTACHMENTS — multimodal content for the latest user message.
// Only OpenAI-compatible providers accept content-part messages. The legacy
// direct adapters (Bedrock/Vertex) cannot take image parts, so requests with
// images are refused with a clear 400 before any provider contact.
// ============================================================
var MULTIMODAL_PROVIDERS = ['litellm', 'openrouter', 'azure'];
function applyImageAttachments(messages, images) {
if (!Array.isArray(images) || !images.length) return messages;
if (MULTIMODAL_PROVIDERS.indexOf(activeProvider) === -1) {
var unsupported = new Error('Image attachments require an OpenAI-compatible provider (LiteLLM, OpenRouter, or Azure). The active provider cannot accept image input, so no request was sent.');
unsupported.statusCode = 400;
unsupported.code = 'IMAGES_UNSUPPORTED_PROVIDER';
throw unsupported;
}
var out = messages.slice();
var index = out.length - 1;
while (index >= 0 && !(out[index] && out[index].role === 'user')) index--;
if (index < 0) return out; // No user message to attach to.
var message = out[index];
var content = Array.isArray(message.content) ? message.content.slice() : [{ type: 'text', text: String(message.content || '') }];
images.forEach(function (image) {
content.push({ type: 'image_url', image_url: { url: 'data:' + image.mimeType + ';base64,' + image.dataBase64 } });
});
out[index] = Object.assign({}, message, { content: content });
return out;
}
// Preserve unrecognized provider statuses; only explicit successful stops are complete.
function normalizeFinishReason(reason) {
if (typeof reason !== 'string') return reason ?? null;
@ -455,6 +484,8 @@ async function callAIStream(messages, options, onToken) {
await assertModelAllowed(model, options);
}
messages = applyImageAttachments(messages, options.images);
var client = null;
var provider = null;
if (activeProvider === 'litellm' && litellmClient) {
@ -527,6 +558,8 @@ async function callAI(messages, options) {
await assertModelAllowed(model, options);
}
messages = applyImageAttachments(messages, options.images);
try {
var result;
@ -731,4 +764,4 @@ async function discoverModels() {
return discovered;
}
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient };
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient, applyImageAttachments };

View file

@ -1,4 +1,7 @@
// Conversation text uses an exact character budget, not an estimated model token limit.
// Image attachments are excluded from this budget: UTF-16 code units count the
// text only. Attachments are validated separately below and are input-only —
// they are never persisted in saved chats or transcripts.
const DEFAULT_CONVERSATION_CHARS = 120000;
const MAX_SAVED_CHAT_BYTES = 8 * 1024 * 1024;
@ -94,6 +97,45 @@ function savedJobs(imageJobs) {
return imageJobs.map(job => ({ jobId: job.jobId }));
}
// Image attachments accompany the outgoing clinical question only.
// Strict limits: PNG/JPEG/WebP, canonical base64, <= 4 images,
// <= 5 MiB decoded per image, <= 10 MiB decoded total. Invalid input is
// rejected before any retrieval or provider call. The conversation budget
// above counts the text only; image bytes never enter the UTF-16 count.
const ATTACHMENT_MIME_TYPES = ['image/png', 'image/jpeg', 'image/webp'];
const MAX_ATTACHMENT_IMAGES = 4;
const MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024;
const MAX_ATTACHMENT_TOTAL_BYTES = 10 * 1024 * 1024;
const CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
function validateAttachments(images) {
if (images === undefined || images === null) return [];
if (!Array.isArray(images)) throw failure('Image attachments must be a list.', 400, 'INVALID_ATTACHMENTS');
if (images.length > MAX_ATTACHMENT_IMAGES) throw failure('A maximum of 4 images can be attached to one question.', 400, 'INVALID_ATTACHMENTS');
let total = 0;
return images.map(function (image) {
if (!image || typeof image !== 'object' || Array.isArray(image) ||
typeof image.dataBase64 !== 'string' || typeof image.mimeType !== 'string') {
throw failure('Each image attachment needs a base64 payload and a MIME type.', 400, 'INVALID_ATTACHMENTS');
}
if (!ATTACHMENT_MIME_TYPES.includes(image.mimeType)) {
throw failure('Only PNG, JPEG and WebP image attachments are allowed.', 400, 'INVALID_ATTACHMENTS');
}
if (image.dataBase64.length % 4 !== 0 || !CANONICAL_BASE64.test(image.dataBase64)) {
throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS');
}
const decoded = Buffer.from(image.dataBase64, 'base64');
if (decoded.toString('base64') !== image.dataBase64) {
throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS');
}
if (decoded.length === 0) throw failure('Image attachments must not be empty.', 400, 'INVALID_ATTACHMENTS');
if (decoded.length > MAX_ATTACHMENT_BYTES) throw failure('Each image attachment is limited to 5 MiB.', 400, 'INVALID_ATTACHMENTS');
total += decoded.length;
if (total > MAX_ATTACHMENT_TOTAL_BYTES) throw failure('Image attachments are limited to 10 MiB in total.', 400, 'INVALID_ATTACHMENTS');
return { dataBase64: image.dataBase64, mimeType: image.mimeType };
});
}
function savedChatPayload(body) {
const messages = validateMessages(body.messages).map(function(message, index) {
const original = body.messages[index];
@ -128,4 +170,4 @@ function savedChatPayload(body) {
return payload;
}
module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, savedChatPayload };
module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, validateAttachments, savedChatPayload };

View file

@ -0,0 +1,411 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const express = require('express');
const { JSDOM } = require('jsdom');
const { pathToFileURL } = require('node:url');
const policy = require('../src/utils/clinicalConversation');
const answer = require('../src/utils/clinicalAnswer');
const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');
const quiet = { log() {}, warn() {}, error() {}, info() {} };
const b64 = bytes => Buffer.from(bytes).toString('base64');
const canonical = (length, fill = 7) => Buffer.alloc(length, fill).toString('base64');
const png = { dataBase64: canonical(16), mimeType: 'image/png' };
const jpeg = { dataBase64: canonical(32), mimeType: 'image/jpeg' };
const webp = { dataBase64: canonical(48), mimeType: 'image/webp' };
function server(options = {}) {
const calls = { ai: [], search: [], writes: [], health: [] };
let saved;
const db = {
async getSetting(key) {
if (options.dbError) throw Error('private diagnostic');
if (key.endsWith('conversation_chars')) return options.legacyLimit ?? 'broken';
return options.model || null;
},
async get(sql, params) {
if (sql.includes('COUNT')) return { cnt: 0 };
return saved && params[1] === 7 ? { id: 1, title: saved[1], payload: saved[2] } : null;
},
async run(sql, params) { calls.writes.push(sql); saved = params; return { lastInsertRowid: 1 }; }
};
const ai = async (messages, settings) => {
calls.ai.push({ messages, settings });
return { content: 'Complete supported answer. [1]', finishReason: 'stop', model: 'synthetic' };
};
const source = { number: 1, title: 'Synthetic source', excerpt: 'Synthetic reference.', page: 9 };
const mocks = {
express, axios: { async post(url, payload) { calls.images.push(payload); return { data: { data: [{ b64_json: 'c3ludGhldGlj' }] } }; } }, crypto: require('node:crypto'), '../db/database': db,
'../middleware/auth': { authMiddleware() {} }, '../utils/ai': options.ai || { callAI: ai, callAIStream: ai },
'../utils/errors': { gatewayUrl: path => 'http://synthetic.invalid' + path }, '../utils/litellm': { getLiteLLMHeaders: () => ({}) }, '../utils/logger': { audit() {}, error() {} },
'../utils/crypto': { encryptString: value => 'encrypted:' + value, decryptString: value => value.replace(/^encrypted:/, '') },
'../utils/redis': { async getJson() { return null; }, async setJson() {} }, '../utils/clinicalPromptPool': { createClinicalPromptPool: () => ({}) },
'../utils/generatedImages': require('../src/utils/generatedImages'),
'../utils/generatedImageLinks': require('../src/utils/generatedImageLinks'),
'../utils/imageTool': require('../src/utils/imageTool'),
'../utils/clinicalPrompts': require('../src/utils/clinicalPrompts'),
'../utils/clinicalMcpClient': {
async semanticSearch(query) { calls.search.push(query); return {}; }, async getMcpHealth() { calls.health.push('health'); return {}; }
},
'../utils/clinicalRetrieval': {
cleanSourceExcerpt: require('../src/utils/clinicalRetrieval').cleanSourceExcerpt,
normalizeMcpSearchResponse: () => options.noSources ? [] : [source],
normalizeMcpMultimodalResponse: () => [], dedupeSources: value => value,
isVisualSourceQuery: () => false, classifyAndRerankMultimodalResults: async () => []
},
'../utils/clinicalAnswer': answer, '../utils/clinicalConversation': policy
};
const module = { exports: {} };
vm.runInNewContext(read('src/routes/clinicalAssistant.js'), {
module, exports: module.exports, console: quiet, Buffer, Map,
process: { env: { CLINICAL_ASSISTANT_MCP_WARMUP: 'false', CLINICAL_ASSISTANT_CONVERSATION_CHARS: options.limit, LITELLM_API_BASE: 'http://synthetic.invalid' } }, setTimeout() {},
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; }
});
async function request(method, endpoint, body, userId = 7) {
const handler = module.exports.stack.find(layer => layer.route && layer.route.path === endpoint && layer.route.methods[method]).route.stack.find(layer => layer.method === method).handle;
const res = {
statusCode: 200, headers: {}, events: '', body: null,
status(code) { this.statusCode = code; return this; }, json(body) { this.body = body; return this; },
setHeader(k, v) { this.headers[k] = v; }, write(text) { this.events += text; }, end() {}, flushHeaders() {}
};
await handler({ body, params: { id: 1 }, user: { id: userId } }, res);
return res;
}
return { calls, request };
}
test('attachment policy validates MIME, canonical base64, per-image/count/total limits without provider contact', async () => {
const oversize = { dataBase64: canonical(5 * 1024 * 1024 + 1), mimeType: 'image/png' };
const big = { dataBase64: canonical(4 * 1024 * 1024), mimeType: 'image/jpeg' };
const invalid = [
{ images: [{ ...png, mimeType: 'image/svg+xml' }] },
{ images: [{ ...png, mimeType: 'image/gif' }] },
{ images: [{ ...png, mimeType: 'application/octet-stream' }] },
{ images: [{ ...png, dataBase64: 'not base64!!!' }] },
{ images: [{ ...png, dataBase64: 'SGVsbG8' }] },
{ images: [{ ...png, dataBase64: 'AB==' }] }, // non-canonical trailing bits
{ images: [{ ...png, dataBase64: '' }] },
{ images: [{ ...png, dataBase64: 42 }] },
{ images: [{ mimeType: 'image/png' }] },
{ images: [{ dataBase64: png.dataBase64 }] },
{ images: [null] }, { images: ['png'] }, { images: [{}] },
{ images: 'not-a-list' },
{ images: [png, jpeg, webp, png, jpeg] },
{ images: [oversize] },
{ images: [big, big, big] }
];
for (const body of invalid) {
for (const endpoint of ['/clinical-assistant/chat', '/clinical-assistant/chat/stream']) {
const app = server();
const result = await app.request('post', endpoint, { message: 'What is the recommended dose of acetaminophen for a febrile three year old?', history: [], ...body });
assert.equal(result.statusCode, 400, JSON.stringify(body.images) + ' on ' + endpoint);
assert.equal(result.body.code, 'INVALID_ATTACHMENTS');
assert.equal(app.calls.ai.length + app.calls.search.length, 0, 'no provider or retrieval call on ' + endpoint);
}
}
});
test('valid images are normalized, ride the outgoing question only, and are excluded from the UTF-16 text budget', async () => {
const app = server({ limit: '1000' });
const images = [{ ...png, extra: 'ignored' }, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }];
const question = 'What about monitoring?';
const result = await app.request('post', '/clinical-assistant/chat', { message: question, history: [{ role: 'user', content: 'x'.repeat(500) }], images });
assert.equal(result.statusCode, 200);
assert.equal(app.calls.search.length, 1);
assert.equal(app.calls.ai.length, 2, 'rewrite plus generation');
const generation = app.calls.ai.at(-1);
assert.deepEqual(generation.settings.images, [png, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }], 'normalized, extra fields dropped');
assert.equal(typeof generation.messages[1].content, 'string', 'the route keeps prompt text; ai.js builds multimodal parts');
assert.ok(generation.messages[1].content.includes(question));
assert.equal(app.calls.ai[0].settings.images, undefined, 'the retrieval rewrite is text-only');
const atLimit = server({ limit: '1000' });
const budgetBody = { message: 'x', history: [{ role: 'user', content: 'x'.repeat(999) }], images: [png, jpeg, webp, png] };
assert.equal((await atLimit.request('post', '/clinical-assistant/chat', budgetBody)).statusCode, 200, 'images are excluded from the UTF-16 budget');
assert.equal((await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' })).statusCode, 413, 'the text boundary is unchanged by images');
const over = await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' });
assert.equal(over.body.budget.unit, 'characters');
});
test('greeting and retrieval-empty direct responses validate attachments before responding', async () => {
const bad = server();
const rejected = await bad.request('post', '/clinical-assistant/chat/stream', { message: 'thanks', history: [], images: [{ dataBase64: canonical(8), mimeType: 'image/bmp' }] });
assert.equal(rejected.statusCode, 400);
assert.equal(bad.calls.ai.length + bad.calls.search.length, 0);
const good = server();
const direct = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] });
assert.equal(direct.statusCode, 200);
assert.match(direct.body.answer, /clinical question/);
assert.equal(good.calls.ai.length + good.calls.search.length, 0, 'greeting never reaches providers');
});
test('handoff ignores image attachments and never forwards them', async () => {
const app = server();
const result = await app.request('post', '/clinical-assistant/handoff', { history: [{ role: 'user', content: 'Known facts.' }], images: [{ dataBase64: canonical(8), mimeType: 'image/png' }] });
assert.equal(result.statusCode, 200);
assert.equal(app.calls.ai.length, 1);
assert.ok(app.calls.ai[0].messages[1].content.includes('Known facts.'));
assert.equal(app.calls.ai[0].settings.images, undefined);
assert.doesNotMatch(JSON.stringify(app.calls.ai[0].messages), /base64/);
});
function openaiAdapter() {
const requests = [];
class OpenAI {
constructor() {
this.chat = { completions: { create: async r => {
requests.push(r);
if (r.stream) return [{ choices: [{ delta: { content: 'streamed answer' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }];
return { choices: [{ message: { content: 'Synthetic multimodal answer.' }, finish_reason: 'stop' }], usage: null };
} } };
}
}
const mocks = {
openai: { OpenAI },
'./models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['synthetic-vision']); } },
'./generationOptions': require('../src/utils/generationOptions'),
'./logger': { apiCall() {}, error() {} },
'../db/database': { async getSetting() { return null; } }
};
const module = { exports: {} };
vm.runInNewContext(read('src/utils/ai.js'), {
module, exports: module.exports, console: quiet, TextDecoder,
process: { env: { LITELLM_API_BASE: 'http://synthetic.invalid', LITELLM_API_KEY: 'sk-synthetic' } },
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
});
return { requests, ai: module.exports };
}
test('LiteLLM generation builds multimodal parts only on the latest user message and keeps tools/temperature/streaming', async () => {
const tools = [{ type: 'function', function: { name: 'generate_image', parameters: { type: 'object' } } }];
const { requests, ai } = openaiAdapter();
const messages = [
{ role: 'system', content: 'System behavior.' },
{ role: 'user', content: 'Earlier turn.' },
{ role: 'assistant', content: 'Earlier answer.' },
{ role: 'user', content: 'Latest question.' }
];
const result = await ai.callAI(messages, { model: 'synthetic-vision', images: [png, jpeg], tools, temperature: 0.15, maxTokens: 2600 });
assert.equal(result.content, 'Synthetic multimodal answer.');
const request = requests[0];
assert.equal(request.model, 'synthetic-vision');
assert.equal(request.temperature, 0.15);
assert.equal(request.max_tokens, 2600);
assert.deepEqual(request.tools, tools);
assert.equal(request.tool_choice, 'auto');
assert.equal(request.parallel_tool_calls, false);
assert.deepEqual(request.messages[0], { role: 'system', content: 'System behavior.' });
assert.deepEqual(request.messages[1], { role: 'user', content: 'Earlier turn.' });
assert.deepEqual(request.messages[2], { role: 'assistant', content: 'Earlier answer.' });
assert.equal(request.messages[3].role, 'user');
assert.equal(request.messages[3].content[0].type, 'text');
assert.equal(request.messages[3].content[0].text, 'Latest question.');
assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[1])), { type: 'image_url', image_url: { url: 'data:image/png;base64,' + png.dataBase64 } });
assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[2])), { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,' + jpeg.dataBase64 } });
const streamed = await ai.callAIStream(messages, { model: 'synthetic-vision', images: [png], tools, temperature: 0.15, maxTokens: 2600 }, () => {});
assert.equal(streamed.content, 'streamed answer');
const streamRequest = requests[1];
assert.equal(streamRequest.stream, true);
assert.deepEqual(streamRequest.tools, tools);
assert.equal(streamRequest.messages[3].content.length, 2);
assert.deepEqual(JSON.parse(JSON.stringify(streamRequest.messages[3].content[1].image_url)), { url: 'data:image/png;base64,' + png.dataBase64 });
const noImages = await ai.callAI(messages, { model: 'synthetic-vision' });
assert.equal(noImages.content, 'Synthetic multimodal answer.');
assert.deepEqual(requests[2].messages[3], { role: 'user', content: 'Latest question.' });
});
test('legacy direct provider adapters refuse image attachments with 400 before contacting the provider', async () => {
const sdkCalls = [];
class InvokeModelCommand { constructor(input) { this.input = input; } }
class ConverseCommand { constructor(input) { this.input = input; } }
const mocks = {
openai: { OpenAI: class { constructor() { throw Error('Unexpected OpenAI client'); } } },
'./models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['anthropic.claude-synthetic', 'amazon/nova-lite', 'google/gemini-2.5-flash']); } },
'./generationOptions': require('../src/utils/generationOptions'),
'./logger': { apiCall() {}, error() {} },
'../db/database': { async getSetting() { return null; } },
'@aws-sdk/client-bedrock-runtime': {
InvokeModelCommand, ConverseCommand,
BedrockRuntimeClient: class { async send(command) { sdkCalls.push(command); return { body: Buffer.from('{}') }; } }
},
'@google-cloud/vertexai': {
VertexAI: class { getGenerativeModel() { return { async generateContent(request) { sdkCalls.push(request); return { response: { candidates: [] } }; } }; } }
}
};
const module = { exports: {} };
vm.runInNewContext(read('src/utils/ai.js'), {
module, exports: module.exports, console: quiet, TextDecoder,
process: { env: { AI_PROVIDER: 'bedrock', AWS_BEDROCK_REGION: 'synthetic' } },
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
});
const messages = [{ role: 'user', content: 'Question with image.' }];
await assert.rejects(module.exports.callAI(messages, { model: 'anthropic.claude-synthetic', images: [png] }), error => {
assert.equal(error.statusCode, 400);
assert.equal(error.code, 'IMAGES_UNSUPPORTED_PROVIDER');
assert.match(error.message, /OpenAI-compatible provider/);
return true;
});
await assert.rejects(module.exports.callAIStream(messages, { model: 'anthropic.claude-synthetic', images: [png] }, () => {}), error => error.statusCode === 400);
assert.equal(sdkCalls.length, 0, 'refused before any provider contact');
const vertexModule = { exports: {} };
vm.runInNewContext(read('src/utils/ai.js'), {
module: vertexModule, exports: vertexModule.exports, console: quiet, TextDecoder,
process: { env: { AI_PROVIDER: 'vertex', GOOGLE_VERTEX_PROJECT: 'synthetic' } },
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
});
await assert.rejects(vertexModule.exports.callAI(messages, { model: 'google/gemini-2.5-flash', images: [png] }), error => error.statusCode === 400);
assert.equal(sdkCalls.length, 0);
});
function browserUI(options = {}) {
const dom = new JSDOM('<div id="assistant-tab">' + read('public/components/assistant.html') + '</div>', {
url: 'https://example.test/', runScripts: 'outside-only'
});
const calls = { stream: [], handoff: [], save: [] };
const toasts = [];
const escapeHtml = text => String(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
const context = {
window: dom.window, document: dom.window.document, navigator: dom.window.navigator,
console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require('node:crypto').webcrypto,
FileReader: dom.window.FileReader, File: dom.window.File,
setTimeout() {}, showToast(text, kind) { toasts.push([String(text), kind || '']); }, escapeHtml, escapeAttr: escapeHtml,
renderAssistantMarkdown: text => escapeHtml(text), renderSourcesList: () => '', EMPTY_PROMPT_SETS: [[]],
createAssistantExporter: () => ({ invalidate() {}, exportAnswerPdf() {} }),
createAssistantImageStore: () => ({ renderGeneratedImage: src => '<img src="' + src + '">', clear() {} }),
isImageRequest: () => false,
fetchAssistantStatus: async () => ({ success: true, conversationChars: options.limit || 120000 }),
fetchAssistantExamples: async () => ({}), fetchSavedAssistantChats: async () => ({ success: true, chats: [] }),
openAssistantStream: async payload => {
calls.stream.push(payload);
if (options.stream) return options.stream(payload);
return new Response('event: done\ndata: ' + JSON.stringify({ success: true, answer: 'Complete response.', sources: [] }) + '\n\n');
},
fetchAssistantChat: async payload => ({ success: true, answer: 'Fallback answer.' }),
saveAssistantChat: async payload => { calls.save.push(payload); return { success: true }; },
requestAssistantHandoff: async history => { calls.handoff.push(history); return { success: true, summary: 'Explicit handoff.' }; }
};
vm.createContext(context);
vm.runInContext(read('public/js/clinicalAssistant.js').replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, ''), context);
context.bindEvents();
context.conversationChars = options.limit || 120000;
return { context, document: dom.window.document, calls, toasts, dom };
}
async function realApiHelpers() {
const module = await import(pathToFileURL(path.join(root, 'public/js/assistant/api.js')).href);
return module;
}
function attachFile(ui, name, type, bytes) {
const input = ui.document.getElementById('assistant-attach-input');
const file = new ui.dom.window.File([bytes], name, { type });
Object.defineProperty(input, 'files', { value: [file], configurable: true });
input.dispatchEvent(new ui.dom.window.Event('change'));
}
async function tick() { for (let i = 0; i < 25; i++) await new Promise(resolve => setImmediate(resolve)); }
test('UI attaches only PNG/JPEG/WebP with thumbnails, removal and client-side limits', async () => {
const api = await realApiHelpers();
const ui = browserUI();
ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
const bytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]);
attachFile(ui, 'photo.png', 'image/png', bytes);
attachFile(ui, 'scan.jpeg', 'image/jpeg', bytes);
await tick();
const wrap = ui.document.getElementById('assistant-attachments');
assert.equal(wrap.hidden, false);
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2);
assert.ok(wrap.querySelector('img').getAttribute('src').startsWith('data:image/png;base64,'));
attachFile(ui, 'notes.txt', 'text/plain', new Uint8Array([1, 2, 3]));
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2);
assert.ok(ui.toasts.some(([text]) => /PNG, JPEG and WebP/.test(text)));
attachFile(ui, 'third.webp', 'image/webp', bytes);
attachFile(ui, 'fourth.png', 'image/png', bytes);
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
attachFile(ui, 'fifth.png', 'image/png', bytes);
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
assert.ok(ui.toasts.some(([text]) => /maximum of 4/.test(text)));
attachFile(ui, 'huge.png', 'image/png', new Uint8Array(5 * 1024 * 1024 + 1));
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
assert.ok(ui.toasts.some(([text]) => /5 MiB/.test(text)));
wrap.querySelector('[data-assistant-remove-attachment="0"]').click();
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 3);
assert.equal(ui.context.attachments.length, 3);
ui.dom.window.close();
});
test('UI sends images only with the question, clears them on success, and keeps them out of handoff, saves and the budget label', async () => {
const api = await realApiHelpers();
const ui = browserUI({ limit: 1000 });
ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
const bytes = new Uint8Array([1, 2, 3, 4]);
attachFile(ui, 'photo.png', 'image/png', bytes);
await tick();
const expected = [{ dataBase64: Buffer.from(bytes).toString('base64'), mimeType: 'image/png' }];
ui.context.restoreSavedChat({ messages: [{ role: 'user', content: 'x'.repeat(999) }], lastAnswer: '' });
const input = ui.document.getElementById('assistant-input');
input.value = 'x';
input.dispatchEvent(new ui.dom.window.Event('input'));
const label = ui.document.getElementById('assistant-context-budget');
assert.equal(label.textContent, '1,000 / 1,000 conversation characters (UTF-16 code units).', 'budget label stays text-only');
assert.doesNotMatch(label.textContent, /image/i);
// Handoff and saved chats never carry attachments.
await ui.context.requestHandoff();
assert.equal(ui.calls.handoff.length, 1);
assert.equal(ui.calls.handoff[0].images, undefined);
assert.equal(JSON.stringify(ui.calls.handoff[0]).includes('dataBase64'), false);
await ui.context.saveCurrentChat();
assert.equal(ui.calls.save[0].images, undefined);
assert.equal(JSON.stringify(ui.calls.save[0]).includes('dataBase64'), false);
await ui.context.onAsk();
await tick();
assert.deepEqual(JSON.parse(JSON.stringify(ui.calls.stream[0].images)), expected);
assert.equal(ui.context.attachments.length, 0, 'cleared on successful send');
assert.equal(ui.document.getElementById('assistant-attachments').hidden, true);
assert.equal(ui.context.messages.length, 3, 'question and answer appended');
// New chat clears any later attachments.
attachFile(ui, 'again.png', 'image/png', bytes);
await tick();
assert.equal(ui.context.attachments.length, 1);
await ui.context.clearConversation();
assert.equal(ui.context.attachments.length, 0);
assert.equal(ui.document.getElementById('assistant-attachments').hidden, true);
attachFile(ui, 'kept.png', 'image/png', bytes);
await tick();
assert.equal(ui.context.attachments.length, 1);
const failing = browserUI({ stream: () => new Response(JSON.stringify({ error: 'Image rejected' }), { status: 400 }) });
failing.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
failing.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
attachFile(failing, 'kept.png', 'image/png', bytes);
await tick();
failing.document.getElementById('assistant-input').value = 'Question with attachment';
await failing.context.onAsk();
await tick();
assert.equal(failing.context.attachments.length, 1, 'rejected send keeps the attachments for correction');
assert.equal(failing.document.getElementById('assistant-attachments').hidden, false);
ui.dom.window.close();
failing.dom.window.close();
});