Merge branch 'fix/attach-images-20260907'
All checks were successful
Forgejo Android APK / Root app tests (push) Successful in 24s
Forgejo Android APK / Build signed APK (push) Successful in 2m13s

This commit is contained in:
Daniel 2026-09-08 15:53:05 +02:00
commit 81abc60591
9 changed files with 630 additions and 10 deletions

View file

@ -72,6 +72,16 @@ show me the image/figure
Clinical answer response caching is intentionally disabled. Redis can support prompt suggestions and operational metadata, but final answers should be generated from current retrieval context. Clinical answer response caching is intentionally disabled. Redis can support prompt suggestions and operational metadata, but final answers should be generated from current retrieval context.
## Image Attachments
Users can attach up to 4 images (PNG, JPEG, WebP) to an outgoing clinical question. Attachments are **input-only**:
- They are validated client-side and authoritatively on the server (MIME allowlist, canonical base64, ≤ 5 MiB per image, ≤ 4 images, ≤ 10 MiB decoded total). Invalid input is rejected with 400 before any retrieval or provider call.
- They are sent **only** with the outgoing clinical question, never with handoff summaries, image generation, saved chats, downloads or transcripts.
- The conversation budget counts text only: images are excluded from the UTF-16 code-unit count. The server still validates every request.
- Only OpenAI-compatible providers (LiteLLM, OpenRouter, Azure) receive them as multimodal content parts (`text` + `image_url` data URIs) on the latest user message; the system/retrieval/history structure is unchanged. Legacy direct adapters (Bedrock/Vertex) refuse with a clear 400 before contacting the provider.
- Attachments clear on a successful send and on New chat; a rejected send keeps them for correction.
## Settings ## Settings
Important settings include: Important settings include:
@ -93,5 +103,6 @@ Add or update tests when changing:
- named-source provenance behavior, - named-source provenance behavior,
- table rendering, - table rendering,
- image intent routing, - image intent routing,
- image attachment validation and multimodal payload shape,
- MCP result normalization, - MCP result normalization,
- model discovery or settings behavior. - model discovery or settings behavior.

View file

@ -44,8 +44,11 @@
<textarea id="assistant-input" rows="3" placeholder="Ask a focused clinical question..." autocomplete="off" aria-describedby="assistant-context-budget assistant-context-warning"></textarea> <textarea id="assistant-input" rows="3" placeholder="Ask a focused clinical question..." autocomplete="off" aria-describedby="assistant-context-budget assistant-context-warning"></textarea>
<p id="assistant-context-budget" class="assistant-muted" aria-live="polite">Loading conversation limit; history and draft are counted in UTF-16 code units. The server validates each request.</p> <p id="assistant-context-budget" class="assistant-muted" aria-live="polite">Loading conversation limit; history and draft are counted in UTF-16 code units. The server validates each request.</p>
<div id="assistant-context-warning" role="alert" hidden></div> <div id="assistant-context-warning" role="alert" hidden></div>
<div id="assistant-attachments" class="assistant-attachments" aria-label="Attached images" hidden></div>
<textarea id="assistant-handoff-text" hidden readonly></textarea> <textarea id="assistant-handoff-text" hidden readonly></textarea>
<div class="assistant-composer-footer"> <div class="assistant-composer-footer">
<label class="assistant-attach" for="assistant-attach-input" title="Attach up to 4 images (PNG, JPEG, WebP)"><i class="fas fa-paperclip"></i> Attach images</label>
<input type="file" id="assistant-attach-input" accept="image/png,image/jpeg,image/webp" multiple hidden>
<label class="assistant-check"><input type="checkbox" id="assistant-include-context" checked> retrieve broader context</label> <label class="assistant-check"><input type="checkbox" id="assistant-include-context" checked> retrieve broader context</label>
<button id="btn-assistant-cancel" class="btn-sm btn-ghost" type="button" hidden><i class="fas fa-stop"></i> Cancel search</button> <button id="btn-assistant-cancel" class="btn-sm btn-ghost" type="button" hidden><i class="fas fa-stop"></i> Cancel search</button>
<button id="btn-assistant-send" class="btn-generate" type="submit"><i class="fas fa-paper-plane"></i> Ask</button> <button id="btn-assistant-send" class="btn-generate" type="submit"><i class="fas fa-paper-plane"></i> Ask</button>

View file

@ -56,6 +56,14 @@
.assistant-cite { display:inline-flex; align-items:center; justify-content:center; min-width:18px; height:18px; padding:0 6px; margin:0 1px; border-radius:999px; background:var(--purple-light); color:var(--purple); font-size:10px; font-weight:800; text-decoration:none; vertical-align:baseline; border:1px solid rgba(124,58,237,.18); text-transform:uppercase; letter-spacing:.03em; } .assistant-cite { display:inline-flex; align-items:center; justify-content:center; min-width:18px; height:18px; padding:0 6px; margin:0 1px; border-radius:999px; background:var(--purple-light); color:var(--purple); font-size:10px; font-weight:800; text-decoration:none; vertical-align:baseline; border:1px solid rgba(124,58,237,.18); text-transform:uppercase; letter-spacing:.03em; }
.assistant-cite:hover { background:var(--purple); color:white; text-decoration:none; } .assistant-cite:hover { background:var(--purple); color:white; text-decoration:none; }
.assistant-composer { border-top:1px solid var(--g200); padding:12px; background:white; display:grid; gap:8px; } .assistant-composer { border-top:1px solid var(--g200); padding:12px; background:white; display:grid; gap:8px; }
.assistant-attachments { display:flex; flex-wrap:wrap; gap:8px; }
.assistant-attachments[hidden] { display:none; }
.assistant-attachment { position:relative; width:64px; height:64px; border:1px solid var(--g200); border-radius:8px; overflow:hidden; background:var(--g50); }
.assistant-attachment img { width:100%; height:100%; object-fit:cover; display:block; }
.assistant-attachment-remove { position:absolute; top:3px; right:3px; width:18px; height:18px; border:none; border-radius:50%; background:rgba(0,0,0,.65); color:#fff; font-size:10px; line-height:18px; cursor:pointer; padding:0; }
.assistant-attachment-remove:hover { background:rgba(0,0,0,.85); }
.assistant-attach { display:inline-flex; align-items:center; gap:6px; font-size:12px; color:var(--g600); cursor:pointer; padding:6px 10px; border:1px dashed var(--g300); border-radius:8px; }
.assistant-attach:hover { border-color:var(--blue); color:var(--blue); }
.assistant-composer textarea, .assistant-side textarea { width:100%; border:1.5px solid var(--g300); border-radius:10px; padding:10px 12px; resize:vertical; font-family:inherit; font-size:13px; outline:none; } .assistant-composer textarea, .assistant-side textarea { width:100%; border:1.5px solid var(--g300); border-radius:10px; padding:10px 12px; resize:vertical; font-family:inherit; font-size:13px; outline:none; }
.assistant-composer textarea:focus, .assistant-side textarea:focus { border-color:var(--blue); box-shadow:0 0 0 3px var(--blue-light); } .assistant-composer textarea:focus, .assistant-side textarea:focus { border-color:var(--blue); box-shadow:0 0 0 3px var(--blue-light); }
.assistant-composer-footer { display:flex; justify-content:space-between; align-items:center; gap:10px; } .assistant-composer-footer { display:flex; justify-content:space-between; align-items:center; gap:10px; }

View file

@ -103,3 +103,19 @@ export function deleteSavedAssistantChat(id) {
credentials: 'same-origin' credentials: 'same-origin'
}).then(function(r) { return r.json(); }); }).then(function(r) { return r.json(); });
} }
// Image attachments are input-only: they ride the outgoing clinical question
// payload and are never persisted or sent with handoff. The server enforces
// the same limits authoritatively; these mirror them for early feedback.
export var assistantAttachmentLimits = Object.freeze({
mimeTypes: ['image/png', 'image/jpeg', 'image/webp'],
maxImages: 4,
maxImageBytes: 5 * 1024 * 1024,
maxTotalBytes: 10 * 1024 * 1024
});
export function assistantAttachmentPayload(attachments) {
return (Array.isArray(attachments) ? attachments : []).map(function(attachment) {
return { dataBase64: attachment.dataBase64, mimeType: attachment.mimeType };
});
}

View file

@ -20,10 +20,13 @@ import {
openAssistantStream, openAssistantStream,
startAssistantImageJob, startAssistantImageJob,
saveAssistantChat, saveAssistantChat,
requestAssistantHandoff requestAssistantHandoff,
assistantAttachmentLimits,
assistantAttachmentPayload
} from './assistant/api.js'; } from './assistant/api.js';
var initialized = false; var initialized = false;
var messages = []; var messages = [];
var attachments = []; // Input-only images: never persisted, never sent with handoff.
var lastAnswer = ''; var lastAnswer = '';
var lastSources = []; var lastSources = [];
var mermaidReady = false; var mermaidReady = false;
@ -63,6 +66,7 @@ import {
var exportBtn = document.getElementById('btn-assistant-export-pdf'); var exportBtn = document.getElementById('btn-assistant-export-pdf');
var imageBtn = document.getElementById('btn-assistant-image'); var imageBtn = document.getElementById('btn-assistant-image');
var imageClearBtn = document.getElementById('btn-assistant-image-clear'); var imageClearBtn = document.getElementById('btn-assistant-image-clear');
var attachInput = document.getElementById('assistant-attach-input');
var input = document.getElementById('assistant-input'); var input = document.getElementById('assistant-input');
if (form) form.addEventListener('submit', onAsk); if (form) form.addEventListener('submit', onAsk);
@ -90,6 +94,7 @@ import {
if (exportBtn) exportBtn.addEventListener('click', exportAnswerPdf); if (exportBtn) exportBtn.addEventListener('click', exportAnswerPdf);
if (imageBtn) imageBtn.addEventListener('click', generateImage); if (imageBtn) imageBtn.addEventListener('click', generateImage);
if (imageClearBtn) imageClearBtn.addEventListener('click', clearGeneratedImage); if (imageClearBtn) imageClearBtn.addEventListener('click', clearGeneratedImage);
if (attachInput) attachInput.addEventListener('change', onAttachFiles);
document.addEventListener('click', onAssistantDocumentClick); document.addEventListener('click', onAssistantDocumentClick);
if (input) input.addEventListener('keydown', function (e) { if (input) input.addEventListener('keydown', function (e) {
if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') onAsk(e); if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') onAsk(e);
@ -162,16 +167,23 @@ import {
var row = appendMessage('user', text); var row = appendMessage('user', text);
if (row && loading && loading.parentNode) loading.parentNode.insertBefore(row, loading); if (row && loading && loading.parentNode) loading.parentNode.insertBefore(row, loading);
if (input) input.value = ''; if (input) input.value = '';
// Attached images are input-only and clear on a successful send.
attachments = [];
renderAttachments();
exporter.invalidate(); exporter.invalidate();
updateConversationBudget(); updateConversationBudget();
}; };
return streamAssistantResponse({ var payload = {
message: text, message: text,
idempotencyKey: crypto.randomUUID(), idempotencyKey: crypto.randomUUID(),
history: history, history: history,
includeContext: !includeContext || includeContext.checked includeContext: !includeContext || includeContext.checked
}, loading, request) };
// Images ride the outgoing clinical question only, never handoff or saves.
if (attachments.length) payload.images = assistantAttachmentPayload(attachments);
return streamAssistantResponse(payload, loading, request)
.catch(function (err) { .catch(function (err) {
if (request.cancelled) return; if (request.cancelled) return;
setBusy(false, 'Error', true); setBusy(false, 'Error', true);
@ -386,6 +398,73 @@ import {
return row; return row;
} }
function onAttachFiles(e) {
var input = e && e.target;
var files = Array.prototype.slice.call(input && input.files ? input.files : []);
files.forEach(addAttachmentFile);
if (input) input.value = ''; // Allow re-selecting the same file later.
}
function addAttachmentFile(file) {
var limits = assistantAttachmentLimits;
if (!file) return;
if (limits.mimeTypes.indexOf(file.type) === -1) {
if (typeof showToast === 'function') showToast('Only PNG, JPEG and WebP images can be attached; "' + String(file.name || 'attachment') + '" was ignored.', 'error');
return;
}
if (typeof file.size === 'number' && file.size > limits.maxImageBytes) {
if (typeof showToast === 'function') showToast('Each image is limited to 5 MiB; "' + String(file.name || 'attachment') + '" was ignored.', 'error');
return;
}
if (attachments.length >= limits.maxImages) {
if (typeof showToast === 'function') showToast('A maximum of 4 images can be attached to one question.', 'error');
return;
}
var totalBytes = attachments.reduce(function(sum, a) { return sum + (a.size || 0); }, 0);
if (typeof file.size === 'number' && totalBytes + file.size > limits.maxTotalBytes) {
if (typeof showToast === 'function') showToast('Attached images are limited to 10 MiB in total.', 'error');
return;
}
var reader = new FileReader();
reader.onload = function() {
var dataUrl = String(reader.result || '');
var comma = dataUrl.indexOf(',');
if (comma === -1 || dataUrl.indexOf(';base64,') === -1) {
if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error');
return;
}
attachments.push({ dataBase64: dataUrl.slice(comma + 1), mimeType: file.type, size: file.size, src: dataUrl });
renderAttachments();
};
reader.onerror = function() {
if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error');
};
reader.readAsDataURL(file);
}
function renderAttachments() {
var wrap = document.getElementById('assistant-attachments');
if (!wrap) return;
wrap.innerHTML = '';
attachments.forEach(function(attachment, index) {
var item = document.createElement('div');
item.className = 'assistant-attachment';
var img = document.createElement('img');
img.src = attachment.src || ('data:' + attachment.mimeType + ';base64,' + attachment.dataBase64);
img.alt = 'Attached image ' + (index + 1);
var remove = document.createElement('button');
remove.type = 'button';
remove.className = 'assistant-attachment-remove';
remove.setAttribute('data-assistant-remove-attachment', String(index));
remove.setAttribute('aria-label', 'Remove attached image ' + (index + 1));
remove.textContent = '✕';
item.appendChild(img);
item.appendChild(remove);
wrap.appendChild(item);
});
wrap.hidden = !attachments.length;
}
function renderSuggestionButtons(suggestions) { function renderSuggestionButtons(suggestions) {
var wrap = document.createElement('div'); var wrap = document.createElement('div');
wrap.className = 'assistant-suggestion-buttons'; wrap.className = 'assistant-suggestion-buttons';
@ -522,6 +601,12 @@ import {
if (bubble && Array.isArray(bubble.assistantSources)) renderSources(bubble.assistantSources); if (bubble && Array.isArray(bubble.assistantSources)) renderSources(bubble.assistantSources);
return; // The native anchor navigates to the matching source in the refreshed panel. return; // The native anchor navigates to the matching source in the refreshed panel.
} }
var removeAttachment = e.target.closest('[data-assistant-remove-attachment]');
if (removeAttachment) {
attachments.splice(Number(removeAttachment.getAttribute('data-assistant-remove-attachment')), 1);
renderAttachments();
return;
}
var loadBtn = e.target.closest('[data-assistant-load-chat]'); var loadBtn = e.target.closest('[data-assistant-load-chat]');
if (loadBtn) { if (loadBtn) {
e.preventDefault(); e.preventDefault();
@ -574,6 +659,8 @@ import {
if (event && messages.length && !window.confirm('Start a new chat? Save or download this conversation first if you want to keep it.')) return; if (event && messages.length && !window.confirm('Start a new chat? Save or download this conversation first if you want to keep it.')) return;
if (activeAssistantRequest) cancelAssistantSearch(); if (activeAssistantRequest) cancelAssistantSearch();
messages = []; messages = [];
attachments = [];
renderAttachments();
lastAnswer = ''; lastAnswer = '';
lastSources = []; lastSources = [];
lastGeneratedImageSrc = ''; lastGeneratedImageSrc = '';
@ -931,6 +1018,7 @@ import {
var send = document.getElementById('btn-assistant-send'); var send = document.getElementById('btn-assistant-send');
var cancel = document.getElementById('btn-assistant-cancel'); var cancel = document.getElementById('btn-assistant-cancel');
var input = document.getElementById('assistant-input'); var input = document.getElementById('assistant-input');
var attachInput = document.getElementById('assistant-attach-input');
if (status) { if (status) {
status.classList.toggle('busy', !!isBusy); status.classList.toggle('busy', !!isBusy);
status.classList.toggle('error', !!isError); status.classList.toggle('error', !!isError);
@ -948,6 +1036,7 @@ import {
cancel.style.display = canCancel ? 'inline-flex' : 'none'; cancel.style.display = canCancel ? 'inline-flex' : 'none';
} }
if (input) input.disabled = !!isBusy; if (input) input.disabled = !!isBusy;
if (attachInput) attachInput.disabled = !!isBusy;
} }
function sanitize(html) { return window.DOMPurify ? window.DOMPurify.sanitize(html, { ADD_ATTR: ['target'] }) : escapeHtml(html); } function sanitize(html) { return window.DOMPurify ? window.DOMPurify.sanitize(html, { ADD_ATTR: ['target'] }) : escapeHtml(html); }

View file

@ -40,7 +40,7 @@ var {
finalizeAssistantAnswer finalizeAssistantAnswer
} = require('../utils/clinicalAnswer'); } = require('../utils/clinicalAnswer');
var { conversationBudget, checkConversation, savedChatPayload } = require('../utils/clinicalConversation'); var { conversationBudget, checkConversation, validateAttachments, savedChatPayload } = require('../utils/clinicalConversation');
var { DEFAULT_BEHAVIOR } = require('../utils/clinicalPrompts'); var { DEFAULT_BEHAVIOR } = require('../utils/clinicalPrompts');
@ -193,7 +193,8 @@ router.post('/clinical-assistant/chat', async function(req, res) {
model: prepared.chatModel || undefined, model: prepared.chatModel || undefined,
temperature: 0.15, temperature: 0.15,
tools: imageTool.tools, tools: imageTool.tools,
maxTokens: 2600 maxTokens: 2600,
images: prepared.images
})); }));
ai = await imageTool.dispatch(ai, { owner: req.user.id, workflow: 'clinical_assistant', body: req.body, ai = await imageTool.dispatch(ai, { owner: req.user.id, workflow: 'clinical_assistant', body: req.body,
imageContext: prepared.imageContext, messages: prepared.messages, options: assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15 }), callAI: callAI }); imageContext: prepared.imageContext, messages: prepared.messages, options: assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15 }), callAI: callAI });
@ -201,7 +202,7 @@ router.post('/clinical-assistant/chat', async function(req, res) {
messages: prepared.messages, messages: prepared.messages,
chatModel: prepared.chatModel, chatModel: prepared.chatModel,
callAI: callAI, callAI: callAI,
generationOptions: assistantGenerationOptions({ temperature: 0.15 }) generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images })
}); });
var answer = finalized.answer; var answer = finalized.answer;
ai = finalized.ai; ai = finalized.ai;
@ -258,7 +259,8 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) {
model: prepared.chatModel || undefined, model: prepared.chatModel || undefined,
temperature: 0.15, temperature: 0.15,
tools: imageTool.tools, tools: imageTool.tools,
maxTokens: 2600 maxTokens: 2600,
images: prepared.images
}), function(delta) { }), function(delta) {
sendEvent('token', { token: delta }); sendEvent('token', { token: delta });
}); });
@ -269,7 +271,7 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) {
messages: prepared.messages, messages: prepared.messages,
chatModel: prepared.chatModel, chatModel: prepared.chatModel,
callAI: callAI, callAI: callAI,
generationOptions: assistantGenerationOptions({ temperature: 0.15 }), generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images }),
streamed: true, streamed: true,
onRegenerating: function() { sendEvent('status', { message: 'Completing answer...' }); } onRegenerating: function() { sendEvent('status', { message: 'Completing answer...' }); }
}); });
@ -354,6 +356,10 @@ async function prepareAssistantChat(body) {
var checked = checkConversation(body.history, body.message, await getConversationLimit()); var checked = checkConversation(body.history, body.message, await getConversationLimit());
var message = checked.message; var message = checked.message;
var history = checked.history; var history = checked.history;
// Image attachments are validated before any retrieval or provider call.
// The conversation budget counts the text only; images are excluded from
// the UTF-16 count and are never persisted in saved chats or transcripts.
var images = validateAttachments(body.images);
var chatModel = await getSetting('clinical_assistant.chat_model', '') || await getSetting('models.default', ''); var chatModel = await getSetting('clinical_assistant.chat_model', '') || await getSetting('models.default', '');
var searchLimit = clampInt(await getSetting('clinical_assistant.search_limit', '8'), 3, 20, 8); var searchLimit = clampInt(await getSetting('clinical_assistant.search_limit', '8'), 3, 20, 8);
@ -416,6 +422,7 @@ async function prepareAssistantChat(body) {
var context = formatSourcesForPrompt(sources); var context = formatSourcesForPrompt(sources);
return { return {
message: message, message: message,
images: images,
imageContext: generatedImages.imageContext(message, history), imageContext: generatedImages.imageContext(message, history),
chatModel: chatModel, chatModel: chatModel,
sources: sources, sources: sources,

View file

@ -139,6 +139,35 @@ if (activeProvider === 'openrouter' && !openrouter) {
console.log('🤖 Active AI provider:', activeProvider); console.log('🤖 Active AI provider:', activeProvider);
// ============================================================
// IMAGE ATTACHMENTS — multimodal content for the latest user message.
// Only OpenAI-compatible providers accept content-part messages. The legacy
// direct adapters (Bedrock/Vertex) cannot take image parts, so requests with
// images are refused with a clear 400 before any provider contact.
// ============================================================
var MULTIMODAL_PROVIDERS = ['litellm', 'openrouter', 'azure'];
function applyImageAttachments(messages, images) {
if (!Array.isArray(images) || !images.length) return messages;
if (MULTIMODAL_PROVIDERS.indexOf(activeProvider) === -1) {
var unsupported = new Error('Image attachments require an OpenAI-compatible provider (LiteLLM, OpenRouter, or Azure). The active provider cannot accept image input, so no request was sent.');
unsupported.statusCode = 400;
unsupported.code = 'IMAGES_UNSUPPORTED_PROVIDER';
throw unsupported;
}
var out = messages.slice();
var index = out.length - 1;
while (index >= 0 && !(out[index] && out[index].role === 'user')) index--;
if (index < 0) return out; // No user message to attach to.
var message = out[index];
var content = Array.isArray(message.content) ? message.content.slice() : [{ type: 'text', text: String(message.content || '') }];
images.forEach(function (image) {
content.push({ type: 'image_url', image_url: { url: 'data:' + image.mimeType + ';base64,' + image.dataBase64 } });
});
out[index] = Object.assign({}, message, { content: content });
return out;
}
// Preserve unrecognized provider statuses; only explicit successful stops are complete. // Preserve unrecognized provider statuses; only explicit successful stops are complete.
function normalizeFinishReason(reason) { function normalizeFinishReason(reason) {
if (typeof reason !== 'string') return reason ?? null; if (typeof reason !== 'string') return reason ?? null;
@ -455,6 +484,8 @@ async function callAIStream(messages, options, onToken) {
await assertModelAllowed(model, options); await assertModelAllowed(model, options);
} }
messages = applyImageAttachments(messages, options.images);
var client = null; var client = null;
var provider = null; var provider = null;
if (activeProvider === 'litellm' && litellmClient) { if (activeProvider === 'litellm' && litellmClient) {
@ -527,6 +558,8 @@ async function callAI(messages, options) {
await assertModelAllowed(model, options); await assertModelAllowed(model, options);
} }
messages = applyImageAttachments(messages, options.images);
try { try {
var result; var result;
@ -731,4 +764,4 @@ async function discoverModels() {
return discovered; return discovered;
} }
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient }; module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient, applyImageAttachments };

View file

@ -1,4 +1,7 @@
// Conversation text uses an exact character budget, not an estimated model token limit. // Conversation text uses an exact character budget, not an estimated model token limit.
// Image attachments are excluded from this budget: UTF-16 code units count the
// text only. Attachments are validated separately below and are input-only —
// they are never persisted in saved chats or transcripts.
const DEFAULT_CONVERSATION_CHARS = 120000; const DEFAULT_CONVERSATION_CHARS = 120000;
const MAX_SAVED_CHAT_BYTES = 8 * 1024 * 1024; const MAX_SAVED_CHAT_BYTES = 8 * 1024 * 1024;
@ -94,6 +97,45 @@ function savedJobs(imageJobs) {
return imageJobs.map(job => ({ jobId: job.jobId })); return imageJobs.map(job => ({ jobId: job.jobId }));
} }
// Image attachments accompany the outgoing clinical question only.
// Strict limits: PNG/JPEG/WebP, canonical base64, <= 4 images,
// <= 5 MiB decoded per image, <= 10 MiB decoded total. Invalid input is
// rejected before any retrieval or provider call. The conversation budget
// above counts the text only; image bytes never enter the UTF-16 count.
const ATTACHMENT_MIME_TYPES = ['image/png', 'image/jpeg', 'image/webp'];
const MAX_ATTACHMENT_IMAGES = 4;
const MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024;
const MAX_ATTACHMENT_TOTAL_BYTES = 10 * 1024 * 1024;
const CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
function validateAttachments(images) {
if (images === undefined || images === null) return [];
if (!Array.isArray(images)) throw failure('Image attachments must be a list.', 400, 'INVALID_ATTACHMENTS');
if (images.length > MAX_ATTACHMENT_IMAGES) throw failure('A maximum of 4 images can be attached to one question.', 400, 'INVALID_ATTACHMENTS');
let total = 0;
return images.map(function (image) {
if (!image || typeof image !== 'object' || Array.isArray(image) ||
typeof image.dataBase64 !== 'string' || typeof image.mimeType !== 'string') {
throw failure('Each image attachment needs a base64 payload and a MIME type.', 400, 'INVALID_ATTACHMENTS');
}
if (!ATTACHMENT_MIME_TYPES.includes(image.mimeType)) {
throw failure('Only PNG, JPEG and WebP image attachments are allowed.', 400, 'INVALID_ATTACHMENTS');
}
if (image.dataBase64.length % 4 !== 0 || !CANONICAL_BASE64.test(image.dataBase64)) {
throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS');
}
const decoded = Buffer.from(image.dataBase64, 'base64');
if (decoded.toString('base64') !== image.dataBase64) {
throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS');
}
if (decoded.length === 0) throw failure('Image attachments must not be empty.', 400, 'INVALID_ATTACHMENTS');
if (decoded.length > MAX_ATTACHMENT_BYTES) throw failure('Each image attachment is limited to 5 MiB.', 400, 'INVALID_ATTACHMENTS');
total += decoded.length;
if (total > MAX_ATTACHMENT_TOTAL_BYTES) throw failure('Image attachments are limited to 10 MiB in total.', 400, 'INVALID_ATTACHMENTS');
return { dataBase64: image.dataBase64, mimeType: image.mimeType };
});
}
function savedChatPayload(body) { function savedChatPayload(body) {
const messages = validateMessages(body.messages).map(function(message, index) { const messages = validateMessages(body.messages).map(function(message, index) {
const original = body.messages[index]; const original = body.messages[index];
@ -128,4 +170,4 @@ function savedChatPayload(body) {
return payload; return payload;
} }
module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, savedChatPayload }; module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, validateAttachments, savedChatPayload };

View file

@ -0,0 +1,411 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const express = require('express');
const { JSDOM } = require('jsdom');
const { pathToFileURL } = require('node:url');
const policy = require('../src/utils/clinicalConversation');
const answer = require('../src/utils/clinicalAnswer');
const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');
const quiet = { log() {}, warn() {}, error() {}, info() {} };
const b64 = bytes => Buffer.from(bytes).toString('base64');
const canonical = (length, fill = 7) => Buffer.alloc(length, fill).toString('base64');
const png = { dataBase64: canonical(16), mimeType: 'image/png' };
const jpeg = { dataBase64: canonical(32), mimeType: 'image/jpeg' };
const webp = { dataBase64: canonical(48), mimeType: 'image/webp' };
function server(options = {}) {
const calls = { ai: [], search: [], writes: [], health: [] };
let saved;
const db = {
async getSetting(key) {
if (options.dbError) throw Error('private diagnostic');
if (key.endsWith('conversation_chars')) return options.legacyLimit ?? 'broken';
return options.model || null;
},
async get(sql, params) {
if (sql.includes('COUNT')) return { cnt: 0 };
return saved && params[1] === 7 ? { id: 1, title: saved[1], payload: saved[2] } : null;
},
async run(sql, params) { calls.writes.push(sql); saved = params; return { lastInsertRowid: 1 }; }
};
const ai = async (messages, settings) => {
calls.ai.push({ messages, settings });
return { content: 'Complete supported answer. [1]', finishReason: 'stop', model: 'synthetic' };
};
const source = { number: 1, title: 'Synthetic source', excerpt: 'Synthetic reference.', page: 9 };
const mocks = {
express, axios: { async post(url, payload) { calls.images.push(payload); return { data: { data: [{ b64_json: 'c3ludGhldGlj' }] } }; } }, crypto: require('node:crypto'), '../db/database': db,
'../middleware/auth': { authMiddleware() {} }, '../utils/ai': options.ai || { callAI: ai, callAIStream: ai },
'../utils/errors': { gatewayUrl: path => 'http://synthetic.invalid' + path }, '../utils/litellm': { getLiteLLMHeaders: () => ({}) }, '../utils/logger': { audit() {}, error() {} },
'../utils/crypto': { encryptString: value => 'encrypted:' + value, decryptString: value => value.replace(/^encrypted:/, '') },
'../utils/redis': { async getJson() { return null; }, async setJson() {} }, '../utils/clinicalPromptPool': { createClinicalPromptPool: () => ({}) },
'../utils/generatedImages': require('../src/utils/generatedImages'),
'../utils/generatedImageLinks': require('../src/utils/generatedImageLinks'),
'../utils/imageTool': require('../src/utils/imageTool'),
'../utils/clinicalPrompts': require('../src/utils/clinicalPrompts'),
'../utils/clinicalMcpClient': {
async semanticSearch(query) { calls.search.push(query); return {}; }, async getMcpHealth() { calls.health.push('health'); return {}; }
},
'../utils/clinicalRetrieval': {
cleanSourceExcerpt: require('../src/utils/clinicalRetrieval').cleanSourceExcerpt,
normalizeMcpSearchResponse: () => options.noSources ? [] : [source],
normalizeMcpMultimodalResponse: () => [], dedupeSources: value => value,
isVisualSourceQuery: () => false, classifyAndRerankMultimodalResults: async () => []
},
'../utils/clinicalAnswer': answer, '../utils/clinicalConversation': policy
};
const module = { exports: {} };
vm.runInNewContext(read('src/routes/clinicalAssistant.js'), {
module, exports: module.exports, console: quiet, Buffer, Map,
process: { env: { CLINICAL_ASSISTANT_MCP_WARMUP: 'false', CLINICAL_ASSISTANT_CONVERSATION_CHARS: options.limit, LITELLM_API_BASE: 'http://synthetic.invalid' } }, setTimeout() {},
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; }
});
async function request(method, endpoint, body, userId = 7) {
const handler = module.exports.stack.find(layer => layer.route && layer.route.path === endpoint && layer.route.methods[method]).route.stack.find(layer => layer.method === method).handle;
const res = {
statusCode: 200, headers: {}, events: '', body: null,
status(code) { this.statusCode = code; return this; }, json(body) { this.body = body; return this; },
setHeader(k, v) { this.headers[k] = v; }, write(text) { this.events += text; }, end() {}, flushHeaders() {}
};
await handler({ body, params: { id: 1 }, user: { id: userId } }, res);
return res;
}
return { calls, request };
}
test('attachment policy validates MIME, canonical base64, per-image/count/total limits without provider contact', async () => {
const oversize = { dataBase64: canonical(5 * 1024 * 1024 + 1), mimeType: 'image/png' };
const big = { dataBase64: canonical(4 * 1024 * 1024), mimeType: 'image/jpeg' };
const invalid = [
{ images: [{ ...png, mimeType: 'image/svg+xml' }] },
{ images: [{ ...png, mimeType: 'image/gif' }] },
{ images: [{ ...png, mimeType: 'application/octet-stream' }] },
{ images: [{ ...png, dataBase64: 'not base64!!!' }] },
{ images: [{ ...png, dataBase64: 'SGVsbG8' }] },
{ images: [{ ...png, dataBase64: 'AB==' }] }, // non-canonical trailing bits
{ images: [{ ...png, dataBase64: '' }] },
{ images: [{ ...png, dataBase64: 42 }] },
{ images: [{ mimeType: 'image/png' }] },
{ images: [{ dataBase64: png.dataBase64 }] },
{ images: [null] }, { images: ['png'] }, { images: [{}] },
{ images: 'not-a-list' },
{ images: [png, jpeg, webp, png, jpeg] },
{ images: [oversize] },
{ images: [big, big, big] }
];
for (const body of invalid) {
for (const endpoint of ['/clinical-assistant/chat', '/clinical-assistant/chat/stream']) {
const app = server();
const result = await app.request('post', endpoint, { message: 'What is the recommended dose of acetaminophen for a febrile three year old?', history: [], ...body });
assert.equal(result.statusCode, 400, JSON.stringify(body.images) + ' on ' + endpoint);
assert.equal(result.body.code, 'INVALID_ATTACHMENTS');
assert.equal(app.calls.ai.length + app.calls.search.length, 0, 'no provider or retrieval call on ' + endpoint);
}
}
});
test('valid images are normalized, ride the outgoing question only, and are excluded from the UTF-16 text budget', async () => {
const app = server({ limit: '1000' });
const images = [{ ...png, extra: 'ignored' }, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }];
const question = 'What about monitoring?';
const result = await app.request('post', '/clinical-assistant/chat', { message: question, history: [{ role: 'user', content: 'x'.repeat(500) }], images });
assert.equal(result.statusCode, 200);
assert.equal(app.calls.search.length, 1);
assert.equal(app.calls.ai.length, 2, 'rewrite plus generation');
const generation = app.calls.ai.at(-1);
assert.deepEqual(generation.settings.images, [png, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }], 'normalized, extra fields dropped');
assert.equal(typeof generation.messages[1].content, 'string', 'the route keeps prompt text; ai.js builds multimodal parts');
assert.ok(generation.messages[1].content.includes(question));
assert.equal(app.calls.ai[0].settings.images, undefined, 'the retrieval rewrite is text-only');
const atLimit = server({ limit: '1000' });
const budgetBody = { message: 'x', history: [{ role: 'user', content: 'x'.repeat(999) }], images: [png, jpeg, webp, png] };
assert.equal((await atLimit.request('post', '/clinical-assistant/chat', budgetBody)).statusCode, 200, 'images are excluded from the UTF-16 budget');
assert.equal((await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' })).statusCode, 413, 'the text boundary is unchanged by images');
const over = await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' });
assert.equal(over.body.budget.unit, 'characters');
});
test('greeting and retrieval-empty direct responses validate attachments before responding', async () => {
const bad = server();
const rejected = await bad.request('post', '/clinical-assistant/chat/stream', { message: 'thanks', history: [], images: [{ dataBase64: canonical(8), mimeType: 'image/bmp' }] });
assert.equal(rejected.statusCode, 400);
assert.equal(bad.calls.ai.length + bad.calls.search.length, 0);
const good = server();
const direct = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] });
assert.equal(direct.statusCode, 200);
assert.match(direct.body.answer, /clinical question/);
assert.equal(good.calls.ai.length + good.calls.search.length, 0, 'greeting never reaches providers');
});
test('handoff ignores image attachments and never forwards them', async () => {
const app = server();
const result = await app.request('post', '/clinical-assistant/handoff', { history: [{ role: 'user', content: 'Known facts.' }], images: [{ dataBase64: canonical(8), mimeType: 'image/png' }] });
assert.equal(result.statusCode, 200);
assert.equal(app.calls.ai.length, 1);
assert.ok(app.calls.ai[0].messages[1].content.includes('Known facts.'));
assert.equal(app.calls.ai[0].settings.images, undefined);
assert.doesNotMatch(JSON.stringify(app.calls.ai[0].messages), /base64/);
});
function openaiAdapter() {
const requests = [];
class OpenAI {
constructor() {
this.chat = { completions: { create: async r => {
requests.push(r);
if (r.stream) return [{ choices: [{ delta: { content: 'streamed answer' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }];
return { choices: [{ message: { content: 'Synthetic multimodal answer.' }, finish_reason: 'stop' }], usage: null };
} } };
}
}
const mocks = {
openai: { OpenAI },
'./models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['synthetic-vision']); } },
'./generationOptions': require('../src/utils/generationOptions'),
'./logger': { apiCall() {}, error() {} },
'../db/database': { async getSetting() { return null; } }
};
const module = { exports: {} };
vm.runInNewContext(read('src/utils/ai.js'), {
module, exports: module.exports, console: quiet, TextDecoder,
process: { env: { LITELLM_API_BASE: 'http://synthetic.invalid', LITELLM_API_KEY: 'sk-synthetic' } },
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
});
return { requests, ai: module.exports };
}
test('LiteLLM generation builds multimodal parts only on the latest user message and keeps tools/temperature/streaming', async () => {
const tools = [{ type: 'function', function: { name: 'generate_image', parameters: { type: 'object' } } }];
const { requests, ai } = openaiAdapter();
const messages = [
{ role: 'system', content: 'System behavior.' },
{ role: 'user', content: 'Earlier turn.' },
{ role: 'assistant', content: 'Earlier answer.' },
{ role: 'user', content: 'Latest question.' }
];
const result = await ai.callAI(messages, { model: 'synthetic-vision', images: [png, jpeg], tools, temperature: 0.15, maxTokens: 2600 });
assert.equal(result.content, 'Synthetic multimodal answer.');
const request = requests[0];
assert.equal(request.model, 'synthetic-vision');
assert.equal(request.temperature, 0.15);
assert.equal(request.max_tokens, 2600);
assert.deepEqual(request.tools, tools);
assert.equal(request.tool_choice, 'auto');
assert.equal(request.parallel_tool_calls, false);
assert.deepEqual(request.messages[0], { role: 'system', content: 'System behavior.' });
assert.deepEqual(request.messages[1], { role: 'user', content: 'Earlier turn.' });
assert.deepEqual(request.messages[2], { role: 'assistant', content: 'Earlier answer.' });
assert.equal(request.messages[3].role, 'user');
assert.equal(request.messages[3].content[0].type, 'text');
assert.equal(request.messages[3].content[0].text, 'Latest question.');
assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[1])), { type: 'image_url', image_url: { url: 'data:image/png;base64,' + png.dataBase64 } });
assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[2])), { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,' + jpeg.dataBase64 } });
const streamed = await ai.callAIStream(messages, { model: 'synthetic-vision', images: [png], tools, temperature: 0.15, maxTokens: 2600 }, () => {});
assert.equal(streamed.content, 'streamed answer');
const streamRequest = requests[1];
assert.equal(streamRequest.stream, true);
assert.deepEqual(streamRequest.tools, tools);
assert.equal(streamRequest.messages[3].content.length, 2);
assert.deepEqual(JSON.parse(JSON.stringify(streamRequest.messages[3].content[1].image_url)), { url: 'data:image/png;base64,' + png.dataBase64 });
const noImages = await ai.callAI(messages, { model: 'synthetic-vision' });
assert.equal(noImages.content, 'Synthetic multimodal answer.');
assert.deepEqual(requests[2].messages[3], { role: 'user', content: 'Latest question.' });
});
test('legacy direct provider adapters refuse image attachments with 400 before contacting the provider', async () => {
const sdkCalls = [];
class InvokeModelCommand { constructor(input) { this.input = input; } }
class ConverseCommand { constructor(input) { this.input = input; } }
const mocks = {
openai: { OpenAI: class { constructor() { throw Error('Unexpected OpenAI client'); } } },
'./models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['anthropic.claude-synthetic', 'amazon/nova-lite', 'google/gemini-2.5-flash']); } },
'./generationOptions': require('../src/utils/generationOptions'),
'./logger': { apiCall() {}, error() {} },
'../db/database': { async getSetting() { return null; } },
'@aws-sdk/client-bedrock-runtime': {
InvokeModelCommand, ConverseCommand,
BedrockRuntimeClient: class { async send(command) { sdkCalls.push(command); return { body: Buffer.from('{}') }; } }
},
'@google-cloud/vertexai': {
VertexAI: class { getGenerativeModel() { return { async generateContent(request) { sdkCalls.push(request); return { response: { candidates: [] } }; } }; } }
}
};
const module = { exports: {} };
vm.runInNewContext(read('src/utils/ai.js'), {
module, exports: module.exports, console: quiet, TextDecoder,
process: { env: { AI_PROVIDER: 'bedrock', AWS_BEDROCK_REGION: 'synthetic' } },
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
});
const messages = [{ role: 'user', content: 'Question with image.' }];
await assert.rejects(module.exports.callAI(messages, { model: 'anthropic.claude-synthetic', images: [png] }), error => {
assert.equal(error.statusCode, 400);
assert.equal(error.code, 'IMAGES_UNSUPPORTED_PROVIDER');
assert.match(error.message, /OpenAI-compatible provider/);
return true;
});
await assert.rejects(module.exports.callAIStream(messages, { model: 'anthropic.claude-synthetic', images: [png] }, () => {}), error => error.statusCode === 400);
assert.equal(sdkCalls.length, 0, 'refused before any provider contact');
const vertexModule = { exports: {} };
vm.runInNewContext(read('src/utils/ai.js'), {
module: vertexModule, exports: vertexModule.exports, console: quiet, TextDecoder,
process: { env: { AI_PROVIDER: 'vertex', GOOGLE_VERTEX_PROJECT: 'synthetic' } },
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
});
await assert.rejects(vertexModule.exports.callAI(messages, { model: 'google/gemini-2.5-flash', images: [png] }), error => error.statusCode === 400);
assert.equal(sdkCalls.length, 0);
});
function browserUI(options = {}) {
const dom = new JSDOM('<div id="assistant-tab">' + read('public/components/assistant.html') + '</div>', {
url: 'https://example.test/', runScripts: 'outside-only'
});
const calls = { stream: [], handoff: [], save: [] };
const toasts = [];
const escapeHtml = text => String(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
const context = {
window: dom.window, document: dom.window.document, navigator: dom.window.navigator,
console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require('node:crypto').webcrypto,
FileReader: dom.window.FileReader, File: dom.window.File,
setTimeout() {}, showToast(text, kind) { toasts.push([String(text), kind || '']); }, escapeHtml, escapeAttr: escapeHtml,
renderAssistantMarkdown: text => escapeHtml(text), renderSourcesList: () => '', EMPTY_PROMPT_SETS: [[]],
createAssistantExporter: () => ({ invalidate() {}, exportAnswerPdf() {} }),
createAssistantImageStore: () => ({ renderGeneratedImage: src => '<img src="' + src + '">', clear() {} }),
isImageRequest: () => false,
fetchAssistantStatus: async () => ({ success: true, conversationChars: options.limit || 120000 }),
fetchAssistantExamples: async () => ({}), fetchSavedAssistantChats: async () => ({ success: true, chats: [] }),
openAssistantStream: async payload => {
calls.stream.push(payload);
if (options.stream) return options.stream(payload);
return new Response('event: done\ndata: ' + JSON.stringify({ success: true, answer: 'Complete response.', sources: [] }) + '\n\n');
},
fetchAssistantChat: async payload => ({ success: true, answer: 'Fallback answer.' }),
saveAssistantChat: async payload => { calls.save.push(payload); return { success: true }; },
requestAssistantHandoff: async history => { calls.handoff.push(history); return { success: true, summary: 'Explicit handoff.' }; }
};
vm.createContext(context);
vm.runInContext(read('public/js/clinicalAssistant.js').replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, ''), context);
context.bindEvents();
context.conversationChars = options.limit || 120000;
return { context, document: dom.window.document, calls, toasts, dom };
}
async function realApiHelpers() {
const module = await import(pathToFileURL(path.join(root, 'public/js/assistant/api.js')).href);
return module;
}
function attachFile(ui, name, type, bytes) {
const input = ui.document.getElementById('assistant-attach-input');
const file = new ui.dom.window.File([bytes], name, { type });
Object.defineProperty(input, 'files', { value: [file], configurable: true });
input.dispatchEvent(new ui.dom.window.Event('change'));
}
async function tick() { for (let i = 0; i < 25; i++) await new Promise(resolve => setImmediate(resolve)); }
test('UI attaches only PNG/JPEG/WebP with thumbnails, removal and client-side limits', async () => {
const api = await realApiHelpers();
const ui = browserUI();
ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
const bytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]);
attachFile(ui, 'photo.png', 'image/png', bytes);
attachFile(ui, 'scan.jpeg', 'image/jpeg', bytes);
await tick();
const wrap = ui.document.getElementById('assistant-attachments');
assert.equal(wrap.hidden, false);
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2);
assert.ok(wrap.querySelector('img').getAttribute('src').startsWith('data:image/png;base64,'));
attachFile(ui, 'notes.txt', 'text/plain', new Uint8Array([1, 2, 3]));
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2);
assert.ok(ui.toasts.some(([text]) => /PNG, JPEG and WebP/.test(text)));
attachFile(ui, 'third.webp', 'image/webp', bytes);
attachFile(ui, 'fourth.png', 'image/png', bytes);
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
attachFile(ui, 'fifth.png', 'image/png', bytes);
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
assert.ok(ui.toasts.some(([text]) => /maximum of 4/.test(text)));
attachFile(ui, 'huge.png', 'image/png', new Uint8Array(5 * 1024 * 1024 + 1));
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
assert.ok(ui.toasts.some(([text]) => /5 MiB/.test(text)));
wrap.querySelector('[data-assistant-remove-attachment="0"]').click();
await tick();
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 3);
assert.equal(ui.context.attachments.length, 3);
ui.dom.window.close();
});
test('UI sends images only with the question, clears them on success, and keeps them out of handoff, saves and the budget label', async () => {
const api = await realApiHelpers();
const ui = browserUI({ limit: 1000 });
ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
const bytes = new Uint8Array([1, 2, 3, 4]);
attachFile(ui, 'photo.png', 'image/png', bytes);
await tick();
const expected = [{ dataBase64: Buffer.from(bytes).toString('base64'), mimeType: 'image/png' }];
ui.context.restoreSavedChat({ messages: [{ role: 'user', content: 'x'.repeat(999) }], lastAnswer: '' });
const input = ui.document.getElementById('assistant-input');
input.value = 'x';
input.dispatchEvent(new ui.dom.window.Event('input'));
const label = ui.document.getElementById('assistant-context-budget');
assert.equal(label.textContent, '1,000 / 1,000 conversation characters (UTF-16 code units).', 'budget label stays text-only');
assert.doesNotMatch(label.textContent, /image/i);
// Handoff and saved chats never carry attachments.
await ui.context.requestHandoff();
assert.equal(ui.calls.handoff.length, 1);
assert.equal(ui.calls.handoff[0].images, undefined);
assert.equal(JSON.stringify(ui.calls.handoff[0]).includes('dataBase64'), false);
await ui.context.saveCurrentChat();
assert.equal(ui.calls.save[0].images, undefined);
assert.equal(JSON.stringify(ui.calls.save[0]).includes('dataBase64'), false);
await ui.context.onAsk();
await tick();
assert.deepEqual(JSON.parse(JSON.stringify(ui.calls.stream[0].images)), expected);
assert.equal(ui.context.attachments.length, 0, 'cleared on successful send');
assert.equal(ui.document.getElementById('assistant-attachments').hidden, true);
assert.equal(ui.context.messages.length, 3, 'question and answer appended');
// New chat clears any later attachments.
attachFile(ui, 'again.png', 'image/png', bytes);
await tick();
assert.equal(ui.context.attachments.length, 1);
await ui.context.clearConversation();
assert.equal(ui.context.attachments.length, 0);
assert.equal(ui.document.getElementById('assistant-attachments').hidden, true);
attachFile(ui, 'kept.png', 'image/png', bytes);
await tick();
assert.equal(ui.context.attachments.length, 1);
const failing = browserUI({ stream: () => new Response(JSON.stringify({ error: 'Image rejected' }), { status: 400 }) });
failing.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
failing.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
attachFile(failing, 'kept.png', 'image/png', bytes);
await tick();
failing.document.getElementById('assistant-input').value = 'Question with attachment';
await failing.context.onAsk();
await tick();
assert.equal(failing.context.attachments.length, 1, 'rejected send keeps the attachments for correction');
assert.equal(failing.document.getElementById('assistant-attachments').hidden, false);
ui.dom.window.close();
failing.dom.window.close();
});