Merge branch 'fix/attach-images-20260907'
This commit is contained in:
commit
81abc60591
9 changed files with 630 additions and 10 deletions
|
|
@ -72,6 +72,16 @@ show me the image/figure
|
|||
|
||||
Clinical answer response caching is intentionally disabled. Redis can support prompt suggestions and operational metadata, but final answers should be generated from current retrieval context.
|
||||
|
||||
## Image Attachments
|
||||
|
||||
Users can attach up to 4 images (PNG, JPEG, WebP) to an outgoing clinical question. Attachments are **input-only**:
|
||||
|
||||
- They are validated client-side and authoritatively on the server (MIME allowlist, canonical base64, ≤ 5 MiB per image, ≤ 4 images, ≤ 10 MiB decoded total). Invalid input is rejected with 400 before any retrieval or provider call.
|
||||
- They are sent **only** with the outgoing clinical question, never with handoff summaries, image generation, saved chats, downloads or transcripts.
|
||||
- The conversation budget counts text only: images are excluded from the UTF-16 code-unit count. The server still validates every request.
|
||||
- Only OpenAI-compatible providers (LiteLLM, OpenRouter, Azure) receive them as multimodal content parts (`text` + `image_url` data URIs) on the latest user message; the system/retrieval/history structure is unchanged. Legacy direct adapters (Bedrock/Vertex) refuse with a clear 400 before contacting the provider.
|
||||
- Attachments clear on a successful send and on New chat; a rejected send keeps them for correction.
|
||||
|
||||
## Settings
|
||||
|
||||
Important settings include:
|
||||
|
|
@ -93,5 +103,6 @@ Add or update tests when changing:
|
|||
- named-source provenance behavior,
|
||||
- table rendering,
|
||||
- image intent routing,
|
||||
- image attachment validation and multimodal payload shape,
|
||||
- MCP result normalization,
|
||||
- model discovery or settings behavior.
|
||||
|
|
|
|||
|
|
@ -44,8 +44,11 @@
|
|||
<textarea id="assistant-input" rows="3" placeholder="Ask a focused clinical question..." autocomplete="off" aria-describedby="assistant-context-budget assistant-context-warning"></textarea>
|
||||
<p id="assistant-context-budget" class="assistant-muted" aria-live="polite">Loading conversation limit; history and draft are counted in UTF-16 code units. The server validates each request.</p>
|
||||
<div id="assistant-context-warning" role="alert" hidden></div>
|
||||
<div id="assistant-attachments" class="assistant-attachments" aria-label="Attached images" hidden></div>
|
||||
<textarea id="assistant-handoff-text" hidden readonly></textarea>
|
||||
<div class="assistant-composer-footer">
|
||||
<label class="assistant-attach" for="assistant-attach-input" title="Attach up to 4 images (PNG, JPEG, WebP)"><i class="fas fa-paperclip"></i> Attach images</label>
|
||||
<input type="file" id="assistant-attach-input" accept="image/png,image/jpeg,image/webp" multiple hidden>
|
||||
<label class="assistant-check"><input type="checkbox" id="assistant-include-context" checked> retrieve broader context</label>
|
||||
<button id="btn-assistant-cancel" class="btn-sm btn-ghost" type="button" hidden><i class="fas fa-stop"></i> Cancel search</button>
|
||||
<button id="btn-assistant-send" class="btn-generate" type="submit"><i class="fas fa-paper-plane"></i> Ask</button>
|
||||
|
|
|
|||
|
|
@ -56,6 +56,14 @@
|
|||
.assistant-cite { display:inline-flex; align-items:center; justify-content:center; min-width:18px; height:18px; padding:0 6px; margin:0 1px; border-radius:999px; background:var(--purple-light); color:var(--purple); font-size:10px; font-weight:800; text-decoration:none; vertical-align:baseline; border:1px solid rgba(124,58,237,.18); text-transform:uppercase; letter-spacing:.03em; }
|
||||
.assistant-cite:hover { background:var(--purple); color:white; text-decoration:none; }
|
||||
.assistant-composer { border-top:1px solid var(--g200); padding:12px; background:white; display:grid; gap:8px; }
|
||||
.assistant-attachments { display:flex; flex-wrap:wrap; gap:8px; }
|
||||
.assistant-attachments[hidden] { display:none; }
|
||||
.assistant-attachment { position:relative; width:64px; height:64px; border:1px solid var(--g200); border-radius:8px; overflow:hidden; background:var(--g50); }
|
||||
.assistant-attachment img { width:100%; height:100%; object-fit:cover; display:block; }
|
||||
.assistant-attachment-remove { position:absolute; top:3px; right:3px; width:18px; height:18px; border:none; border-radius:50%; background:rgba(0,0,0,.65); color:#fff; font-size:10px; line-height:18px; cursor:pointer; padding:0; }
|
||||
.assistant-attachment-remove:hover { background:rgba(0,0,0,.85); }
|
||||
.assistant-attach { display:inline-flex; align-items:center; gap:6px; font-size:12px; color:var(--g600); cursor:pointer; padding:6px 10px; border:1px dashed var(--g300); border-radius:8px; }
|
||||
.assistant-attach:hover { border-color:var(--blue); color:var(--blue); }
|
||||
.assistant-composer textarea, .assistant-side textarea { width:100%; border:1.5px solid var(--g300); border-radius:10px; padding:10px 12px; resize:vertical; font-family:inherit; font-size:13px; outline:none; }
|
||||
.assistant-composer textarea:focus, .assistant-side textarea:focus { border-color:var(--blue); box-shadow:0 0 0 3px var(--blue-light); }
|
||||
.assistant-composer-footer { display:flex; justify-content:space-between; align-items:center; gap:10px; }
|
||||
|
|
|
|||
|
|
@ -103,3 +103,19 @@ export function deleteSavedAssistantChat(id) {
|
|||
credentials: 'same-origin'
|
||||
}).then(function(r) { return r.json(); });
|
||||
}
|
||||
|
||||
// Image attachments are input-only: they ride the outgoing clinical question
|
||||
// payload and are never persisted or sent with handoff. The server enforces
|
||||
// the same limits authoritatively; these mirror them for early feedback.
|
||||
export var assistantAttachmentLimits = Object.freeze({
|
||||
mimeTypes: ['image/png', 'image/jpeg', 'image/webp'],
|
||||
maxImages: 4,
|
||||
maxImageBytes: 5 * 1024 * 1024,
|
||||
maxTotalBytes: 10 * 1024 * 1024
|
||||
});
|
||||
|
||||
export function assistantAttachmentPayload(attachments) {
|
||||
return (Array.isArray(attachments) ? attachments : []).map(function(attachment) {
|
||||
return { dataBase64: attachment.dataBase64, mimeType: attachment.mimeType };
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,10 +20,13 @@ import {
|
|||
openAssistantStream,
|
||||
startAssistantImageJob,
|
||||
saveAssistantChat,
|
||||
requestAssistantHandoff
|
||||
requestAssistantHandoff,
|
||||
assistantAttachmentLimits,
|
||||
assistantAttachmentPayload
|
||||
} from './assistant/api.js';
|
||||
var initialized = false;
|
||||
var messages = [];
|
||||
var attachments = []; // Input-only images: never persisted, never sent with handoff.
|
||||
var lastAnswer = '';
|
||||
var lastSources = [];
|
||||
var mermaidReady = false;
|
||||
|
|
@ -63,6 +66,7 @@ import {
|
|||
var exportBtn = document.getElementById('btn-assistant-export-pdf');
|
||||
var imageBtn = document.getElementById('btn-assistant-image');
|
||||
var imageClearBtn = document.getElementById('btn-assistant-image-clear');
|
||||
var attachInput = document.getElementById('assistant-attach-input');
|
||||
var input = document.getElementById('assistant-input');
|
||||
|
||||
if (form) form.addEventListener('submit', onAsk);
|
||||
|
|
@ -90,6 +94,7 @@ import {
|
|||
if (exportBtn) exportBtn.addEventListener('click', exportAnswerPdf);
|
||||
if (imageBtn) imageBtn.addEventListener('click', generateImage);
|
||||
if (imageClearBtn) imageClearBtn.addEventListener('click', clearGeneratedImage);
|
||||
if (attachInput) attachInput.addEventListener('change', onAttachFiles);
|
||||
document.addEventListener('click', onAssistantDocumentClick);
|
||||
if (input) input.addEventListener('keydown', function (e) {
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') onAsk(e);
|
||||
|
|
@ -162,16 +167,23 @@ import {
|
|||
var row = appendMessage('user', text);
|
||||
if (row && loading && loading.parentNode) loading.parentNode.insertBefore(row, loading);
|
||||
if (input) input.value = '';
|
||||
// Attached images are input-only and clear on a successful send.
|
||||
attachments = [];
|
||||
renderAttachments();
|
||||
exporter.invalidate();
|
||||
updateConversationBudget();
|
||||
};
|
||||
|
||||
return streamAssistantResponse({
|
||||
var payload = {
|
||||
message: text,
|
||||
idempotencyKey: crypto.randomUUID(),
|
||||
history: history,
|
||||
includeContext: !includeContext || includeContext.checked
|
||||
}, loading, request)
|
||||
};
|
||||
// Images ride the outgoing clinical question only, never handoff or saves.
|
||||
if (attachments.length) payload.images = assistantAttachmentPayload(attachments);
|
||||
|
||||
return streamAssistantResponse(payload, loading, request)
|
||||
.catch(function (err) {
|
||||
if (request.cancelled) return;
|
||||
setBusy(false, 'Error', true);
|
||||
|
|
@ -386,6 +398,73 @@ import {
|
|||
return row;
|
||||
}
|
||||
|
||||
function onAttachFiles(e) {
|
||||
var input = e && e.target;
|
||||
var files = Array.prototype.slice.call(input && input.files ? input.files : []);
|
||||
files.forEach(addAttachmentFile);
|
||||
if (input) input.value = ''; // Allow re-selecting the same file later.
|
||||
}
|
||||
|
||||
function addAttachmentFile(file) {
|
||||
var limits = assistantAttachmentLimits;
|
||||
if (!file) return;
|
||||
if (limits.mimeTypes.indexOf(file.type) === -1) {
|
||||
if (typeof showToast === 'function') showToast('Only PNG, JPEG and WebP images can be attached; "' + String(file.name || 'attachment') + '" was ignored.', 'error');
|
||||
return;
|
||||
}
|
||||
if (typeof file.size === 'number' && file.size > limits.maxImageBytes) {
|
||||
if (typeof showToast === 'function') showToast('Each image is limited to 5 MiB; "' + String(file.name || 'attachment') + '" was ignored.', 'error');
|
||||
return;
|
||||
}
|
||||
if (attachments.length >= limits.maxImages) {
|
||||
if (typeof showToast === 'function') showToast('A maximum of 4 images can be attached to one question.', 'error');
|
||||
return;
|
||||
}
|
||||
var totalBytes = attachments.reduce(function(sum, a) { return sum + (a.size || 0); }, 0);
|
||||
if (typeof file.size === 'number' && totalBytes + file.size > limits.maxTotalBytes) {
|
||||
if (typeof showToast === 'function') showToast('Attached images are limited to 10 MiB in total.', 'error');
|
||||
return;
|
||||
}
|
||||
var reader = new FileReader();
|
||||
reader.onload = function() {
|
||||
var dataUrl = String(reader.result || '');
|
||||
var comma = dataUrl.indexOf(',');
|
||||
if (comma === -1 || dataUrl.indexOf(';base64,') === -1) {
|
||||
if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error');
|
||||
return;
|
||||
}
|
||||
attachments.push({ dataBase64: dataUrl.slice(comma + 1), mimeType: file.type, size: file.size, src: dataUrl });
|
||||
renderAttachments();
|
||||
};
|
||||
reader.onerror = function() {
|
||||
if (typeof showToast === 'function') showToast('Could not read "' + String(file.name || 'attachment') + '".', 'error');
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
|
||||
function renderAttachments() {
|
||||
var wrap = document.getElementById('assistant-attachments');
|
||||
if (!wrap) return;
|
||||
wrap.innerHTML = '';
|
||||
attachments.forEach(function(attachment, index) {
|
||||
var item = document.createElement('div');
|
||||
item.className = 'assistant-attachment';
|
||||
var img = document.createElement('img');
|
||||
img.src = attachment.src || ('data:' + attachment.mimeType + ';base64,' + attachment.dataBase64);
|
||||
img.alt = 'Attached image ' + (index + 1);
|
||||
var remove = document.createElement('button');
|
||||
remove.type = 'button';
|
||||
remove.className = 'assistant-attachment-remove';
|
||||
remove.setAttribute('data-assistant-remove-attachment', String(index));
|
||||
remove.setAttribute('aria-label', 'Remove attached image ' + (index + 1));
|
||||
remove.textContent = '✕';
|
||||
item.appendChild(img);
|
||||
item.appendChild(remove);
|
||||
wrap.appendChild(item);
|
||||
});
|
||||
wrap.hidden = !attachments.length;
|
||||
}
|
||||
|
||||
function renderSuggestionButtons(suggestions) {
|
||||
var wrap = document.createElement('div');
|
||||
wrap.className = 'assistant-suggestion-buttons';
|
||||
|
|
@ -522,6 +601,12 @@ import {
|
|||
if (bubble && Array.isArray(bubble.assistantSources)) renderSources(bubble.assistantSources);
|
||||
return; // The native anchor navigates to the matching source in the refreshed panel.
|
||||
}
|
||||
var removeAttachment = e.target.closest('[data-assistant-remove-attachment]');
|
||||
if (removeAttachment) {
|
||||
attachments.splice(Number(removeAttachment.getAttribute('data-assistant-remove-attachment')), 1);
|
||||
renderAttachments();
|
||||
return;
|
||||
}
|
||||
var loadBtn = e.target.closest('[data-assistant-load-chat]');
|
||||
if (loadBtn) {
|
||||
e.preventDefault();
|
||||
|
|
@ -574,6 +659,8 @@ import {
|
|||
if (event && messages.length && !window.confirm('Start a new chat? Save or download this conversation first if you want to keep it.')) return;
|
||||
if (activeAssistantRequest) cancelAssistantSearch();
|
||||
messages = [];
|
||||
attachments = [];
|
||||
renderAttachments();
|
||||
lastAnswer = '';
|
||||
lastSources = [];
|
||||
lastGeneratedImageSrc = '';
|
||||
|
|
@ -931,6 +1018,7 @@ import {
|
|||
var send = document.getElementById('btn-assistant-send');
|
||||
var cancel = document.getElementById('btn-assistant-cancel');
|
||||
var input = document.getElementById('assistant-input');
|
||||
var attachInput = document.getElementById('assistant-attach-input');
|
||||
if (status) {
|
||||
status.classList.toggle('busy', !!isBusy);
|
||||
status.classList.toggle('error', !!isError);
|
||||
|
|
@ -948,6 +1036,7 @@ import {
|
|||
cancel.style.display = canCancel ? 'inline-flex' : 'none';
|
||||
}
|
||||
if (input) input.disabled = !!isBusy;
|
||||
if (attachInput) attachInput.disabled = !!isBusy;
|
||||
}
|
||||
|
||||
function sanitize(html) { return window.DOMPurify ? window.DOMPurify.sanitize(html, { ADD_ATTR: ['target'] }) : escapeHtml(html); }
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ var {
|
|||
finalizeAssistantAnswer
|
||||
} = require('../utils/clinicalAnswer');
|
||||
|
||||
var { conversationBudget, checkConversation, savedChatPayload } = require('../utils/clinicalConversation');
|
||||
var { conversationBudget, checkConversation, validateAttachments, savedChatPayload } = require('../utils/clinicalConversation');
|
||||
|
||||
var { DEFAULT_BEHAVIOR } = require('../utils/clinicalPrompts');
|
||||
|
||||
|
|
@ -193,7 +193,8 @@ router.post('/clinical-assistant/chat', async function(req, res) {
|
|||
model: prepared.chatModel || undefined,
|
||||
temperature: 0.15,
|
||||
tools: imageTool.tools,
|
||||
maxTokens: 2600
|
||||
maxTokens: 2600,
|
||||
images: prepared.images
|
||||
}));
|
||||
ai = await imageTool.dispatch(ai, { owner: req.user.id, workflow: 'clinical_assistant', body: req.body,
|
||||
imageContext: prepared.imageContext, messages: prepared.messages, options: assistantGenerationOptions({ model: prepared.chatModel || undefined, temperature: 0.15 }), callAI: callAI });
|
||||
|
|
@ -201,7 +202,7 @@ router.post('/clinical-assistant/chat', async function(req, res) {
|
|||
messages: prepared.messages,
|
||||
chatModel: prepared.chatModel,
|
||||
callAI: callAI,
|
||||
generationOptions: assistantGenerationOptions({ temperature: 0.15 })
|
||||
generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images })
|
||||
});
|
||||
var answer = finalized.answer;
|
||||
ai = finalized.ai;
|
||||
|
|
@ -258,7 +259,8 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) {
|
|||
model: prepared.chatModel || undefined,
|
||||
temperature: 0.15,
|
||||
tools: imageTool.tools,
|
||||
maxTokens: 2600
|
||||
maxTokens: 2600,
|
||||
images: prepared.images
|
||||
}), function(delta) {
|
||||
sendEvent('token', { token: delta });
|
||||
});
|
||||
|
|
@ -269,7 +271,7 @@ router.post('/clinical-assistant/chat/stream', async function(req, res) {
|
|||
messages: prepared.messages,
|
||||
chatModel: prepared.chatModel,
|
||||
callAI: callAI,
|
||||
generationOptions: assistantGenerationOptions({ temperature: 0.15 }),
|
||||
generationOptions: assistantGenerationOptions({ temperature: 0.15, images: prepared.images }),
|
||||
streamed: true,
|
||||
onRegenerating: function() { sendEvent('status', { message: 'Completing answer...' }); }
|
||||
});
|
||||
|
|
@ -354,6 +356,10 @@ async function prepareAssistantChat(body) {
|
|||
var checked = checkConversation(body.history, body.message, await getConversationLimit());
|
||||
var message = checked.message;
|
||||
var history = checked.history;
|
||||
// Image attachments are validated before any retrieval or provider call.
|
||||
// The conversation budget counts the text only; images are excluded from
|
||||
// the UTF-16 count and are never persisted in saved chats or transcripts.
|
||||
var images = validateAttachments(body.images);
|
||||
|
||||
var chatModel = await getSetting('clinical_assistant.chat_model', '') || await getSetting('models.default', '');
|
||||
var searchLimit = clampInt(await getSetting('clinical_assistant.search_limit', '8'), 3, 20, 8);
|
||||
|
|
@ -416,6 +422,7 @@ async function prepareAssistantChat(body) {
|
|||
var context = formatSourcesForPrompt(sources);
|
||||
return {
|
||||
message: message,
|
||||
images: images,
|
||||
imageContext: generatedImages.imageContext(message, history),
|
||||
chatModel: chatModel,
|
||||
sources: sources,
|
||||
|
|
|
|||
|
|
@ -139,6 +139,35 @@ if (activeProvider === 'openrouter' && !openrouter) {
|
|||
|
||||
console.log('🤖 Active AI provider:', activeProvider);
|
||||
|
||||
// ============================================================
|
||||
// IMAGE ATTACHMENTS — multimodal content for the latest user message.
|
||||
// Only OpenAI-compatible providers accept content-part messages. The legacy
|
||||
// direct adapters (Bedrock/Vertex) cannot take image parts, so requests with
|
||||
// images are refused with a clear 400 before any provider contact.
|
||||
// ============================================================
|
||||
var MULTIMODAL_PROVIDERS = ['litellm', 'openrouter', 'azure'];
|
||||
|
||||
function applyImageAttachments(messages, images) {
|
||||
if (!Array.isArray(images) || !images.length) return messages;
|
||||
if (MULTIMODAL_PROVIDERS.indexOf(activeProvider) === -1) {
|
||||
var unsupported = new Error('Image attachments require an OpenAI-compatible provider (LiteLLM, OpenRouter, or Azure). The active provider cannot accept image input, so no request was sent.');
|
||||
unsupported.statusCode = 400;
|
||||
unsupported.code = 'IMAGES_UNSUPPORTED_PROVIDER';
|
||||
throw unsupported;
|
||||
}
|
||||
var out = messages.slice();
|
||||
var index = out.length - 1;
|
||||
while (index >= 0 && !(out[index] && out[index].role === 'user')) index--;
|
||||
if (index < 0) return out; // No user message to attach to.
|
||||
var message = out[index];
|
||||
var content = Array.isArray(message.content) ? message.content.slice() : [{ type: 'text', text: String(message.content || '') }];
|
||||
images.forEach(function (image) {
|
||||
content.push({ type: 'image_url', image_url: { url: 'data:' + image.mimeType + ';base64,' + image.dataBase64 } });
|
||||
});
|
||||
out[index] = Object.assign({}, message, { content: content });
|
||||
return out;
|
||||
}
|
||||
|
||||
// Preserve unrecognized provider statuses; only explicit successful stops are complete.
|
||||
function normalizeFinishReason(reason) {
|
||||
if (typeof reason !== 'string') return reason ?? null;
|
||||
|
|
@ -455,6 +484,8 @@ async function callAIStream(messages, options, onToken) {
|
|||
await assertModelAllowed(model, options);
|
||||
}
|
||||
|
||||
messages = applyImageAttachments(messages, options.images);
|
||||
|
||||
var client = null;
|
||||
var provider = null;
|
||||
if (activeProvider === 'litellm' && litellmClient) {
|
||||
|
|
@ -527,6 +558,8 @@ async function callAI(messages, options) {
|
|||
await assertModelAllowed(model, options);
|
||||
}
|
||||
|
||||
messages = applyImageAttachments(messages, options.images);
|
||||
|
||||
try {
|
||||
var result;
|
||||
|
||||
|
|
@ -731,4 +764,4 @@ async function discoverModels() {
|
|||
return discovered;
|
||||
}
|
||||
|
||||
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient };
|
||||
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient, applyImageAttachments };
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
// Conversation text uses an exact character budget, not an estimated model token limit.
|
||||
// Image attachments are excluded from this budget: UTF-16 code units count the
|
||||
// text only. Attachments are validated separately below and are input-only —
|
||||
// they are never persisted in saved chats or transcripts.
|
||||
const DEFAULT_CONVERSATION_CHARS = 120000;
|
||||
const MAX_SAVED_CHAT_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
|
|
@ -94,6 +97,45 @@ function savedJobs(imageJobs) {
|
|||
return imageJobs.map(job => ({ jobId: job.jobId }));
|
||||
}
|
||||
|
||||
// Image attachments accompany the outgoing clinical question only.
|
||||
// Strict limits: PNG/JPEG/WebP, canonical base64, <= 4 images,
|
||||
// <= 5 MiB decoded per image, <= 10 MiB decoded total. Invalid input is
|
||||
// rejected before any retrieval or provider call. The conversation budget
|
||||
// above counts the text only; image bytes never enter the UTF-16 count.
|
||||
const ATTACHMENT_MIME_TYPES = ['image/png', 'image/jpeg', 'image/webp'];
|
||||
const MAX_ATTACHMENT_IMAGES = 4;
|
||||
const MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024;
|
||||
const MAX_ATTACHMENT_TOTAL_BYTES = 10 * 1024 * 1024;
|
||||
const CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
|
||||
|
||||
function validateAttachments(images) {
|
||||
if (images === undefined || images === null) return [];
|
||||
if (!Array.isArray(images)) throw failure('Image attachments must be a list.', 400, 'INVALID_ATTACHMENTS');
|
||||
if (images.length > MAX_ATTACHMENT_IMAGES) throw failure('A maximum of 4 images can be attached to one question.', 400, 'INVALID_ATTACHMENTS');
|
||||
let total = 0;
|
||||
return images.map(function (image) {
|
||||
if (!image || typeof image !== 'object' || Array.isArray(image) ||
|
||||
typeof image.dataBase64 !== 'string' || typeof image.mimeType !== 'string') {
|
||||
throw failure('Each image attachment needs a base64 payload and a MIME type.', 400, 'INVALID_ATTACHMENTS');
|
||||
}
|
||||
if (!ATTACHMENT_MIME_TYPES.includes(image.mimeType)) {
|
||||
throw failure('Only PNG, JPEG and WebP image attachments are allowed.', 400, 'INVALID_ATTACHMENTS');
|
||||
}
|
||||
if (image.dataBase64.length % 4 !== 0 || !CANONICAL_BASE64.test(image.dataBase64)) {
|
||||
throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS');
|
||||
}
|
||||
const decoded = Buffer.from(image.dataBase64, 'base64');
|
||||
if (decoded.toString('base64') !== image.dataBase64) {
|
||||
throw failure('Image attachments must use canonical base64.', 400, 'INVALID_ATTACHMENTS');
|
||||
}
|
||||
if (decoded.length === 0) throw failure('Image attachments must not be empty.', 400, 'INVALID_ATTACHMENTS');
|
||||
if (decoded.length > MAX_ATTACHMENT_BYTES) throw failure('Each image attachment is limited to 5 MiB.', 400, 'INVALID_ATTACHMENTS');
|
||||
total += decoded.length;
|
||||
if (total > MAX_ATTACHMENT_TOTAL_BYTES) throw failure('Image attachments are limited to 10 MiB in total.', 400, 'INVALID_ATTACHMENTS');
|
||||
return { dataBase64: image.dataBase64, mimeType: image.mimeType };
|
||||
});
|
||||
}
|
||||
|
||||
function savedChatPayload(body) {
|
||||
const messages = validateMessages(body.messages).map(function(message, index) {
|
||||
const original = body.messages[index];
|
||||
|
|
@ -128,4 +170,4 @@ function savedChatPayload(body) {
|
|||
return payload;
|
||||
}
|
||||
|
||||
module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, savedChatPayload };
|
||||
module.exports = { DEFAULT_CONVERSATION_CHARS, MAX_SAVED_CHAT_BYTES, conversationLimit, conversationBudget, checkConversation, validateAttachments, savedChatPayload };
|
||||
|
|
|
|||
411
test/assistant-image-attachments.test.js
Normal file
411
test/assistant-image-attachments.test.js
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
const { test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const express = require('express');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const { pathToFileURL } = require('node:url');
|
||||
const policy = require('../src/utils/clinicalConversation');
|
||||
const answer = require('../src/utils/clinicalAnswer');
|
||||
const root = path.join(__dirname, '..');
|
||||
const read = file => fs.readFileSync(path.join(root, file), 'utf8');
|
||||
const quiet = { log() {}, warn() {}, error() {}, info() {} };
|
||||
const b64 = bytes => Buffer.from(bytes).toString('base64');
|
||||
const canonical = (length, fill = 7) => Buffer.alloc(length, fill).toString('base64');
|
||||
const png = { dataBase64: canonical(16), mimeType: 'image/png' };
|
||||
const jpeg = { dataBase64: canonical(32), mimeType: 'image/jpeg' };
|
||||
const webp = { dataBase64: canonical(48), mimeType: 'image/webp' };
|
||||
|
||||
function server(options = {}) {
|
||||
const calls = { ai: [], search: [], writes: [], health: [] };
|
||||
let saved;
|
||||
const db = {
|
||||
async getSetting(key) {
|
||||
if (options.dbError) throw Error('private diagnostic');
|
||||
if (key.endsWith('conversation_chars')) return options.legacyLimit ?? 'broken';
|
||||
return options.model || null;
|
||||
},
|
||||
async get(sql, params) {
|
||||
if (sql.includes('COUNT')) return { cnt: 0 };
|
||||
return saved && params[1] === 7 ? { id: 1, title: saved[1], payload: saved[2] } : null;
|
||||
},
|
||||
async run(sql, params) { calls.writes.push(sql); saved = params; return { lastInsertRowid: 1 }; }
|
||||
};
|
||||
const ai = async (messages, settings) => {
|
||||
calls.ai.push({ messages, settings });
|
||||
return { content: 'Complete supported answer. [1]', finishReason: 'stop', model: 'synthetic' };
|
||||
};
|
||||
const source = { number: 1, title: 'Synthetic source', excerpt: 'Synthetic reference.', page: 9 };
|
||||
const mocks = {
|
||||
express, axios: { async post(url, payload) { calls.images.push(payload); return { data: { data: [{ b64_json: 'c3ludGhldGlj' }] } }; } }, crypto: require('node:crypto'), '../db/database': db,
|
||||
'../middleware/auth': { authMiddleware() {} }, '../utils/ai': options.ai || { callAI: ai, callAIStream: ai },
|
||||
'../utils/errors': { gatewayUrl: path => 'http://synthetic.invalid' + path }, '../utils/litellm': { getLiteLLMHeaders: () => ({}) }, '../utils/logger': { audit() {}, error() {} },
|
||||
'../utils/crypto': { encryptString: value => 'encrypted:' + value, decryptString: value => value.replace(/^encrypted:/, '') },
|
||||
'../utils/redis': { async getJson() { return null; }, async setJson() {} }, '../utils/clinicalPromptPool': { createClinicalPromptPool: () => ({}) },
|
||||
'../utils/generatedImages': require('../src/utils/generatedImages'),
|
||||
'../utils/generatedImageLinks': require('../src/utils/generatedImageLinks'),
|
||||
'../utils/imageTool': require('../src/utils/imageTool'),
|
||||
'../utils/clinicalPrompts': require('../src/utils/clinicalPrompts'),
|
||||
'../utils/clinicalMcpClient': {
|
||||
async semanticSearch(query) { calls.search.push(query); return {}; }, async getMcpHealth() { calls.health.push('health'); return {}; }
|
||||
},
|
||||
'../utils/clinicalRetrieval': {
|
||||
cleanSourceExcerpt: require('../src/utils/clinicalRetrieval').cleanSourceExcerpt,
|
||||
normalizeMcpSearchResponse: () => options.noSources ? [] : [source],
|
||||
normalizeMcpMultimodalResponse: () => [], dedupeSources: value => value,
|
||||
isVisualSourceQuery: () => false, classifyAndRerankMultimodalResults: async () => []
|
||||
},
|
||||
'../utils/clinicalAnswer': answer, '../utils/clinicalConversation': policy
|
||||
};
|
||||
const module = { exports: {} };
|
||||
vm.runInNewContext(read('src/routes/clinicalAssistant.js'), {
|
||||
module, exports: module.exports, console: quiet, Buffer, Map,
|
||||
process: { env: { CLINICAL_ASSISTANT_MCP_WARMUP: 'false', CLINICAL_ASSISTANT_CONVERSATION_CHARS: options.limit, LITELLM_API_BASE: 'http://synthetic.invalid' } }, setTimeout() {},
|
||||
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; }
|
||||
});
|
||||
async function request(method, endpoint, body, userId = 7) {
|
||||
const handler = module.exports.stack.find(layer => layer.route && layer.route.path === endpoint && layer.route.methods[method]).route.stack.find(layer => layer.method === method).handle;
|
||||
const res = {
|
||||
statusCode: 200, headers: {}, events: '', body: null,
|
||||
status(code) { this.statusCode = code; return this; }, json(body) { this.body = body; return this; },
|
||||
setHeader(k, v) { this.headers[k] = v; }, write(text) { this.events += text; }, end() {}, flushHeaders() {}
|
||||
};
|
||||
await handler({ body, params: { id: 1 }, user: { id: userId } }, res);
|
||||
return res;
|
||||
}
|
||||
return { calls, request };
|
||||
}
|
||||
|
||||
test('attachment policy validates MIME, canonical base64, per-image/count/total limits without provider contact', async () => {
|
||||
const oversize = { dataBase64: canonical(5 * 1024 * 1024 + 1), mimeType: 'image/png' };
|
||||
const big = { dataBase64: canonical(4 * 1024 * 1024), mimeType: 'image/jpeg' };
|
||||
const invalid = [
|
||||
{ images: [{ ...png, mimeType: 'image/svg+xml' }] },
|
||||
{ images: [{ ...png, mimeType: 'image/gif' }] },
|
||||
{ images: [{ ...png, mimeType: 'application/octet-stream' }] },
|
||||
{ images: [{ ...png, dataBase64: 'not base64!!!' }] },
|
||||
{ images: [{ ...png, dataBase64: 'SGVsbG8' }] },
|
||||
{ images: [{ ...png, dataBase64: 'AB==' }] }, // non-canonical trailing bits
|
||||
{ images: [{ ...png, dataBase64: '' }] },
|
||||
{ images: [{ ...png, dataBase64: 42 }] },
|
||||
{ images: [{ mimeType: 'image/png' }] },
|
||||
{ images: [{ dataBase64: png.dataBase64 }] },
|
||||
{ images: [null] }, { images: ['png'] }, { images: [{}] },
|
||||
{ images: 'not-a-list' },
|
||||
{ images: [png, jpeg, webp, png, jpeg] },
|
||||
{ images: [oversize] },
|
||||
{ images: [big, big, big] }
|
||||
];
|
||||
for (const body of invalid) {
|
||||
for (const endpoint of ['/clinical-assistant/chat', '/clinical-assistant/chat/stream']) {
|
||||
const app = server();
|
||||
const result = await app.request('post', endpoint, { message: 'What is the recommended dose of acetaminophen for a febrile three year old?', history: [], ...body });
|
||||
assert.equal(result.statusCode, 400, JSON.stringify(body.images) + ' on ' + endpoint);
|
||||
assert.equal(result.body.code, 'INVALID_ATTACHMENTS');
|
||||
assert.equal(app.calls.ai.length + app.calls.search.length, 0, 'no provider or retrieval call on ' + endpoint);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('valid images are normalized, ride the outgoing question only, and are excluded from the UTF-16 text budget', async () => {
|
||||
const app = server({ limit: '1000' });
|
||||
const images = [{ ...png, extra: 'ignored' }, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }];
|
||||
const question = 'What about monitoring?';
|
||||
const result = await app.request('post', '/clinical-assistant/chat', { message: question, history: [{ role: 'user', content: 'x'.repeat(500) }], images });
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(app.calls.search.length, 1);
|
||||
assert.equal(app.calls.ai.length, 2, 'rewrite plus generation');
|
||||
const generation = app.calls.ai.at(-1);
|
||||
assert.deepEqual(generation.settings.images, [png, jpeg, webp, { dataBase64: canonical(64), mimeType: 'image/webp' }], 'normalized, extra fields dropped');
|
||||
assert.equal(typeof generation.messages[1].content, 'string', 'the route keeps prompt text; ai.js builds multimodal parts');
|
||||
assert.ok(generation.messages[1].content.includes(question));
|
||||
assert.equal(app.calls.ai[0].settings.images, undefined, 'the retrieval rewrite is text-only');
|
||||
|
||||
const atLimit = server({ limit: '1000' });
|
||||
const budgetBody = { message: 'x', history: [{ role: 'user', content: 'x'.repeat(999) }], images: [png, jpeg, webp, png] };
|
||||
assert.equal((await atLimit.request('post', '/clinical-assistant/chat', budgetBody)).statusCode, 200, 'images are excluded from the UTF-16 budget');
|
||||
assert.equal((await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' })).statusCode, 413, 'the text boundary is unchanged by images');
|
||||
const over = await atLimit.request('post', '/clinical-assistant/chat', { ...budgetBody, message: 'xx' });
|
||||
assert.equal(over.body.budget.unit, 'characters');
|
||||
});
|
||||
|
||||
test('greeting and retrieval-empty direct responses validate attachments before responding', async () => {
|
||||
const bad = server();
|
||||
const rejected = await bad.request('post', '/clinical-assistant/chat/stream', { message: 'thanks', history: [], images: [{ dataBase64: canonical(8), mimeType: 'image/bmp' }] });
|
||||
assert.equal(rejected.statusCode, 400);
|
||||
assert.equal(bad.calls.ai.length + bad.calls.search.length, 0);
|
||||
const good = server();
|
||||
const direct = await good.request('post', '/clinical-assistant/chat', { message: 'hi', history: [], images: [png] });
|
||||
assert.equal(direct.statusCode, 200);
|
||||
assert.match(direct.body.answer, /clinical question/);
|
||||
assert.equal(good.calls.ai.length + good.calls.search.length, 0, 'greeting never reaches providers');
|
||||
});
|
||||
|
||||
test('handoff ignores image attachments and never forwards them', async () => {
|
||||
const app = server();
|
||||
const result = await app.request('post', '/clinical-assistant/handoff', { history: [{ role: 'user', content: 'Known facts.' }], images: [{ dataBase64: canonical(8), mimeType: 'image/png' }] });
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(app.calls.ai.length, 1);
|
||||
assert.ok(app.calls.ai[0].messages[1].content.includes('Known facts.'));
|
||||
assert.equal(app.calls.ai[0].settings.images, undefined);
|
||||
assert.doesNotMatch(JSON.stringify(app.calls.ai[0].messages), /base64/);
|
||||
});
|
||||
|
||||
function openaiAdapter() {
|
||||
const requests = [];
|
||||
class OpenAI {
|
||||
constructor() {
|
||||
this.chat = { completions: { create: async r => {
|
||||
requests.push(r);
|
||||
if (r.stream) return [{ choices: [{ delta: { content: 'streamed answer' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }];
|
||||
return { choices: [{ message: { content: 'Synthetic multimodal answer.' }, finish_reason: 'stop' }], usage: null };
|
||||
} } };
|
||||
}
|
||||
}
|
||||
const mocks = {
|
||||
openai: { OpenAI },
|
||||
'./models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['synthetic-vision']); } },
|
||||
'./generationOptions': require('../src/utils/generationOptions'),
|
||||
'./logger': { apiCall() {}, error() {} },
|
||||
'../db/database': { async getSetting() { return null; } }
|
||||
};
|
||||
const module = { exports: {} };
|
||||
vm.runInNewContext(read('src/utils/ai.js'), {
|
||||
module, exports: module.exports, console: quiet, TextDecoder,
|
||||
process: { env: { LITELLM_API_BASE: 'http://synthetic.invalid', LITELLM_API_KEY: 'sk-synthetic' } },
|
||||
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
|
||||
});
|
||||
return { requests, ai: module.exports };
|
||||
}
|
||||
|
||||
test('LiteLLM generation builds multimodal parts only on the latest user message and keeps tools/temperature/streaming', async () => {
|
||||
const tools = [{ type: 'function', function: { name: 'generate_image', parameters: { type: 'object' } } }];
|
||||
const { requests, ai } = openaiAdapter();
|
||||
const messages = [
|
||||
{ role: 'system', content: 'System behavior.' },
|
||||
{ role: 'user', content: 'Earlier turn.' },
|
||||
{ role: 'assistant', content: 'Earlier answer.' },
|
||||
{ role: 'user', content: 'Latest question.' }
|
||||
];
|
||||
const result = await ai.callAI(messages, { model: 'synthetic-vision', images: [png, jpeg], tools, temperature: 0.15, maxTokens: 2600 });
|
||||
assert.equal(result.content, 'Synthetic multimodal answer.');
|
||||
const request = requests[0];
|
||||
assert.equal(request.model, 'synthetic-vision');
|
||||
assert.equal(request.temperature, 0.15);
|
||||
assert.equal(request.max_tokens, 2600);
|
||||
assert.deepEqual(request.tools, tools);
|
||||
assert.equal(request.tool_choice, 'auto');
|
||||
assert.equal(request.parallel_tool_calls, false);
|
||||
assert.deepEqual(request.messages[0], { role: 'system', content: 'System behavior.' });
|
||||
assert.deepEqual(request.messages[1], { role: 'user', content: 'Earlier turn.' });
|
||||
assert.deepEqual(request.messages[2], { role: 'assistant', content: 'Earlier answer.' });
|
||||
assert.equal(request.messages[3].role, 'user');
|
||||
assert.equal(request.messages[3].content[0].type, 'text');
|
||||
assert.equal(request.messages[3].content[0].text, 'Latest question.');
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[1])), { type: 'image_url', image_url: { url: 'data:image/png;base64,' + png.dataBase64 } });
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(request.messages[3].content[2])), { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,' + jpeg.dataBase64 } });
|
||||
|
||||
const streamed = await ai.callAIStream(messages, { model: 'synthetic-vision', images: [png], tools, temperature: 0.15, maxTokens: 2600 }, () => {});
|
||||
assert.equal(streamed.content, 'streamed answer');
|
||||
const streamRequest = requests[1];
|
||||
assert.equal(streamRequest.stream, true);
|
||||
assert.deepEqual(streamRequest.tools, tools);
|
||||
assert.equal(streamRequest.messages[3].content.length, 2);
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(streamRequest.messages[3].content[1].image_url)), { url: 'data:image/png;base64,' + png.dataBase64 });
|
||||
|
||||
const noImages = await ai.callAI(messages, { model: 'synthetic-vision' });
|
||||
assert.equal(noImages.content, 'Synthetic multimodal answer.');
|
||||
assert.deepEqual(requests[2].messages[3], { role: 'user', content: 'Latest question.' });
|
||||
});
|
||||
|
||||
test('legacy direct provider adapters refuse image attachments with 400 before contacting the provider', async () => {
|
||||
const sdkCalls = [];
|
||||
class InvokeModelCommand { constructor(input) { this.input = input; } }
|
||||
class ConverseCommand { constructor(input) { this.input = input; } }
|
||||
const mocks = {
|
||||
openai: { OpenAI: class { constructor() { throw Error('Unexpected OpenAI client'); } } },
|
||||
'./models': { ...require('../src/utils/models'), async getAllowedModelIds() { return new Set(['anthropic.claude-synthetic', 'amazon/nova-lite', 'google/gemini-2.5-flash']); } },
|
||||
'./generationOptions': require('../src/utils/generationOptions'),
|
||||
'./logger': { apiCall() {}, error() {} },
|
||||
'../db/database': { async getSetting() { return null; } },
|
||||
'@aws-sdk/client-bedrock-runtime': {
|
||||
InvokeModelCommand, ConverseCommand,
|
||||
BedrockRuntimeClient: class { async send(command) { sdkCalls.push(command); return { body: Buffer.from('{}') }; } }
|
||||
},
|
||||
'@google-cloud/vertexai': {
|
||||
VertexAI: class { getGenerativeModel() { return { async generateContent(request) { sdkCalls.push(request); return { response: { candidates: [] } }; } }; } }
|
||||
}
|
||||
};
|
||||
const module = { exports: {} };
|
||||
vm.runInNewContext(read('src/utils/ai.js'), {
|
||||
module, exports: module.exports, console: quiet, TextDecoder,
|
||||
process: { env: { AI_PROVIDER: 'bedrock', AWS_BEDROCK_REGION: 'synthetic' } },
|
||||
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
|
||||
});
|
||||
const messages = [{ role: 'user', content: 'Question with image.' }];
|
||||
await assert.rejects(module.exports.callAI(messages, { model: 'anthropic.claude-synthetic', images: [png] }), error => {
|
||||
assert.equal(error.statusCode, 400);
|
||||
assert.equal(error.code, 'IMAGES_UNSUPPORTED_PROVIDER');
|
||||
assert.match(error.message, /OpenAI-compatible provider/);
|
||||
return true;
|
||||
});
|
||||
await assert.rejects(module.exports.callAIStream(messages, { model: 'anthropic.claude-synthetic', images: [png] }, () => {}), error => error.statusCode === 400);
|
||||
assert.equal(sdkCalls.length, 0, 'refused before any provider contact');
|
||||
|
||||
const vertexModule = { exports: {} };
|
||||
vm.runInNewContext(read('src/utils/ai.js'), {
|
||||
module: vertexModule, exports: vertexModule.exports, console: quiet, TextDecoder,
|
||||
process: { env: { AI_PROVIDER: 'vertex', GOOGLE_VERTEX_PROJECT: 'synthetic' } },
|
||||
require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected adapter import: ' + name); return mocks[name]; }
|
||||
});
|
||||
await assert.rejects(vertexModule.exports.callAI(messages, { model: 'google/gemini-2.5-flash', images: [png] }), error => error.statusCode === 400);
|
||||
assert.equal(sdkCalls.length, 0);
|
||||
});
|
||||
|
||||
function browserUI(options = {}) {
|
||||
const dom = new JSDOM('<div id="assistant-tab">' + read('public/components/assistant.html') + '</div>', {
|
||||
url: 'https://example.test/', runScripts: 'outside-only'
|
||||
});
|
||||
const calls = { stream: [], handoff: [], save: [] };
|
||||
const toasts = [];
|
||||
const escapeHtml = text => String(text).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
const context = {
|
||||
window: dom.window, document: dom.window.document, navigator: dom.window.navigator,
|
||||
console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require('node:crypto').webcrypto,
|
||||
FileReader: dom.window.FileReader, File: dom.window.File,
|
||||
setTimeout() {}, showToast(text, kind) { toasts.push([String(text), kind || '']); }, escapeHtml, escapeAttr: escapeHtml,
|
||||
renderAssistantMarkdown: text => escapeHtml(text), renderSourcesList: () => '', EMPTY_PROMPT_SETS: [[]],
|
||||
createAssistantExporter: () => ({ invalidate() {}, exportAnswerPdf() {} }),
|
||||
createAssistantImageStore: () => ({ renderGeneratedImage: src => '<img src="' + src + '">', clear() {} }),
|
||||
isImageRequest: () => false,
|
||||
fetchAssistantStatus: async () => ({ success: true, conversationChars: options.limit || 120000 }),
|
||||
fetchAssistantExamples: async () => ({}), fetchSavedAssistantChats: async () => ({ success: true, chats: [] }),
|
||||
openAssistantStream: async payload => {
|
||||
calls.stream.push(payload);
|
||||
if (options.stream) return options.stream(payload);
|
||||
return new Response('event: done\ndata: ' + JSON.stringify({ success: true, answer: 'Complete response.', sources: [] }) + '\n\n');
|
||||
},
|
||||
fetchAssistantChat: async payload => ({ success: true, answer: 'Fallback answer.' }),
|
||||
saveAssistantChat: async payload => { calls.save.push(payload); return { success: true }; },
|
||||
requestAssistantHandoff: async history => { calls.handoff.push(history); return { success: true, summary: 'Explicit handoff.' }; }
|
||||
};
|
||||
vm.createContext(context);
|
||||
vm.runInContext(read('public/js/clinicalAssistant.js').replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, ''), context);
|
||||
context.bindEvents();
|
||||
context.conversationChars = options.limit || 120000;
|
||||
return { context, document: dom.window.document, calls, toasts, dom };
|
||||
}
|
||||
|
||||
async function realApiHelpers() {
|
||||
const module = await import(pathToFileURL(path.join(root, 'public/js/assistant/api.js')).href);
|
||||
return module;
|
||||
}
|
||||
|
||||
function attachFile(ui, name, type, bytes) {
|
||||
const input = ui.document.getElementById('assistant-attach-input');
|
||||
const file = new ui.dom.window.File([bytes], name, { type });
|
||||
Object.defineProperty(input, 'files', { value: [file], configurable: true });
|
||||
input.dispatchEvent(new ui.dom.window.Event('change'));
|
||||
}
|
||||
|
||||
async function tick() { for (let i = 0; i < 25; i++) await new Promise(resolve => setImmediate(resolve)); }
|
||||
|
||||
test('UI attaches only PNG/JPEG/WebP with thumbnails, removal and client-side limits', async () => {
|
||||
const api = await realApiHelpers();
|
||||
const ui = browserUI();
|
||||
ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
|
||||
ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
|
||||
const bytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]);
|
||||
attachFile(ui, 'photo.png', 'image/png', bytes);
|
||||
attachFile(ui, 'scan.jpeg', 'image/jpeg', bytes);
|
||||
await tick();
|
||||
const wrap = ui.document.getElementById('assistant-attachments');
|
||||
assert.equal(wrap.hidden, false);
|
||||
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2);
|
||||
assert.ok(wrap.querySelector('img').getAttribute('src').startsWith('data:image/png;base64,'));
|
||||
|
||||
attachFile(ui, 'notes.txt', 'text/plain', new Uint8Array([1, 2, 3]));
|
||||
await tick();
|
||||
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 2);
|
||||
assert.ok(ui.toasts.some(([text]) => /PNG, JPEG and WebP/.test(text)));
|
||||
|
||||
attachFile(ui, 'third.webp', 'image/webp', bytes);
|
||||
attachFile(ui, 'fourth.png', 'image/png', bytes);
|
||||
await tick();
|
||||
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
|
||||
attachFile(ui, 'fifth.png', 'image/png', bytes);
|
||||
await tick();
|
||||
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
|
||||
assert.ok(ui.toasts.some(([text]) => /maximum of 4/.test(text)));
|
||||
|
||||
attachFile(ui, 'huge.png', 'image/png', new Uint8Array(5 * 1024 * 1024 + 1));
|
||||
await tick();
|
||||
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 4);
|
||||
assert.ok(ui.toasts.some(([text]) => /5 MiB/.test(text)));
|
||||
|
||||
wrap.querySelector('[data-assistant-remove-attachment="0"]').click();
|
||||
await tick();
|
||||
assert.equal(wrap.querySelectorAll('.assistant-attachment').length, 3);
|
||||
assert.equal(ui.context.attachments.length, 3);
|
||||
ui.dom.window.close();
|
||||
});
|
||||
|
||||
test('UI sends images only with the question, clears them on success, and keeps them out of handoff, saves and the budget label', async () => {
|
||||
const api = await realApiHelpers();
|
||||
const ui = browserUI({ limit: 1000 });
|
||||
ui.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
|
||||
ui.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
|
||||
const bytes = new Uint8Array([1, 2, 3, 4]);
|
||||
attachFile(ui, 'photo.png', 'image/png', bytes);
|
||||
await tick();
|
||||
const expected = [{ dataBase64: Buffer.from(bytes).toString('base64'), mimeType: 'image/png' }];
|
||||
|
||||
ui.context.restoreSavedChat({ messages: [{ role: 'user', content: 'x'.repeat(999) }], lastAnswer: '' });
|
||||
const input = ui.document.getElementById('assistant-input');
|
||||
input.value = 'x';
|
||||
input.dispatchEvent(new ui.dom.window.Event('input'));
|
||||
const label = ui.document.getElementById('assistant-context-budget');
|
||||
assert.equal(label.textContent, '1,000 / 1,000 conversation characters (UTF-16 code units).', 'budget label stays text-only');
|
||||
assert.doesNotMatch(label.textContent, /image/i);
|
||||
|
||||
// Handoff and saved chats never carry attachments.
|
||||
await ui.context.requestHandoff();
|
||||
assert.equal(ui.calls.handoff.length, 1);
|
||||
assert.equal(ui.calls.handoff[0].images, undefined);
|
||||
assert.equal(JSON.stringify(ui.calls.handoff[0]).includes('dataBase64'), false);
|
||||
await ui.context.saveCurrentChat();
|
||||
assert.equal(ui.calls.save[0].images, undefined);
|
||||
assert.equal(JSON.stringify(ui.calls.save[0]).includes('dataBase64'), false);
|
||||
|
||||
await ui.context.onAsk();
|
||||
await tick();
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(ui.calls.stream[0].images)), expected);
|
||||
assert.equal(ui.context.attachments.length, 0, 'cleared on successful send');
|
||||
assert.equal(ui.document.getElementById('assistant-attachments').hidden, true);
|
||||
assert.equal(ui.context.messages.length, 3, 'question and answer appended');
|
||||
|
||||
// New chat clears any later attachments.
|
||||
attachFile(ui, 'again.png', 'image/png', bytes);
|
||||
await tick();
|
||||
assert.equal(ui.context.attachments.length, 1);
|
||||
await ui.context.clearConversation();
|
||||
assert.equal(ui.context.attachments.length, 0);
|
||||
assert.equal(ui.document.getElementById('assistant-attachments').hidden, true);
|
||||
|
||||
attachFile(ui, 'kept.png', 'image/png', bytes);
|
||||
await tick();
|
||||
assert.equal(ui.context.attachments.length, 1);
|
||||
const failing = browserUI({ stream: () => new Response(JSON.stringify({ error: 'Image rejected' }), { status: 400 }) });
|
||||
failing.context.assistantAttachmentLimits = api.assistantAttachmentLimits;
|
||||
failing.context.assistantAttachmentPayload = api.assistantAttachmentPayload;
|
||||
attachFile(failing, 'kept.png', 'image/png', bytes);
|
||||
await tick();
|
||||
failing.document.getElementById('assistant-input').value = 'Question with attachment';
|
||||
await failing.context.onAsk();
|
||||
await tick();
|
||||
assert.equal(failing.context.attachments.length, 1, 'rejected send keeps the attachments for correction');
|
||||
assert.equal(failing.document.getElementById('assistant-attachments').hidden, false);
|
||||
ui.dom.window.close();
|
||||
failing.dom.window.close();
|
||||
});
|
||||
Loading…
Reference in a new issue