From 628e6aff7f9d97faeda5eb03ab3859587dd111fa Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 7 May 2026 04:41:49 +0200 Subject: [PATCH] feat: use visual captions in assistant sources --- public/js/clinicalAssistant.js | 2 ++ src/routes/clinicalAssistant.js | 53 ++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js index 9de7eff..722a7db 100644 --- a/public/js/clinicalAssistant.js +++ b/public/js/clinicalAssistant.js @@ -249,6 +249,7 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection } var meta = []; if (page) meta.push('page ' + page); if (s.source_type === 'multimodal_page') meta.push('visual PDF page'); + if (s.visual_caption_source) meta.push('caption: ' + s.visual_caption_source); if (s.visual_kind || s.source_priority) meta.push(s.visual_kind || s.source_priority); if (s.category) meta.push(s.category); if (s.doc_type || s.type) meta.push(s.doc_type || s.type); @@ -267,6 +268,7 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection } var badges = []; if (source.source_priority) badges.push(source.source_priority); if (source.visual_kind && badges.indexOf(source.visual_kind) === -1) badges.push(source.visual_kind); + if (source.visual_caption_source && badges.indexOf(source.visual_caption_source) === -1) badges.push(source.visual_caption_source); if (source.category) badges.push(source.category); if (source.source_boost && Number(source.source_boost) !== 1) badges.push(Number(source.source_boost).toFixed(2) + 'x'); if (!badges.length) return ''; diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js index 316541f..0abc3e6 100644 --- a/src/routes/clinicalAssistant.js +++ b/src/routes/clinicalAssistant.js @@ -686,6 +686,12 @@ function normalizeMcpMultimodalResponse(result) { if (!data || !Array.isArray(data.results)) return []; return data.results.map(function(r, idx) { var nearby = clip(cleanSourceExcerpt(r.nearby_text || ''), 1400); + var caption = clip(cleanSourceExcerpt(r.visual_caption || ''), 900); + var labels = Array.isArray(r.visual_labels) ? r.visual_labels.filter(Boolean).slice(0, 20) : []; + var excerptParts = []; + if (caption) excerptParts.push('[Visual caption] ' + caption); + if (labels.length) excerptParts.push('[Visual labels] ' + labels.join(', ')); + if (nearby) excerptParts.push('[Page text] ' + nearby); return { number: idx + 1, id: r.id, @@ -698,7 +704,10 @@ function normalizeMcpMultimodalResponse(result) { category_path: r.category_path || '', page: r.page_number || r.pageNumber || null, page_count: r.page_count || r.pageCount || null, - excerpt: nearby ? '[Page-image match] ' + nearby : '[Page-image match] Rendered PDF page matched the visual/text query.', + visual_caption: caption, + visual_labels: labels, + visual_caption_source: r.visual_caption_source || '', + excerpt: excerptParts.length ? '[Page-image match] ' + excerptParts.join('\n') : '[Page-image match] Rendered PDF page matched the visual/text query.', score: r.score, image_path_internal: r.image_path || '', image_url: multimodalImageUrl(r.image_path), @@ -893,7 +902,7 @@ function visualClassifierLabels() { } function visualMetadataScore(query, source) { - var haystack = [source.title, source.excerpt, source.file_path, source.category, source.subcategory, source.category_path].filter(Boolean).join(' '); + var haystack = [source.title, source.excerpt, source.visual_caption, (source.visual_labels || []).join(' '), source.file_path, source.category, source.subcategory, source.category_path].filter(Boolean).join(' '); var score = 0; var intent = visualIntent(query); if (isRadiologyQuery(query) && /\b(radiology|radiograph|x-?ray|cxr|imaging|ct|mri|ultrasound|film)\b/i.test(haystack)) score += 0.12; @@ -1139,28 +1148,38 @@ async function generateImage(prompt, model) { if (!process.env.LITELLM_API_BASE) throw new Error('LiteLLM is required for image generation'); var headers = { 'Content-Type': 'application/json' }; if (process.env.LITELLM_API_KEY) headers.Authorization = 'Bearer ' + process.env.LITELLM_API_KEY; - var size = imageSizeForPrompt(prompt); - var resp = await axios.post(gatewayUrl('/images/generations'), { - model: model, - prompt: imagePromptForCanvas(prompt, size), - size: size - }, { headers: headers, timeout: 120000 }); + var size = process.env.CLINICAL_ASSISTANT_IMAGE_SIZE || 'auto'; + var resp = await generateImageRequest(model, imagePromptForCanvas(prompt), size, headers).catch(async function(e) { + if (!isInvalidImageSizeError(e) || size === '1024x1024') throw e; + return generateImageRequest(model, imagePromptForCanvas(prompt), '1024x1024', headers); + }); var item = resp.data && resp.data.data && resp.data.data[0] ? resp.data.data[0] : {}; return { imageUrl: item.url || null, base64: item.b64_json || null, raw: (!item.url && !item.b64_json) ? resp.data : undefined }; } -function imageSizeForPrompt(prompt) { - var text = String(prompt || ''); - if (/\b(flow\s*chart|flowchart|algorithm|pathway|timeline|vertical|stepwise|decision\s*tree|age\s*group|0-21|22-28|29-60)\b/i.test(text)) return '1024x1536'; - if (/\b(table|matrix|comparison|wide|landscape|side-by-side)\b/i.test(text)) return '1536x1024'; - return '1024x1024'; +function generateImageRequest(model, prompt, size, headers) { + return axios.post(gatewayUrl('/images/generations'), { + model: model, + prompt: prompt, + size: size + }, { headers: headers, timeout: 120000 }); } -function imagePromptForCanvas(prompt, size) { +function imagePromptForCanvas(prompt) { + var text = String(prompt || ''); var guidance = ' Keep all text and boxes fully inside the canvas with generous margins. Use fewer words per box, large readable type, and avoid cropping at edges.'; - if (size === '1024x1536') guidance += ' Use a vertical portrait layout with top-to-bottom flow and ample spacing between decision nodes.'; - if (size === '1536x1024') guidance += ' Use a wide landscape layout with columns and ample horizontal spacing.'; - return String(prompt || '').trim() + guidance; + if (/\b(flow\s*chart|flowchart|algorithm|pathway|timeline|vertical|stepwise|decision\s*tree|age\s*group|0-21|22-28|29-60)\b/i.test(text)) { + guidance += ' Prefer a tall portrait canvas with top-to-bottom flow and ample spacing between decision nodes.'; + } + if (/\b(table|matrix|comparison|wide|landscape|side-by-side)\b/i.test(text)) { + guidance += ' Prefer a wide landscape canvas with columns and ample horizontal spacing.'; + } + return text.trim() + guidance; +} + +function isInvalidImageSizeError(e) { + var detail = e && e.response && e.response.data ? JSON.stringify(e.response.data) : (e && e.message ? e.message : ''); + return /invalid size|unsupported size|supported sizes/i.test(detail); } async function getSetting(key, fallback) {