danvics/pediatric-ai-scribe-v3
1
0
Fork 0
Code Issues Pull requests Projects Releases 14 Packages Wiki Activity Actions

feat: use visual captions in assistant sources

Browse source
This commit is contained in:
Daniel 2026-05-07 04:41:49 +02:00
parent dd29ba7e98
commit 628e6aff7f
2 changed files with 38 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
public/js/clinicalAssistant.js
View file

@ -249,6 +249,7 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection }
var meta = [];
if (page) meta.push('page ' + page);
if (s.source_type === 'multimodal_page') meta.push('visual PDF page');
if (s.visual_caption_source) meta.push('caption: ' + s.visual_caption_source);
if (s.visual_kind || s.source_priority) meta.push(s.visual_kind || s.source_priority);
if (s.category) meta.push(s.category);
if (s.doc_type || s.type) meta.push(s.doc_type || s.type);
@ -267,6 +268,7 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection }
var badges = [];
if (source.source_priority) badges.push(source.source_priority);
if (source.visual_kind && badges.indexOf(source.visual_kind) === -1) badges.push(source.visual_kind);
if (source.visual_caption_source && badges.indexOf(source.visual_caption_source) === -1) badges.push(source.visual_caption_source);
if (source.category) badges.push(source.category);
if (source.source_boost && Number(source.source_boost) !== 1) badges.push(Number(source.source_boost).toFixed(2) + 'x');
if (!badges.length) return '';

53
src/routes/clinicalAssistant.js
View file

@ -686,6 +686,12 @@ function normalizeMcpMultimodalResponse(result) {
if (!data || !Array.isArray(data.results)) return [];
return data.results.map(function(r, idx) {
var nearby = clip(cleanSourceExcerpt(r.nearby_text || ''), 1400);
var caption = clip(cleanSourceExcerpt(r.visual_caption || ''), 900);
var labels = Array.isArray(r.visual_labels) ? r.visual_labels.filter(Boolean).slice(0, 20) : [];
var excerptParts = [];
if (caption) excerptParts.push('[Visual caption] ' + caption);
if (labels.length) excerptParts.push('[Visual labels] ' + labels.join(', '));
if (nearby) excerptParts.push('[Page text] ' + nearby);
return {
number: idx + 1,
id: r.id,
@ -698,7 +704,10 @@ function normalizeMcpMultimodalResponse(result) {
category_path: r.category_path || '',
page: r.page_number || r.pageNumber || null,
page_count: r.page_count || r.pageCount || null,
excerpt: nearby ? '[Page-image match] ' + nearby : '[Page-image match] Rendered PDF page matched the visual/text query.',
visual_caption: caption,
visual_labels: labels,
visual_caption_source: r.visual_caption_source || '',
excerpt: excerptParts.length ? '[Page-image match] ' + excerptParts.join('\n') : '[Page-image match] Rendered PDF page matched the visual/text query.',
score: r.score,
image_path_internal: r.image_path || '',
image_url: multimodalImageUrl(r.image_path),
@ -893,7 +902,7 @@ function visualClassifierLabels() {
}
function visualMetadataScore(query, source) {
var haystack = [source.title, source.excerpt, source.file_path, source.category, source.subcategory, source.category_path].filter(Boolean).join(' ');
var haystack = [source.title, source.excerpt, source.visual_caption, (source.visual_labels || []).join(' '), source.file_path, source.category, source.subcategory, source.category_path].filter(Boolean).join(' ');
var score = 0;
var intent = visualIntent(query);
if (isRadiologyQuery(query) && /\b(radiology|radiograph|x-?ray|cxr|imaging|ct|mri|ultrasound|film)\b/i.test(haystack)) score += 0.12;
@ -1139,28 +1148,38 @@ async function generateImage(prompt, model) {
if (!process.env.LITELLM_API_BASE) throw new Error('LiteLLM is required for image generation');
var headers = { 'Content-Type': 'application/json' };
if (process.env.LITELLM_API_KEY) headers.Authorization = 'Bearer ' + process.env.LITELLM_API_KEY;
var size = imageSizeForPrompt(prompt);
var resp = await axios.post(gatewayUrl('/images/generations'), {
model: model,
prompt: imagePromptForCanvas(prompt, size),
size: size
}, { headers: headers, timeout: 120000 });
var size = process.env.CLINICAL_ASSISTANT_IMAGE_SIZE || 'auto';
var resp = await generateImageRequest(model, imagePromptForCanvas(prompt), size, headers).catch(async function(e) {
if (!isInvalidImageSizeError(e) || size === '1024x1024') throw e;
return generateImageRequest(model, imagePromptForCanvas(prompt), '1024x1024', headers);
});
var item = resp.data && resp.data.data && resp.data.data[0] ? resp.data.data[0] : {};
return { imageUrl: item.url || null, base64: item.b64_json || null, raw: (!item.url && !item.b64_json) ? resp.data : undefined };
}
function imageSizeForPrompt(prompt) {
var text = String(prompt || '');
if (/\b(flow\s*chart|flowchart|algorithm|pathway|timeline|vertical|stepwise|decision\s*tree|age\s*group|0-21|22-28|29-60)\b/i.test(text)) return '1024x1536';
if (/\b(table|matrix|comparison|wide|landscape|side-by-side)\b/i.test(text)) return '1536x1024';
return '1024x1024';
function generateImageRequest(model, prompt, size, headers) {
return axios.post(gatewayUrl('/images/generations'), {
model: model,
prompt: prompt,
size: size
}, { headers: headers, timeout: 120000 });
}
function imagePromptForCanvas(prompt, size) {
function imagePromptForCanvas(prompt) {
var text = String(prompt || '');
var guidance = ' Keep all text and boxes fully inside the canvas with generous margins. Use fewer words per box, large readable type, and avoid cropping at edges.';
if (size === '1024x1536') guidance += ' Use a vertical portrait layout with top-to-bottom flow and ample spacing between decision nodes.';
if (size === '1536x1024') guidance += ' Use a wide landscape layout with columns and ample horizontal spacing.';
return String(prompt || '').trim() + guidance;
if (/\b(flow\s*chart|flowchart|algorithm|pathway|timeline|vertical|stepwise|decision\s*tree|age\s*group|0-21|22-28|29-60)\b/i.test(text)) {
guidance += ' Prefer a tall portrait canvas with top-to-bottom flow and ample spacing between decision nodes.';
}
if (/\b(table|matrix|comparison|wide|landscape|side-by-side)\b/i.test(text)) {
guidance += ' Prefer a wide landscape canvas with columns and ample horizontal spacing.';
}
return text.trim() + guidance;
}
function isInvalidImageSizeError(e) {
var detail = e && e.response && e.response.data ? JSON.stringify(e.response.data) : (e && e.message ? e.message : '');
return /invalid size|unsupported size|supported sizes/i.test(detail);
}
async function getSetting(key, fallback) {