danvics/pediatric-ai-scribe-v3
1
0
Fork 0
Code Issues Pull requests Projects Releases 14 Packages Wiki Activity Actions

feat: map assistant visual source intent

Browse source
This commit is contained in:
Daniel 2026-05-07 03:01:37 +02:00
parent c0f83c704e
commit 34a53e35df
3 changed files with 52 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
public/components/assistant.html
View file

@ -165,6 +165,8 @@
.assistant-save-actions { display:flex; gap:6px; flex-wrap:wrap; }
.assistant-source { border:1px solid var(--g200); border-radius:10px; padding:9px; background:white; font-size:12px; line-height:1.5; }
.assistant-source strong { color:var(--g800); }
.assistant-source-badges { display:flex; gap:5px; flex-wrap:wrap; margin-top:6px; }
.assistant-source-badges span { border:1px solid var(--g200); border-radius:999px; background:var(--g50); color:var(--g600); padding:2px 7px; font-size:10px; font-weight:700; text-transform:uppercase; letter-spacing:.03em; }
.assistant-source-meta { color:var(--g500); font-size:11px; margin-top:3px; }
.assistant-source-preview { margin-top:8px; }
.assistant-source-preview button { border:0; padding:0; background:transparent; cursor:pointer; width:100%; display:block; }

15
public/js/clinicalAssistant.js
View file

@ -249,10 +249,13 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection }
var meta = [];
if (page) meta.push('page ' + page);
if (s.source_type === 'multimodal_page') meta.push('visual PDF page');
if (s.visual_kind || s.source_priority) meta.push(s.visual_kind || s.source_priority);
if (s.category) meta.push(s.category);
if (s.doc_type || s.type) meta.push(s.doc_type || s.type);
if (s.score != null) meta.push('score ' + Number(s.score).toFixed(3));
return '<div class="assistant-source" id="assistant-source-' + n + '">' +
'<strong>[' + n + '] ' + escapeHtml(s.title || s.resource || 'Untitled source') + '</strong>' +
renderSourceBadges(s) +
'<div class="assistant-source-meta">' + escapeHtml(meta.join(' · ') || 'indexed source') + '</div>' +
renderSourceImage(s, n) +
(s.excerpt ? '<div class="assistant-source-excerpt"><p>' + escapeHtml(cleanSourceExcerpt(s.excerpt).slice(0, 900)) + '</p></div>' : '') +
@ -260,6 +263,18 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection }
}).join('');
}
function renderSourceBadges(source) {
var badges = [];
if (source.source_priority) badges.push(source.source_priority);
if (source.visual_kind && badges.indexOf(source.visual_kind) === -1) badges.push(source.visual_kind);
if (source.category) badges.push(source.category);
if (source.source_boost && Number(source.source_boost) !== 1) badges.push(Number(source.source_boost).toFixed(2) + 'x');
if (!badges.length) return '';
return '<div class="assistant-source-badges">' + badges.slice(0, 4).map(function (badge) {
return '<span>' + escapeHtml(badge) + '</span>';
}).join('') + '</div>';
}
function renderSourceImage(source, number) {
var src = source && (source.image_url || source.imageUrl);
if (!src || source.source_type !== 'multimodal_page') return '';

43
src/routes/clinicalAssistant.js
View file

@ -639,6 +639,11 @@ function normalizeMcpSearchResponse(result) {
title: cleanTitle(r.title || 'Untitled resource'),
page: r.page_number || r.pageNumber || null,
page_count: r.page_count || r.pageCount || null,
folder_path: r.folder_path || r.folderPath || '',
category: r.category || '',
source_priority: r.source_priority || r.sourcePriority || '',
source_boost: r.source_boost || r.sourceBoost || null,
tags: Array.isArray(r.tags) ? r.tags : [],
excerpt: clip(text, 1800),
score: r.score,
chunk_index: r.chunk_index,
@ -761,16 +766,19 @@ function parseCitationCluster(cluster) {
}
function isVisualSourceQuery(query) {
return /\b(visual|image|figure|photo|picture|diagram|graph|chart|radiology|radiologic|radiograph|x-?ray|cxr|imaging|ultrasound|ct|mri|scan|film)\b/i.test(String(query || ''));
return visualIntent(query).wanted.length > 0;
}
function isRadiologyQuery(query) {
return /\b(radiology|radiologic|radiograph|x-?ray|cxr|chest\s*(film|x-?ray|radiograph)|imaging|ultrasound|ct|mri|scan|film)\b/i.test(String(query || ''));
return visualIntent(query).wanted.indexOf('radiology') !== -1;
}
function buildMultimodalSearchQuery(query) {
query = String(query || '').trim();
if (isRadiologyQuery(query)) return query + ' radiograph x-ray chest imaging radiology';
var intent = visualIntent(query);
if (intent.wanted.indexOf('radiology') !== -1) return query + ' radiograph x-ray chest imaging radiology';
if (intent.wanted.indexOf('diagram') !== -1) return query + ' figure diagram pathway flowchart algorithm illustration';
if (intent.wanted.indexOf('photo') !== -1) return query + ' clinical photograph image skin lesion pathology microscopy';
return query;
}
@ -779,7 +787,7 @@ async function classifyAndRerankMultimodalResults(query, results) {
if (!results.length) return [];
var candidates = results.slice(0, VISUAL_CLASSIFIER_LIMIT);
if (!VISUAL_CLASSIFIER_ENABLED) return selectMultimodalResults(query, candidates);
var radiology = isRadiologyQuery(query);
var intent = visualIntent(query);
var classified = [];
for (var i = 0; i < candidates.length; i++) {
var candidate = candidates[i];
@ -790,10 +798,11 @@ async function classifyAndRerankMultimodalResults(query, results) {
if (!classification) continue;
var metaScore = visualMetadataScore(query, candidate);
var confident = classification.confidence >= VISUAL_CLASSIFIER_MIN_CONFIDENCE || metaScore > 0.1;
var accepted = radiology ? (classification.kind === 'radiology' && confident) : (classification.kind !== 'text_page' && confident);
var accepted = intent.wanted.indexOf(classification.kind) !== -1 && confident;
if (!accepted) continue;
candidate.visual_kind = classification.kind;
candidate.visual_confidence = classification.confidence;
candidate.source_priority = classification.kind;
candidate.visual_score = (Number(candidate.score) || 0) + (classification.confidence * 0.35) + metaScore;
classified.push(candidate);
}
@ -813,6 +822,24 @@ function selectMultimodalResults(query, results) {
});
}
function visualIntent(query) {
var text = String(query || '').toLowerCase();
var wanted = [];
if (/\b(radiology|radiologic|radiograph|x-?ray|cxr|chest\s*(film|x-?ray|radiograph)|imaging|ultrasound|ct|mri|scan|film)\b/i.test(text)) {
wanted.push('radiology');
}
if (/\b(algorithm|flow\s*chart|flowchart|pathway|diagram|graph|chart|figure|figures|illustration|infographic|schema|schematic|table)\b/i.test(text)) {
wanted.push('diagram');
}
if (/\b(photo|photograph|picture|skin|rash|lesion|wound|gross|microscopy|histology|pathology image|clinical image)\b/i.test(text)) {
wanted.push('photo');
}
if (!wanted.length && /\b(visual|image|images|show|find|display|preview)\b/i.test(text)) {
wanted.push('diagram', 'photo');
}
return { wanted: Array.from(new Set(wanted)) };
}
async function classifyVisualSource(source) {
var imagePath = resolveRawMultimodalImagePath(source.image_path_internal || '');
if (!imagePath) throw new Error('No local page image');
@ -850,7 +877,10 @@ function visualClassifierLabels() {
function visualMetadataScore(query, source) {
var haystack = [source.title, source.excerpt, source.file_path, source.category, source.subcategory, source.category_path].filter(Boolean).join(' ');
var score = 0;
var intent = visualIntent(query);
if (isRadiologyQuery(query) && /\b(radiology|radiograph|x-?ray|cxr|imaging|ct|mri|ultrasound|film)\b/i.test(haystack)) score += 0.12;
if (intent.wanted.indexOf('diagram') !== -1 && /\b(figure|diagram|algorithm|flowchart|pathway|chart|illustration|infographic)\b/i.test(haystack)) score += 0.12;
if (intent.wanted.indexOf('photo') !== -1 && /\b(photo|photograph|skin|rash|lesion|wound|gross|microscopy|histology)\b/i.test(haystack)) score += 0.12;
if (/\b(pneumonia|bronchiolitis|lung|airway|hyperinflation|infiltrate|consolidation|atelectasis)\b/i.test(haystack)) score += 0.06;
if (looksLikeTextOnlyPage(source)) score -= 0.18;
return score;
@ -908,9 +938,6 @@ function sanitizeSourcesForClient(sources) {
var out = Object.assign({}, source);
delete out.image_path_internal;
delete out.file_path;
delete out.category;
delete out.subcategory;
delete out.category_path;
return out;
});
}