From 34a53e35dfaed5ec5bec3fbe7fb13e140cafeb04 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 7 May 2026 03:01:37 +0200 Subject: [PATCH] feat: map assistant visual source intent --- public/components/assistant.html | 2 ++ public/js/clinicalAssistant.js | 15 +++++++++++ src/routes/clinicalAssistant.js | 43 ++++++++++++++++++++++++++------ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/public/components/assistant.html b/public/components/assistant.html index d679cd6..66a3cee 100644 --- a/public/components/assistant.html +++ b/public/components/assistant.html @@ -165,6 +165,8 @@ .assistant-save-actions { display:flex; gap:6px; flex-wrap:wrap; } .assistant-source { border:1px solid var(--g200); border-radius:10px; padding:9px; background:white; font-size:12px; line-height:1.5; } .assistant-source strong { color:var(--g800); } +.assistant-source-badges { display:flex; gap:5px; flex-wrap:wrap; margin-top:6px; } +.assistant-source-badges span { border:1px solid var(--g200); border-radius:999px; background:var(--g50); color:var(--g600); padding:2px 7px; font-size:10px; font-weight:700; text-transform:uppercase; letter-spacing:.03em; } .assistant-source-meta { color:var(--g500); font-size:11px; margin-top:3px; } .assistant-source-preview { margin-top:8px; } .assistant-source-preview button { border:0; padding:0; background:transparent; cursor:pointer; width:100%; display:block; } diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js index 697e594..9de7eff 100644 --- a/public/js/clinicalAssistant.js +++ b/public/js/clinicalAssistant.js @@ -249,10 +249,13 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection } var meta = []; if (page) meta.push('page ' + page); if (s.source_type === 'multimodal_page') meta.push('visual PDF page'); + if (s.visual_kind || s.source_priority) meta.push(s.visual_kind || s.source_priority); + if (s.category) meta.push(s.category); if (s.doc_type || s.type) meta.push(s.doc_type || s.type); if (s.score != null) meta.push('score ' + Number(s.score).toFixed(3)); return '
' + '[' + n + '] ' + escapeHtml(s.title || s.resource || 'Untitled source') + '' + + renderSourceBadges(s) + '
' + escapeHtml(meta.join(' ยท ') || 'indexed source') + '
' + renderSourceImage(s, n) + (s.excerpt ? '

' + escapeHtml(cleanSourceExcerpt(s.excerpt).slice(0, 900)) + '

' : '') + @@ -260,6 +263,18 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown, stripSourcesSection } }).join(''); } + function renderSourceBadges(source) { + var badges = []; + if (source.source_priority) badges.push(source.source_priority); + if (source.visual_kind && badges.indexOf(source.visual_kind) === -1) badges.push(source.visual_kind); + if (source.category) badges.push(source.category); + if (source.source_boost && Number(source.source_boost) !== 1) badges.push(Number(source.source_boost).toFixed(2) + 'x'); + if (!badges.length) return ''; + return '
' + badges.slice(0, 4).map(function (badge) { + return '' + escapeHtml(badge) + ''; + }).join('') + '
'; + } + function renderSourceImage(source, number) { var src = source && (source.image_url || source.imageUrl); if (!src || source.source_type !== 'multimodal_page') return ''; diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js index 1c9f6d3..1369ee4 100644 --- a/src/routes/clinicalAssistant.js +++ b/src/routes/clinicalAssistant.js @@ -639,6 +639,11 @@ function normalizeMcpSearchResponse(result) { title: cleanTitle(r.title || 'Untitled resource'), page: r.page_number || r.pageNumber || null, page_count: r.page_count || r.pageCount || null, + folder_path: r.folder_path || r.folderPath || '', + category: r.category || '', + source_priority: r.source_priority || r.sourcePriority || '', + source_boost: r.source_boost || r.sourceBoost || null, + tags: Array.isArray(r.tags) ? r.tags : [], excerpt: clip(text, 1800), score: r.score, chunk_index: r.chunk_index, @@ -761,16 +766,19 @@ function parseCitationCluster(cluster) { } function isVisualSourceQuery(query) { - return /\b(visual|image|figure|photo|picture|diagram|graph|chart|radiology|radiologic|radiograph|x-?ray|cxr|imaging|ultrasound|ct|mri|scan|film)\b/i.test(String(query || '')); + return visualIntent(query).wanted.length > 0; } function isRadiologyQuery(query) { - return /\b(radiology|radiologic|radiograph|x-?ray|cxr|chest\s*(film|x-?ray|radiograph)|imaging|ultrasound|ct|mri|scan|film)\b/i.test(String(query || '')); + return visualIntent(query).wanted.indexOf('radiology') !== -1; } function buildMultimodalSearchQuery(query) { query = String(query || '').trim(); - if (isRadiologyQuery(query)) return query + ' radiograph x-ray chest imaging radiology'; + var intent = visualIntent(query); + if (intent.wanted.indexOf('radiology') !== -1) return query + ' radiograph x-ray chest imaging radiology'; + if (intent.wanted.indexOf('diagram') !== -1) return query + ' figure diagram pathway flowchart algorithm illustration'; + if (intent.wanted.indexOf('photo') !== -1) return query + ' clinical photograph image skin lesion pathology microscopy'; return query; } @@ -779,7 +787,7 @@ async function classifyAndRerankMultimodalResults(query, results) { if (!results.length) return []; var candidates = results.slice(0, VISUAL_CLASSIFIER_LIMIT); if (!VISUAL_CLASSIFIER_ENABLED) return selectMultimodalResults(query, candidates); - var radiology = isRadiologyQuery(query); + var intent = visualIntent(query); var classified = []; for (var i = 0; i < candidates.length; i++) { var candidate = candidates[i]; @@ -790,10 +798,11 @@ async function classifyAndRerankMultimodalResults(query, results) { if (!classification) continue; var metaScore = visualMetadataScore(query, candidate); var confident = classification.confidence >= VISUAL_CLASSIFIER_MIN_CONFIDENCE || metaScore > 0.1; - var accepted = radiology ? (classification.kind === 'radiology' && confident) : (classification.kind !== 'text_page' && confident); + var accepted = intent.wanted.indexOf(classification.kind) !== -1 && confident; if (!accepted) continue; candidate.visual_kind = classification.kind; candidate.visual_confidence = classification.confidence; + candidate.source_priority = classification.kind; candidate.visual_score = (Number(candidate.score) || 0) + (classification.confidence * 0.35) + metaScore; classified.push(candidate); } @@ -813,6 +822,24 @@ function selectMultimodalResults(query, results) { }); } +function visualIntent(query) { + var text = String(query || '').toLowerCase(); + var wanted = []; + if (/\b(radiology|radiologic|radiograph|x-?ray|cxr|chest\s*(film|x-?ray|radiograph)|imaging|ultrasound|ct|mri|scan|film)\b/i.test(text)) { + wanted.push('radiology'); + } + if (/\b(algorithm|flow\s*chart|flowchart|pathway|diagram|graph|chart|figure|figures|illustration|infographic|schema|schematic|table)\b/i.test(text)) { + wanted.push('diagram'); + } + if (/\b(photo|photograph|picture|skin|rash|lesion|wound|gross|microscopy|histology|pathology image|clinical image)\b/i.test(text)) { + wanted.push('photo'); + } + if (!wanted.length && /\b(visual|image|images|show|find|display|preview)\b/i.test(text)) { + wanted.push('diagram', 'photo'); + } + return { wanted: Array.from(new Set(wanted)) }; +} + async function classifyVisualSource(source) { var imagePath = resolveRawMultimodalImagePath(source.image_path_internal || ''); if (!imagePath) throw new Error('No local page image'); @@ -850,7 +877,10 @@ function visualClassifierLabels() { function visualMetadataScore(query, source) { var haystack = [source.title, source.excerpt, source.file_path, source.category, source.subcategory, source.category_path].filter(Boolean).join(' '); var score = 0; + var intent = visualIntent(query); if (isRadiologyQuery(query) && /\b(radiology|radiograph|x-?ray|cxr|imaging|ct|mri|ultrasound|film)\b/i.test(haystack)) score += 0.12; + if (intent.wanted.indexOf('diagram') !== -1 && /\b(figure|diagram|algorithm|flowchart|pathway|chart|illustration|infographic)\b/i.test(haystack)) score += 0.12; + if (intent.wanted.indexOf('photo') !== -1 && /\b(photo|photograph|skin|rash|lesion|wound|gross|microscopy|histology)\b/i.test(haystack)) score += 0.12; if (/\b(pneumonia|bronchiolitis|lung|airway|hyperinflation|infiltrate|consolidation|atelectasis)\b/i.test(haystack)) score += 0.06; if (looksLikeTextOnlyPage(source)) score -= 0.18; return score; @@ -908,9 +938,6 @@ function sanitizeSourcesForClient(sources) { var out = Object.assign({}, source); delete out.image_path_internal; delete out.file_path; - delete out.category; - delete out.subcategory; - delete out.category_path; return out; }); }