diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js
index 5c286f5..e537f3b 100644
--- a/public/js/clinicalAssistant.js
+++ b/public/js/clinicalAssistant.js
@@ -301,6 +301,7 @@
var page = s.page || s.page_number || s.pageNumber;
var meta = [];
if (page) meta.push('page ' + page);
+ if (s.source_type === 'multimodal_page') meta.push('visual PDF page');
if (s.doc_type || s.type) meta.push(s.doc_type || s.type);
if (s.score != null) meta.push('score ' + Number(s.score).toFixed(3));
return '
' +
diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js
index 0f38edc..7f0c76f 100644
--- a/src/routes/clinicalAssistant.js
+++ b/src/routes/clinicalAssistant.js
@@ -149,8 +149,20 @@ router.post('/clinical-assistant/chat', async function(req, res) {
includeContext: includeContext,
contextChars: contextChars
});
- var rawResults = normalizeMcpSearchResponse(searchResponse);
- var sources = dedupeSources(rawResults).slice(0, searchLimit);
+ var multimodalResponse = await multimodalSearch(message, { limit: 3 }).catch(function(e) {
+ console.warn('[clinical-assistant] multimodal search skipped:', e.message);
+ return null;
+ });
+ var rawTextResults = normalizeMcpSearchResponse(searchResponse);
+ var rawMultimodalResults = normalizeMcpMultimodalResponse(multimodalResponse);
+ var rawResults = rawTextResults.concat(rawMultimodalResults);
+ var visualSlots = rawMultimodalResults.length ? Math.min(2, Math.max(1, Math.floor(searchLimit / 4))) : 0;
+ var textSlots = searchLimit - visualSlots;
+ var sources = dedupeSources(
+ dedupeSources(rawTextResults).slice(0, textSlots).concat(
+ dedupeSources(rawMultimodalResults).slice(0, visualSlots)
+ )
+ ).slice(0, searchLimit);
if (sources.length === 0) {
return res.json({
@@ -192,6 +204,7 @@ router.post('/clinical-assistant/chat', async function(req, res) {
duration: Date.now() - started,
search: {
totalFound: rawResults.length,
+ multimodalFound: rawMultimodalResults.length,
verifiedChunkCount: searchResponse.verified_chunk_count || searchResponse.verifiedChunkCount || 0,
droppedDocumentCount: searchResponse.dropped_document_count || searchResponse.droppedDocumentCount || 0
}
@@ -252,6 +265,25 @@ router.post('/clinical-assistant/export-summary', async function(req, res) {
});
async function semanticSearch(query, opts) {
+ return callMcpTool('nc_semantic_search', {
+ query: query,
+ limit: opts.limit,
+ doc_types: ['file'],
+ score_threshold: 0,
+ fusion: 'rrf',
+ include_context: opts.includeContext,
+ context_chars: opts.contextChars
+ });
+}
+
+async function multimodalSearch(query, opts) {
+ return callMcpTool('nc_multimodal_search', {
+ query: query,
+ limit: opts.limit
+ });
+}
+
+async function callMcpTool(name, args) {
var session = await mcpRequest({
jsonrpc: '2.0',
id: 1,
@@ -267,16 +299,8 @@ async function semanticSearch(query, opts) {
id: 2,
method: 'tools/call',
params: {
- name: 'nc_semantic_search',
- arguments: {
- query: query,
- limit: opts.limit,
- doc_types: ['file'],
- score_threshold: 0,
- fusion: 'rrf',
- include_context: opts.includeContext,
- context_chars: opts.contextChars
- }
+ name: name,
+ arguments: args
}
}, session.sessionId, session.mcpUrl);
return search.result || search;
@@ -420,6 +444,39 @@ function normalizeMcpSearchResponse(result) {
}).filter(function(r) { return r.excerpt; });
}
+function normalizeMcpMultimodalResponse(result) {
+ var data = result && (result.structuredContent || result.data || result);
+ if ((!data || !Array.isArray(data.results)) && result && Array.isArray(result.content)) {
+ for (var i = 0; i < result.content.length; i++) {
+ var c = result.content[i];
+ if (c && c.type === 'text' && c.text) {
+ try {
+ var parsed = JSON.parse(c.text);
+ if (parsed && Array.isArray(parsed.results)) data = parsed;
+ } catch (e) {}
+ }
+ }
+ }
+ if (!data || !Array.isArray(data.results)) return [];
+ return data.results.map(function(r, idx) {
+ var nearby = clip(r.nearby_text || '', 1400);
+ return {
+ number: idx + 1,
+ id: r.id,
+ doc_type: r.doc_type || 'file',
+ source_type: 'multimodal_page',
+ title: cleanTitle(r.title || r.file_path || 'PDF page image'),
+ page: r.page_number || r.pageNumber || null,
+ page_count: r.page_count || r.pageCount || null,
+ excerpt: nearby ? '[Page-image match] ' + nearby : '[Page-image match] Rendered PDF page matched the visual/text query.',
+ score: r.score,
+ image_path: r.image_path || null,
+ chunk_index: 'page-image-' + (r.page_number || idx + 1),
+ total_chunks: r.page_count || null
+ };
+ }).filter(function(r) { return r.page; });
+}
+
function dedupeSources(results) {
var seen = new Set();
var out = [];
@@ -434,7 +491,8 @@ function dedupeSources(results) {
function formatSourcesForPrompt(sources) {
return sources.map(function(s) {
- return '[' + s.number + '] ' + s.title + (s.page ? ', page ' + s.page : '') + '\n' + s.excerpt;
+ var label = s.source_type === 'multimodal_page' ? ' [visual PDF page match]' : '';
+ return '[' + s.number + '] ' + s.title + (s.page ? ', page ' + s.page : '') + label + '\n' + s.excerpt;
}).join('\n\n---\n\n');
}