feat: blend multimodal assistant sources

public/js/clinicalAssistant.js

@@ -301,6 +301,7 @@
       var page = s.page || s.page_number || s.pageNumber;
       var meta = [];
       if (page) meta.push('page ' + page);
+      if (s.source_type === 'multimodal_page') meta.push('visual PDF page');
       if (s.doc_type || s.type) meta.push(s.doc_type || s.type);
       if (s.score != null) meta.push('score ' + Number(s.score).toFixed(3));
       return '<div class="assistant-source" id="assistant-source-' + n + '">' +

src/routes/clinicalAssistant.js

@@ -149,8 +149,20 @@ router.post('/clinical-assistant/chat', async function(req, res) {
       includeContext: includeContext,
       contextChars: contextChars
     });
-    var rawResults = normalizeMcpSearchResponse(searchResponse);
-    var sources = dedupeSources(rawResults).slice(0, searchLimit);
+    var multimodalResponse = await multimodalSearch(message, { limit: 3 }).catch(function(e) {
+      console.warn('[clinical-assistant] multimodal search skipped:', e.message);
+      return null;
+    });
+    var rawTextResults = normalizeMcpSearchResponse(searchResponse);
+    var rawMultimodalResults = normalizeMcpMultimodalResponse(multimodalResponse);
+    var rawResults = rawTextResults.concat(rawMultimodalResults);
+    var visualSlots = rawMultimodalResults.length ? Math.min(2, Math.max(1, Math.floor(searchLimit / 4))) : 0;
+    var textSlots = searchLimit - visualSlots;
+    var sources = dedupeSources(
+      dedupeSources(rawTextResults).slice(0, textSlots).concat(
+        dedupeSources(rawMultimodalResults).slice(0, visualSlots)
+      )
+    ).slice(0, searchLimit);
 
     if (sources.length === 0) {
       return res.json({
@@ -192,6 +204,7 @@ router.post('/clinical-assistant/chat', async function(req, res) {
       duration: Date.now() - started,
       search: {
         totalFound: rawResults.length,
+        multimodalFound: rawMultimodalResults.length,
         verifiedChunkCount: searchResponse.verified_chunk_count || searchResponse.verifiedChunkCount || 0,
         droppedDocumentCount: searchResponse.dropped_document_count || searchResponse.droppedDocumentCount || 0
       }
@@ -252,6 +265,25 @@ router.post('/clinical-assistant/export-summary', async function(req, res) {
 });
 
 async function semanticSearch(query, opts) {
+  return callMcpTool('nc_semantic_search', {
+    query: query,
+    limit: opts.limit,
+    doc_types: ['file'],
+    score_threshold: 0,
+    fusion: 'rrf',
+    include_context: opts.includeContext,
+    context_chars: opts.contextChars
+  });
+}
+
+async function multimodalSearch(query, opts) {
+  return callMcpTool('nc_multimodal_search', {
+    query: query,
+    limit: opts.limit
+  });
+}
+
+async function callMcpTool(name, args) {
   var session = await mcpRequest({
     jsonrpc: '2.0',
     id: 1,
@@ -267,16 +299,8 @@ async function semanticSearch(query, opts) {
     id: 2,
     method: 'tools/call',
     params: {
-      name: 'nc_semantic_search',
-      arguments: {
-        query: query,
-        limit: opts.limit,
-        doc_types: ['file'],
-        score_threshold: 0,
-        fusion: 'rrf',
-        include_context: opts.includeContext,
-        context_chars: opts.contextChars
-      }
+      name: name,
+      arguments: args
     }
   }, session.sessionId, session.mcpUrl);
   return search.result || search;
@@ -420,6 +444,39 @@ function normalizeMcpSearchResponse(result) {
   }).filter(function(r) { return r.excerpt; });
 }
 
+function normalizeMcpMultimodalResponse(result) {
+  var data = result && (result.structuredContent || result.data || result);
+  if ((!data || !Array.isArray(data.results)) && result && Array.isArray(result.content)) {
+    for (var i = 0; i < result.content.length; i++) {
+      var c = result.content[i];
+      if (c && c.type === 'text' && c.text) {
+        try {
+          var parsed = JSON.parse(c.text);
+          if (parsed && Array.isArray(parsed.results)) data = parsed;
+        } catch (e) {}
+      }
+    }
+  }
+  if (!data || !Array.isArray(data.results)) return [];
+  return data.results.map(function(r, idx) {
+    var nearby = clip(r.nearby_text || '', 1400);
+    return {
+      number: idx + 1,
+      id: r.id,
+      doc_type: r.doc_type || 'file',
+      source_type: 'multimodal_page',
+      title: cleanTitle(r.title || r.file_path || 'PDF page image'),
+      page: r.page_number || r.pageNumber || null,
+      page_count: r.page_count || r.pageCount || null,
+      excerpt: nearby ? '[Page-image match] ' + nearby : '[Page-image match] Rendered PDF page matched the visual/text query.',
+      score: r.score,
+      image_path: r.image_path || null,
+      chunk_index: 'page-image-' + (r.page_number || idx + 1),
+      total_chunks: r.page_count || null
+    };
+  }).filter(function(r) { return r.page; });
+}
+
 function dedupeSources(results) {
   var seen = new Set();
   var out = [];
@@ -434,7 +491,8 @@ function dedupeSources(results) {
 
 function formatSourcesForPrompt(sources) {
   return sources.map(function(s) {
-    return '[' + s.number + '] ' + s.title + (s.page ? ', page ' + s.page : '') + '\n' + s.excerpt;
+    var label = s.source_type === 'multimodal_page' ? ' [visual PDF page match]' : '';
+    return '[' + s.number + '] ' + s.title + (s.page ? ', page ' + s.page : '') + label + '\n' + s.excerpt;
   }).join('\n\n---\n\n');
 }