diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js index 5c286f5..e537f3b 100644 --- a/public/js/clinicalAssistant.js +++ b/public/js/clinicalAssistant.js @@ -301,6 +301,7 @@ var page = s.page || s.page_number || s.pageNumber; var meta = []; if (page) meta.push('page ' + page); + if (s.source_type === 'multimodal_page') meta.push('visual PDF page'); if (s.doc_type || s.type) meta.push(s.doc_type || s.type); if (s.score != null) meta.push('score ' + Number(s.score).toFixed(3)); return '
' + diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js index 0f38edc..7f0c76f 100644 --- a/src/routes/clinicalAssistant.js +++ b/src/routes/clinicalAssistant.js @@ -149,8 +149,20 @@ router.post('/clinical-assistant/chat', async function(req, res) { includeContext: includeContext, contextChars: contextChars }); - var rawResults = normalizeMcpSearchResponse(searchResponse); - var sources = dedupeSources(rawResults).slice(0, searchLimit); + var multimodalResponse = await multimodalSearch(message, { limit: 3 }).catch(function(e) { + console.warn('[clinical-assistant] multimodal search skipped:', e.message); + return null; + }); + var rawTextResults = normalizeMcpSearchResponse(searchResponse); + var rawMultimodalResults = normalizeMcpMultimodalResponse(multimodalResponse); + var rawResults = rawTextResults.concat(rawMultimodalResults); + var visualSlots = rawMultimodalResults.length ? Math.min(2, Math.max(1, Math.floor(searchLimit / 4))) : 0; + var textSlots = searchLimit - visualSlots; + var sources = dedupeSources( + dedupeSources(rawTextResults).slice(0, textSlots).concat( + dedupeSources(rawMultimodalResults).slice(0, visualSlots) + ) + ).slice(0, searchLimit); if (sources.length === 0) { return res.json({ @@ -192,6 +204,7 @@ router.post('/clinical-assistant/chat', async function(req, res) { duration: Date.now() - started, search: { totalFound: rawResults.length, + multimodalFound: rawMultimodalResults.length, verifiedChunkCount: searchResponse.verified_chunk_count || searchResponse.verifiedChunkCount || 0, droppedDocumentCount: searchResponse.dropped_document_count || searchResponse.droppedDocumentCount || 0 } @@ -252,6 +265,25 @@ router.post('/clinical-assistant/export-summary', async function(req, res) { }); async function semanticSearch(query, opts) { + return callMcpTool('nc_semantic_search', { + query: query, + limit: opts.limit, + doc_types: ['file'], + score_threshold: 0, + fusion: 'rrf', + include_context: opts.includeContext, + context_chars: opts.contextChars + }); +} + +async function multimodalSearch(query, opts) { + return callMcpTool('nc_multimodal_search', { + query: query, + limit: opts.limit + }); +} + +async function callMcpTool(name, args) { var session = await mcpRequest({ jsonrpc: '2.0', id: 1, @@ -267,16 +299,8 @@ async function semanticSearch(query, opts) { id: 2, method: 'tools/call', params: { - name: 'nc_semantic_search', - arguments: { - query: query, - limit: opts.limit, - doc_types: ['file'], - score_threshold: 0, - fusion: 'rrf', - include_context: opts.includeContext, - context_chars: opts.contextChars - } + name: name, + arguments: args } }, session.sessionId, session.mcpUrl); return search.result || search; @@ -420,6 +444,39 @@ function normalizeMcpSearchResponse(result) { }).filter(function(r) { return r.excerpt; }); } +function normalizeMcpMultimodalResponse(result) { + var data = result && (result.structuredContent || result.data || result); + if ((!data || !Array.isArray(data.results)) && result && Array.isArray(result.content)) { + for (var i = 0; i < result.content.length; i++) { + var c = result.content[i]; + if (c && c.type === 'text' && c.text) { + try { + var parsed = JSON.parse(c.text); + if (parsed && Array.isArray(parsed.results)) data = parsed; + } catch (e) {} + } + } + } + if (!data || !Array.isArray(data.results)) return []; + return data.results.map(function(r, idx) { + var nearby = clip(r.nearby_text || '', 1400); + return { + number: idx + 1, + id: r.id, + doc_type: r.doc_type || 'file', + source_type: 'multimodal_page', + title: cleanTitle(r.title || r.file_path || 'PDF page image'), + page: r.page_number || r.pageNumber || null, + page_count: r.page_count || r.pageCount || null, + excerpt: nearby ? '[Page-image match] ' + nearby : '[Page-image match] Rendered PDF page matched the visual/text query.', + score: r.score, + image_path: r.image_path || null, + chunk_index: 'page-image-' + (r.page_number || idx + 1), + total_chunks: r.page_count || null + }; + }).filter(function(r) { return r.page; }); +} + function dedupeSources(results) { var seen = new Set(); var out = []; @@ -434,7 +491,8 @@ function dedupeSources(results) { function formatSourcesForPrompt(sources) { return sources.map(function(s) { - return '[' + s.number + '] ' + s.title + (s.page ? ', page ' + s.page : '') + '\n' + s.excerpt; + var label = s.source_type === 'multimodal_page' ? ' [visual PDF page match]' : ''; + return '[' + s.number + '] ' + s.title + (s.page ? ', page ' + s.page : '') + label + '\n' + s.excerpt; }).join('\n\n---\n\n'); }