diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js index 2a25e8d..f111913 100644 --- a/public/js/clinicalAssistant.js +++ b/public/js/clinicalAssistant.js @@ -123,7 +123,7 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown } from './assistant/cit setBusy(true, 'Looking up sources...'); var loading = appendLoadingMessage('Looking up sources', 'Retrieving and synthesizing references...'); - fetchAssistantResponse({ + streamAssistantResponse({ message: text, history: messages.slice(-8), includeContext: !includeContext || includeContext.checked @@ -149,6 +149,106 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown } from './assistant/cit } } + async function streamAssistantResponse(payload, loading) { + var response = await fetch('/api/clinical-assistant/chat/stream', { + method: 'POST', + headers: getAuthHeaders(), + credentials: 'same-origin', + body: JSON.stringify(payload) + }); + if (!response.ok || !response.body) { + var fallback = await response.json().catch(function () { return {}; }); + throw new Error(fallback.error || ('Request failed (' + response.status + ')')); + } + + var partial = ''; + var streamSources = []; + var finalData = null; + var lastRender = 0; + var bubble = loading ? loading.querySelector('.assistant-bubble') : null; + var decoder = new TextDecoder(); + var buffer = ''; + + function renderProvisional(force) { + var now = Date.now(); + if (!force && now - lastRender < 180) return; + lastRender = now; + if (!bubble) return; + loading.classList.remove('assistant-loading-msg'); + bubble.classList.remove('assistant-thinking'); + bubble.innerHTML = partial ? renderAssistantBubbleHtml(partial, streamSources, false) : '
Generating answer...
'; + renderEmbeddedBlocks(bubble); + var wrap = document.getElementById('assistant-messages'); + if (wrap) wrap.scrollTop = wrap.scrollHeight; + } + + function handleEvent(type, data) { + if (type === 'status') { + updateLoadingMessage(loading, data.message || 'Working...'); + return; + } + if (type === 'sources') { + streamSources = data.sources || []; + renderSources(streamSources); + return; + } + if (type === 'token') { + partial += data.token || ''; + renderProvisional(false); + return; + } + if (type === 'done') { + finalData = data || {}; + return; + } + if (type === 'error') throw new Error(data.error || 'Assistant stream failed'); + } + + var reader = response.body.getReader(); + while (true) { + var chunk = await reader.read(); + if (chunk.done) break; + buffer += decoder.decode(chunk.value, { stream: true }); + var parts = buffer.split('\n\n'); + buffer = parts.pop() || ''; + parts.forEach(function (part) { + var parsed = parseSseEvent(part); + if (parsed) handleEvent(parsed.type, parsed.data); + }); + } + if (buffer.trim()) { + var tail = parseSseEvent(buffer); + if (tail) handleEvent(tail.type, tail.data); + } + + if (!finalData) { + updateLoadingMessage(loading, 'Stream ended early. Retrying without streaming...'); + finalData = await fetchAssistantFallback(payload); + } + + setBusy(false, 'Ready'); + lastAnswer = finalData.answer || finalData.markdown || ''; + lastSources = finalData.sources || finalData.citations || streamSources; + replaceLoadingMessage(loading, lastAnswer, lastSources, finalData.suggestions || []); + renderSources(lastSources); + if (finalData.model) { + var label = document.getElementById('assistant-model-label'); + if (label) label.textContent = 'Chat: ' + finalData.model; + } + } + + function parseSseEvent(block) { + var type = 'message'; + var data = ''; + String(block || '').split(/\r?\n/).forEach(function (line) { + if (line.indexOf('event:') === 0) type = line.substring(6).trim(); + if (line.indexOf('data:') === 0) data += line.substring(5).trim(); + }); + if (!data) return null; + try { return { type: type, data: JSON.parse(data) }; } + catch (e) { return null; } + } + async function fetchAssistantFallback(payload) { var response = await fetch('/api/clinical-assistant/chat', { method: 'POST', diff --git a/src/routes/clinicalAssistant.js b/src/routes/clinicalAssistant.js index 04f6871..dc36fd3 100644 --- a/src/routes/clinicalAssistant.js +++ b/src/routes/clinicalAssistant.js @@ -10,7 +10,7 @@ var axios = require('axios'); var router = express.Router(); var db = require('../db/database'); var { authMiddleware } = require('../middleware/auth'); -var { callAI } = require('../utils/ai'); +var { callAI, callAIStream } = require('../utils/ai'); var { gatewayUrl } = require('../utils/errors'); var logger = require('../utils/logger'); var cryptoUtil = require('../utils/crypto'); @@ -273,6 +273,77 @@ router.post('/clinical-assistant/chat', async function(req, res) { } }); +router.post('/clinical-assistant/chat/stream', async function(req, res) { + var started = Date.now(); + var streamOpen = false; + function sendEvent(type, data) { + if (!streamOpen) return; + res.write('event: ' + type + '\n'); + res.write('data: ' + JSON.stringify(data || {}) + '\n\n'); + } + + try { + res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); + if (typeof res.flushHeaders === 'function') res.flushHeaders(); + streamOpen = true; + sendEvent('status', { message: 'Looking up sources...' }); + + var prepared = await prepareAssistantChat(req.body); + if (prepared.direct) { + sendEvent('done', Object.assign({ duration: Date.now() - started }, prepared.direct)); + return res.end(); + } + + var safeSources = sanitizeSourcesForClient(prepared.sources); + sendEvent('sources', { sources: safeSources, search: prepared.search }); + sendEvent('status', { message: 'Generating answer...' }); + + var ai = await callAIStream(prepared.messages, { + model: prepared.chatModel || undefined, + temperature: 0.15, + maxTokens: 2600 + }, function(delta) { + sendEvent('token', { token: delta }); + }); + + var answer = stripModelSourcesSection(String(ai.content || '').trim()); + if (shouldRegenerateTruncatedAnswer(answer, ai.finishReason)) { + console.warn('[clinical-assistant] streamed answer looked truncated; regenerating final answer', { finishReason: ai.finishReason, chars: answer.length }); + sendEvent('status', { message: 'Completing answer...' }); + var completed = await callAI(prepared.messages, { + model: prepared.chatModel || undefined, + temperature: 0.15, + maxTokens: 5000 + }); + answer = stripModelSourcesSection(String(completed.content || '').trim()) || answer; + ai.model = completed.model || ai.model; + ai.provider = completed.provider || ai.provider; + ai.finishReason = completed.finishReason || ai.finishReason; + } + + logger.audit(req.user.id, 'clinical_assistant_query', 'Clinical assistant streaming query', req, { + category: 'clinical', model: ai.model || prepared.chatModel, duration: Date.now() - started + }); + sendEvent('done', { + success: true, + answer: answer, + sources: safeSources, + model: ai.model || prepared.chatModel || null, + provider: ai.provider || null, + duration: Date.now() - started, + search: prepared.search + }); + res.end(); + } catch (e) { + console.error('[clinical-assistant stream]', e.message, e.stack || ''); + if (!streamOpen) return res.status(e.statusCode || 500).json({ error: assistantErrorMessage(e) }); + sendEvent('error', { error: assistantErrorMessage(e) }); + res.end(); + } +}); + router.post('/clinical-assistant/image', async function(req, res) { try { var prompt = String(req.body.prompt || '').trim(); diff --git a/src/utils/ai.js b/src/utils/ai.js index 8555f85..d38589e 100644 --- a/src/utils/ai.js +++ b/src/utils/ai.js @@ -417,6 +417,52 @@ async function assertModelAllowed(requestedModel, options) { } } +async function callAIStream(messages, options, onToken) { + options = options || {}; + var requestedModel = options.model; + var model = requestedModel || DEFAULT_MODEL; + var temperature = options.temperature || 0.3; + var maxTokens = options.maxTokens || 4000; + var startTime = Date.now(); + await assertModelAllowed(requestedModel, options); + + var client = null; + var provider = null; + if (activeProvider === 'litellm' && litellmClient) { + client = litellmClient; + provider = 'litellm'; + } else if (activeProvider === 'openrouter' && openrouter) { + client = openrouter; + provider = 'openrouter'; + } else if (activeProvider === 'azure' && azureClient) { + client = azureClient; + provider = 'azure'; + model = process.env.AZURE_DEPLOYMENT_NAME || model; + } + if (!client) throw new Error('Streaming is only configured for OpenAI-compatible providers'); + + var content = ''; + var finishReason = null; + var stream = await client.chat.completions.create({ + model: model, + messages: messages, + temperature: temperature, + max_tokens: maxTokens, + stream: true + }); + for await (var part of stream) { + var choice = part && part.choices && part.choices[0] ? part.choices[0] : null; + if (choice && choice.finish_reason) finishReason = choice.finish_reason; + var delta = choice && choice.delta ? choice.delta.content : ''; + if (!delta) continue; + content += delta; + if (typeof onToken === 'function') onToken(delta); + } + var duration = Date.now() - startTime; + logger.apiCall(null, provider + '/' + model, { model: model, duration: duration, statusCode: 200 }); + return { success: true, content: content, model: model, provider: provider, duration: duration, finishReason: finishReason }; +} + // ============================================================ // MAIN CALL AI FUNCTION — Routes to correct provider // ============================================================ @@ -637,4 +683,4 @@ async function discoverModels() { return discovered; } -module.exports = { callAI, whisperClient, activeProvider, discoverModels, vertexClient, litellmClient }; +module.exports = { callAI, callAIStream, whisperClient, activeProvider, discoverModels, vertexClient, litellmClient };