restore assistant streaming
This commit is contained in:
Daniel
parent
a8f364f177
commit
10f39fc4ff
3 changed files with 220 additions and 3 deletions
|
|
@ -123,7 +123,7 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown } from './assistant/cit
|
|||
setBusy(true, 'Looking up sources...');
|
||||
var loading = appendLoadingMessage('Looking up sources', 'Retrieving and synthesizing references...');
|
||||
|
||||
fetchAssistantResponse({
|
||||
streamAssistantResponse({
|
||||
message: text,
|
||||
history: messages.slice(-8),
|
||||
includeContext: !includeContext || includeContext.checked
|
||||
|
|
@ -149,6 +149,106 @@ import { escapeAttr, escapeHtml, renderAssistantMarkdown } from './assistant/cit
|
|||
}
|
||||
}
|
||||
|
||||
async function streamAssistantResponse(payload, loading) {
|
||||
var response = await fetch('/api/clinical-assistant/chat/stream', {
|
||||
method: 'POST',
|
||||
headers: getAuthHeaders(),
|
||||
credentials: 'same-origin',
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
if (!response.ok || !response.body) {
|
||||
var fallback = await response.json().catch(function () { return {}; });
|
||||
throw new Error(fallback.error || ('Request failed (' + response.status + ')'));
|
||||
}
|
||||
|
||||
var partial = '';
|
||||
var streamSources = [];
|
||||
var finalData = null;
|
||||
var lastRender = 0;
|
||||
var bubble = loading ? loading.querySelector('.assistant-bubble') : null;
|
||||
var decoder = new TextDecoder();
|
||||
var buffer = '';
|
||||
|
||||
function renderProvisional(force) {
|
||||
var now = Date.now();
|
||||
if (!force && now - lastRender < 180) return;
|
||||
lastRender = now;
|
||||
if (!bubble) return;
|
||||
loading.classList.remove('assistant-loading-msg');
|
||||
bubble.classList.remove('assistant-thinking');
|
||||
bubble.innerHTML = partial ? renderAssistantBubbleHtml(partial, streamSources, false) : '<p class="assistant-muted">Generating answer...</p>';
|
||||
renderEmbeddedBlocks(bubble);
|
||||
var wrap = document.getElementById('assistant-messages');
|
||||
if (wrap) wrap.scrollTop = wrap.scrollHeight;
|
||||
}
|
||||
|
||||
function handleEvent(type, data) {
|
||||
if (type === 'status') {
|
||||
updateLoadingMessage(loading, data.message || 'Working...');
|
||||
return;
|
||||
}
|
||||
if (type === 'sources') {
|
||||
streamSources = data.sources || [];
|
||||
renderSources(streamSources);
|
||||
return;
|
||||
}
|
||||
if (type === 'token') {
|
||||
partial += data.token || '';
|
||||
renderProvisional(false);
|
||||
return;
|
||||
}
|
||||
if (type === 'done') {
|
||||
finalData = data || {};
|
||||
return;
|
||||
}
|
||||
if (type === 'error') throw new Error(data.error || 'Assistant stream failed');
|
||||
}
|
||||
|
||||
var reader = response.body.getReader();
|
||||
while (true) {
|
||||
var chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
buffer += decoder.decode(chunk.value, { stream: true });
|
||||
var parts = buffer.split('\n\n');
|
||||
buffer = parts.pop() || '';
|
||||
parts.forEach(function (part) {
|
||||
var parsed = parseSseEvent(part);
|
||||
if (parsed) handleEvent(parsed.type, parsed.data);
|
||||
});
|
||||
}
|
||||
if (buffer.trim()) {
|
||||
var tail = parseSseEvent(buffer);
|
||||
if (tail) handleEvent(tail.type, tail.data);
|
||||
}
|
||||
|
||||
if (!finalData) {
|
||||
updateLoadingMessage(loading, 'Stream ended early. Retrying without streaming...');
|
||||
finalData = await fetchAssistantFallback(payload);
|
||||
}
|
||||
|
||||
setBusy(false, 'Ready');
|
||||
lastAnswer = finalData.answer || finalData.markdown || '';
|
||||
lastSources = finalData.sources || finalData.citations || streamSources;
|
||||
replaceLoadingMessage(loading, lastAnswer, lastSources, finalData.suggestions || []);
|
||||
renderSources(lastSources);
|
||||
if (finalData.model) {
|
||||
var label = document.getElementById('assistant-model-label');
|
||||
if (label) label.textContent = 'Chat: ' + finalData.model;
|
||||
}
|
||||
}
|
||||
|
||||
function parseSseEvent(block) {
|
||||
var type = 'message';
|
||||
var data = '';
|
||||
String(block || '').split(/\r?\n/).forEach(function (line) {
|
||||
if (line.indexOf('event:') === 0) type = line.substring(6).trim();
|
||||
if (line.indexOf('data:') === 0) data += line.substring(5).trim();
|
||||
});
|
||||
if (!data) return null;
|
||||
try { return { type: type, data: JSON.parse(data) }; }
|
||||
catch (e) { return null; }
|
||||
}
|
||||
|
||||
async function fetchAssistantFallback(payload) {
|
||||
var response = await fetch('/api/clinical-assistant/chat', {
|
||||
method: 'POST',
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ var axios = require('axios');
|
|||
var router = express.Router();
|
||||
var db = require('../db/database');
|
||||
var { authMiddleware } = require('../middleware/auth');
|
||||
var { callAI } = require('../utils/ai');
|
||||
var { callAI, callAIStream } = require('../utils/ai');
|
||||
var { gatewayUrl } = require('../utils/errors');
|
||||
var logger = require('../utils/logger');
|
||||
var cryptoUtil = require('../utils/crypto');
|
||||
|
|
@ -273,6 +273,77 @@ router.post('/clinical-assistant/chat', async function(req, res) {
|
|||
}
|
||||
});
|
||||
|
||||
router.post('/clinical-assistant/chat/stream', async function(req, res) {
|
||||
var started = Date.now();
|
||||
var streamOpen = false;
|
||||
function sendEvent(type, data) {
|
||||
if (!streamOpen) return;
|
||||
res.write('event: ' + type + '\n');
|
||||
res.write('data: ' + JSON.stringify(data || {}) + '\n\n');
|
||||
}
|
||||
|
||||
try {
|
||||
res.setHeader('Content-Type', 'text/event-stream; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
if (typeof res.flushHeaders === 'function') res.flushHeaders();
|
||||
streamOpen = true;
|
||||
sendEvent('status', { message: 'Looking up sources...' });
|
||||
|
||||
var prepared = await prepareAssistantChat(req.body);
|
||||
if (prepared.direct) {
|
||||
sendEvent('done', Object.assign({ duration: Date.now() - started }, prepared.direct));
|
||||
return res.end();
|
||||
}
|
||||
|
||||
var safeSources = sanitizeSourcesForClient(prepared.sources);
|
||||
sendEvent('sources', { sources: safeSources, search: prepared.search });
|
||||
sendEvent('status', { message: 'Generating answer...' });
|
||||
|
||||
var ai = await callAIStream(prepared.messages, {
|
||||
model: prepared.chatModel || undefined,
|
||||
temperature: 0.15,
|
||||
maxTokens: 2600
|
||||
}, function(delta) {
|
||||
sendEvent('token', { token: delta });
|
||||
});
|
||||
|
||||
var answer = stripModelSourcesSection(String(ai.content || '').trim());
|
||||
if (shouldRegenerateTruncatedAnswer(answer, ai.finishReason)) {
|
||||
console.warn('[clinical-assistant] streamed answer looked truncated; regenerating final answer', { finishReason: ai.finishReason, chars: answer.length });
|
||||
sendEvent('status', { message: 'Completing answer...' });
|
||||
var completed = await callAI(prepared.messages, {
|
||||
model: prepared.chatModel || undefined,
|
||||
temperature: 0.15,
|
||||
maxTokens: 5000
|
||||
});
|
||||
answer = stripModelSourcesSection(String(completed.content || '').trim()) || answer;
|
||||
ai.model = completed.model || ai.model;
|
||||
ai.provider = completed.provider || ai.provider;
|
||||
ai.finishReason = completed.finishReason || ai.finishReason;
|
||||
}
|
||||
|
||||
logger.audit(req.user.id, 'clinical_assistant_query', 'Clinical assistant streaming query', req, {
|
||||
category: 'clinical', model: ai.model || prepared.chatModel, duration: Date.now() - started
|
||||
});
|
||||
sendEvent('done', {
|
||||
success: true,
|
||||
answer: answer,
|
||||
sources: safeSources,
|
||||
model: ai.model || prepared.chatModel || null,
|
||||
provider: ai.provider || null,
|
||||
duration: Date.now() - started,
|
||||
search: prepared.search
|
||||
});
|
||||
res.end();
|
||||
} catch (e) {
|
||||
console.error('[clinical-assistant stream]', e.message, e.stack || '');
|
||||
if (!streamOpen) return res.status(e.statusCode || 500).json({ error: assistantErrorMessage(e) });
|
||||
sendEvent('error', { error: assistantErrorMessage(e) });
|
||||
res.end();
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/clinical-assistant/image', async function(req, res) {
|
||||
try {
|
||||
var prompt = String(req.body.prompt || '').trim();
|
||||
|
|
|
|||
|
|
@ -417,6 +417,52 @@ async function assertModelAllowed(requestedModel, options) {
|
|||
}
|
||||
}
|
||||
|
||||
async function callAIStream(messages, options, onToken) {
|
||||
options = options || {};
|
||||
var requestedModel = options.model;
|
||||
var model = requestedModel || DEFAULT_MODEL;
|
||||
var temperature = options.temperature || 0.3;
|
||||
var maxTokens = options.maxTokens || 4000;
|
||||
var startTime = Date.now();
|
||||
await assertModelAllowed(requestedModel, options);
|
||||
|
||||
var client = null;
|
||||
var provider = null;
|
||||
if (activeProvider === 'litellm' && litellmClient) {
|
||||
client = litellmClient;
|
||||
provider = 'litellm';
|
||||
} else if (activeProvider === 'openrouter' && openrouter) {
|
||||
client = openrouter;
|
||||
provider = 'openrouter';
|
||||
} else if (activeProvider === 'azure' && azureClient) {
|
||||
client = azureClient;
|
||||
provider = 'azure';
|
||||
model = process.env.AZURE_DEPLOYMENT_NAME || model;
|
||||
}
|
||||
if (!client) throw new Error('Streaming is only configured for OpenAI-compatible providers');
|
||||
|
||||
var content = '';
|
||||
var finishReason = null;
|
||||
var stream = await client.chat.completions.create({
|
||||
model: model,
|
||||
messages: messages,
|
||||
temperature: temperature,
|
||||
max_tokens: maxTokens,
|
||||
stream: true
|
||||
});
|
||||
for await (var part of stream) {
|
||||
var choice = part && part.choices && part.choices[0] ? part.choices[0] : null;
|
||||
if (choice && choice.finish_reason) finishReason = choice.finish_reason;
|
||||
var delta = choice && choice.delta ? choice.delta.content : '';
|
||||
if (!delta) continue;
|
||||
content += delta;
|
||||
if (typeof onToken === 'function') onToken(delta);
|
||||
}
|
||||
var duration = Date.now() - startTime;
|
||||
logger.apiCall(null, provider + '/' + model, { model: model, duration: duration, statusCode: 200 });
|
||||
return { success: true, content: content, model: model, provider: provider, duration: duration, finishReason: finishReason };
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// MAIN CALL AI FUNCTION — Routes to correct provider
|
||||
// ============================================================
|
||||
|
|
@ -637,4 +683,4 @@ async function discoverModels() {
|
|||
return discovered;
|
||||
}
|
||||
|
||||
module.exports = { callAI, whisperClient, activeProvider, discoverModels, vertexClient, litellmClient };
|
||||
module.exports = { callAI, callAIStream, whisperClient, activeProvider, discoverModels, vertexClient, litellmClient };
|
||||
|
|
|
|||
Loading…
Reference in a new issue