danvics/pediatric-ai-scribe-v3
1
0
Fork 0
Code Issues Pull requests Projects Releases 14 Packages Wiki Activity Actions

Fix LiteLLM STT: use chat/completions with Gemini audio instead of broken /audio/transcriptions

Browse source 
LiteLLM /audio/transcriptions gives 'Unmapped provider' for Vertex AI Chirp.
The correct approach: use /v1/chat/completions with a Gemini model and send
audio as base64 input_audio content block — Gemini natively understands audio.
Set LITELLM_STT_MODEL to your Gemini model name (e.g. gemini-2.5-flash).
This commit is contained in:
Daniel Onyejesi 2026-03-30 19:52:58 -04:00
parent 136a3402ab
commit bb459f57b8
1 changed files with 35 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
src/routes/transcribe.js
View file

@ -78,24 +78,41 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
if (provider === 'litellm') {
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
// Default to whisper-1 — vertex_ai/chirp does NOT work via LiteLLM audio proxy.
// User should set LITELLM_STT_MODEL to their configured whisper alias, or
// use TRANSCRIBE_PROVIDER=google for Vertex AI / Gemini transcription instead.
var sttModel = process.env.LITELLM_STT_MODEL || 'whisper-1';
var FormData = require('form-data');
var axios = require('axios');
var form = new FormData();
form.append('file', req.file.buffer, { filename: 'audio.webm', contentType: req.file.mimetype || 'audio/webm' });
form.append('model', sttModel);
var sttHeaders = Object.assign({}, form.getHeaders());
if (process.env.LITELLM_API_KEY) sttHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
var sttResp = await require('axios').post(
process.env.LITELLM_API_BASE.replace(/\/+$/, '') + '/v1/audio/transcriptions',
form, { headers: sttHeaders, timeout: 120000 }
);
var text = (sttResp.data && sttResp.data.text) ? sttResp.data.text : String(sttResp.data || '');
console.log('[Transcribe] LiteLLM (' + sttModel + ') done in ' + (Date.now() - startTime) + 'ms');
return res.json({ success: true, text: text, provider: 'litellm-stt', duration: Date.now() - startTime });
// LiteLLM /audio/transcriptions does NOT support Vertex AI Chirp (Unmapped provider error).
// Instead, use /v1/chat/completions with a Gemini model — Gemini understands audio natively
// via base64 inline data. Set LITELLM_STT_MODEL to your Gemini model name in LiteLLM.
var sttModel = process.env.LITELLM_STT_MODEL || 'gemini-2.0-flash';
var axios = require('axios');
var base64audio = req.file.buffer.toString('base64');
var mimeType = req.file.mimetype || 'audio/webm';
var base = process.env.LITELLM_API_BASE.replace(/\/+$/, '');
var headers = { 'Content-Type': 'application/json' };
if (process.env.LITELLM_API_KEY) headers['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
var sttResp = await axios.post(base + '/v1/chat/completions', {
model: sttModel,
messages: [{
role: 'user',
content: [
{
type: 'input_audio',
input_audio: { data: base64audio, format: mimeType.split('/')[1] || 'webm' }
},
{
type: 'text',
text: 'Transcribe this audio of a medical encounter. Output the spoken words only, exactly as heard. No commentary, no formatting.'
}
]
}]
}, { headers: headers, timeout: 120000 });
var text = '';
if (sttResp.data && sttResp.data.choices && sttResp.data.choices[0]) {
var msg = sttResp.data.choices[0].message;
text = (msg && msg.content) ? msg.content : '';
}
console.log('[Transcribe] LiteLLM/' + sttModel + ' done in ' + (Date.now() - startTime) + 'ms');
return res.json({ success: true, text: text.trim(), provider: 'litellm-gemini', duration: Date.now() - startTime });
}
// OpenAI Whisper (direct)