danvics/pediatric-ai-scribe-v3
1
0
Fork 0
Code Issues Pull requests Projects Releases 14 Packages Wiki Activity Actions

Fix STT/TTS properly per LiteLLM docs

Browse source 
STT: Vertex AI Chirp not supported via LiteLLM proxy (confirmed by docs).
     Now uses Gemini directly (transcribeGoogle.js) — auto-detected when
     GOOGLE_VERTEX_PROJECT is set, fallback to AWS then OpenAI.

TTS: LiteLLM Vertex TTS DOES work but requires the model_list ALIAS
     (tts-1) not the underlying path (vertex_ai/text-to-speech).
     Also pass voice param — LiteLLM supports Google Cloud voice names.
     Auto-detected when LITELLM_API_BASE is set.
This commit is contained in:
Daniel Onyejesi 2026-03-30 07:28:47 -04:00
parent b40dc5584b
commit a2263d9530
5 changed files with 174 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.env.example
View file

@ -26,6 +26,15 @@ OPENROUTER_API_KEY=sk-or-v1-your-key
# GOOGLE_VERTEX_LOCATION=us-central1
# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
# (Or use default credentials if running on GCE/GKE/Cloud Run)
#
# Google STT — Gemini inline audio (auto-detected when GOOGLE_VERTEX_PROJECT set)
# TRANSCRIBE_PROVIDER=google
# GOOGLE_STT_MODEL=gemini-2.0-flash # or gemini-2.5-flash for better accuracy
#
# Google TTS — Google Cloud Text-to-Speech (auto-detected when GOOGLE_VERTEX_PROJECT set)
# TTS_PROVIDER=google
# GOOGLE_TTS_VOICE=en-US-Journey-F # female | en-US-Journey-D = male
# Other options: en-US-Studio-O, en-US-Neural2-C, en-US-Neural2-J
# Option 5: LiteLLM Proxy (self-hosted, routes to any provider)
# AI_PROVIDER=litellm
@ -35,12 +44,13 @@ OPENROUTER_API_KEY=sk-or-v1-your-key
#
# LiteLLM Speech-to-Text (auto-enabled when LITELLM_API_BASE is set)
# TRANSCRIBE_PROVIDER=litellm
# LITELLM_STT_MODEL=vertex_ai/chirp # or: vertex_ai/chirp_2 for newer model
# LiteLLM STT: NOT supported for Vertex AI Chirp via LiteLLM proxy.
# Use TRANSCRIBE_PROVIDER=google for Vertex AI / Gemini STT.
#
# LiteLLM Text-to-Speech (auto-enabled when LITELLM_API_BASE is set)
# TTS_PROVIDER=litellm
# LITELLM_TTS_MODEL=vertex_ai/text-to-speech # voice is set inside LiteLLM per model
# LITELLM_TTS_VOICE is ignored for Vertex AI — voice is configured in LiteLLM model entry
# LiteLLM TTS: WORKS — use the model_name alias from your LiteLLM model_list
# TTS_PROVIDER=litellm (auto-detected when LITELLM_API_BASE set)
# LITELLM_TTS_MODEL=tts-1 # model_name alias in LiteLLM config
# LITELLM_TTS_VOICE=en-US-Journey-F # passed to LiteLLM, supports Google Cloud voice names
# ============================================================
# TRANSCRIPTION (speech-to-text)

69
src/routes/transcribe.js
View file

@ -4,32 +4,33 @@ const multer = require('multer');
const { whisperClient, litellmClient } = require('../utils/ai');
const { transcribeWithAWS, isAWSTranscribeConfigured } = require('../utils/transcribeAWS');
const { transcribeWithLocal, isLocalWhisperConfigured } = require('../utils/transcribeLocal');
const { transcribeWithGemini, isGoogleSTTConfigured } = require('../utils/transcribeGoogle');
const { authMiddleware } = require('../middleware/auth');
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } });
// Determine which provider to use:
// TRANSCRIBE_PROVIDER=litellm → LiteLLM proxy (any Whisper-compatible model)
// TRANSCRIBE_PROVIDER=aws → Amazon Transcribe
// TRANSCRIBE_PROVIDER=local → local whisper.cpp / faster-whisper
// TRANSCRIBE_PROVIDER=openai → OpenAI Whisper directly
// (auto) LITELLM_API_BASE set → litellm
// (auto) AWS Bedrock region set → aws
// (default) → openai whisper
// Provider priority (auto-detect):
// TRANSCRIBE_PROVIDER=google → Vertex AI / Gemini (direct, HIPAA eligible)
// TRANSCRIBE_PROVIDER=aws → Amazon Transcribe (HIPAA eligible)
// TRANSCRIBE_PROVIDER=local → local whisper.cpp / faster-whisper
// TRANSCRIBE_PROVIDER=openai → OpenAI Whisper (direct)
// TRANSCRIBE_PROVIDER=litellm→ LiteLLM proxy (explicit only — audio routing bugs)
// Auto: google > aws > openai
function getTranscribeProvider() {
var env = process.env.TRANSCRIBE_PROVIDER;
if (env === 'litellm') return 'litellm';
if (env === 'local') return 'local';
if (env === 'google') return 'google';
if (env === 'aws') return 'aws';
if (env === 'local') return 'local';
if (env === 'openai') return 'openai';
// Auto-detect priority: AWS > LiteLLM > OpenAI
// LiteLLM's Vertex AI audio proxy has routing bugs — only use it if explicitly set
if (env === 'litellm') return 'litellm';
// Auto-detect
if (isGoogleSTTConfigured()) return 'google';
if (isAWSTranscribeConfigured()) return 'aws';
if (process.env.LITELLM_API_BASE && litellmClient) return 'litellm';
return 'openai';
}
function isTranscribeAvailable() {
if (isGoogleSTTConfigured()) return true;
if (isLocalWhisperConfigured()) return true;
if (isAWSTranscribeConfigured()) return true;
if (litellmClient) return true;
@ -38,11 +39,13 @@ function isTranscribeAvailable() {
}
var provider = getTranscribeProvider();
var medical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true';
var medical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true';
var available = isTranscribeAvailable();
console.log('🎙️ Transcribe provider:', provider + (provider === 'aws' && medical ? ' (Medical)' : '') + (available ? '' : ' (NOT CONFIGURED — browser speech only)'));
console.log('🎙️ Transcribe provider:', provider +
(provider === 'aws' && medical ? ' (Medical)' : '') +
(provider === 'google' ? ' (model: ' + (process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash') + ')' : '') +
(available ? '' : ' (NOT CONFIGURED — browser speech only)'));
// Status endpoint — frontend checks this to decide whether to upload audio
router.get('/transcribe/status', authMiddleware, (req, res) => {
res.json({ available: available, provider: available ? provider : 'none' });
});
@ -51,9 +54,15 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
try {
if (!req.file) return res.status(400).json({ error: 'No audio' });
var startTime = Date.now();
var fileSize = req.file.size;
var fileSize = req.file.size;
console.log('[Transcribe] Received ' + (fileSize / 1024).toFixed(0) + 'KB audio (' + (req.file.mimetype || 'unknown') + ') via ' + provider);
if (provider === 'google') {
var text = await transcribeWithGemini(req.file.buffer, req.file.mimetype || 'audio/webm');
console.log('[Transcribe] Google/Gemini done in ' + (Date.now() - startTime) + 'ms');
return res.json({ success: true, text: text, provider: 'google-gemini', duration: Date.now() - startTime });
}
if (provider === 'local') {
var text = await transcribeWithLocal(req.file.buffer, req.file.mimetype || 'audio/webm');
console.log('[Transcribe] Local done in ' + (Date.now() - startTime) + 'ms');
@ -61,9 +70,7 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
}
if (provider === 'aws') {
if (!isAWSTranscribeConfigured()) {
return res.status(400).json({ error: 'AWS Transcribe not configured. Set AWS_BEDROCK_REGION.' });
}
if (!isAWSTranscribeConfigured()) return res.status(400).json({ error: 'AWS Transcribe not configured.' });
var text = await transcribeWithAWS(req.file.buffer, req.file.mimetype || 'audio/webm');
console.log('[Transcribe] AWS done in ' + (Date.now() - startTime) + 'ms');
return res.json({ success: true, text: text, provider: 'aws-transcribe', duration: Date.now() - startTime });
@ -73,22 +80,16 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
var sttModel = process.env.LITELLM_STT_MODEL || 'vertex_ai/chirp';
var FormData = require('form-data');
var axios = require('axios');
var axios = require('axios');
var form = new FormData();
form.append('file', req.file.buffer, {
filename: 'audio.webm',
contentType: req.file.mimetype || 'audio/webm'
});
form.append('file', req.file.buffer, { filename: 'audio.webm', contentType: req.file.mimetype || 'audio/webm' });
form.append('model', sttModel);
// Note: Vertex AI Chirp via LiteLLM does not support 'prompt' or 'response_format'
// Only send parameters Chirp actually accepts
var sttHeaders = Object.assign({}, form.getHeaders());
if (process.env.LITELLM_API_KEY) sttHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
var base = process.env.LITELLM_API_BASE.replace(/\/+$/, '');
var sttResp = await axios.post(base + '/v1/audio/transcriptions', form, {
headers: sttHeaders,
timeout: 120000
});
var sttResp = await require('axios').post(
process.env.LITELLM_API_BASE.replace(/\/+$/, '') + '/v1/audio/transcriptions',
form, { headers: sttHeaders, timeout: 120000 }
);
var text = (sttResp.data && sttResp.data.text) ? sttResp.data.text : String(sttResp.data || '');
console.log('[Transcribe] LiteLLM (' + sttModel + ') done in ' + (Date.now() - startTime) + 'ms');
return res.json({ success: true, text: text, provider: 'litellm-stt', duration: Date.now() - startTime });
@ -96,7 +97,7 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
// OpenAI Whisper (direct)
if (!whisperClient) return res.status(400).json({ error: 'Whisper not configured. Set OPENAI_API_KEY.' });
var file = new File([req.file.buffer], 'audio.webm', { type: req.file.mimetype || 'audio/webm' });
var file = new File([req.file.buffer], 'audio.webm', { type: req.file.mimetype || 'audio/webm' });
var result = await whisperClient.audio.transcriptions.create({
file, model: 'whisper-1', language: 'en',
response_format: 'text',
@ -105,8 +106,8 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
var text = typeof result === 'string' ? result : result.text;
console.log('[Transcribe] Whisper done in ' + (Date.now() - startTime) + 'ms');
res.json({ success: true, text: text, provider: 'openai-whisper', duration: Date.now() - startTime });
} catch (err) {
// Log full LiteLLM/upstream error body so we can diagnose
var detail = err.response && err.response.data
? JSON.stringify(err.response.data).substring(0, 500)
: err.message;

55
src/routes/tts.js
View file

@ -1,53 +1,62 @@
const express = require('express');
const router = express.Router();
const axios = require('axios');
const router = express.Router();
const axios = require('axios');
const { synthesizeWithGoogleTTS, isGoogleTTSConfigured } = require('../utils/ttsGoogle');
const { authMiddleware } = require('../middleware/auth');
// TTS_PROVIDER=litellm → LiteLLM proxy (Vertex AI TTS, OpenAI TTS, etc.)
// Provider priority (auto-detect):
// TTS_PROVIDER=google → Google Cloud TTS direct (HIPAA eligible)
// TTS_PROVIDER=litellm → LiteLLM proxy
// TTS_PROVIDER=elevenlabs → ElevenLabs (not HIPAA)
// (auto) LITELLM_API_BASE set → litellm
// (auto) ELEVENLABS_API_KEY set → elevenlabs
// Auto: google > litellm > elevenlabs
function getTTSProvider() {
var env = process.env.TTS_PROVIDER;
if (env === 'google') return 'google';
if (env === 'litellm') return 'litellm';
if (env === 'elevenlabs') return 'elevenlabs';
if (process.env.LITELLM_API_BASE) return 'litellm';
// Auto-detect: litellm > google > elevenlabs
// LiteLLM Vertex TTS works correctly via alias (tts-1)
if (process.env.LITELLM_API_BASE) return 'litellm';
if (isGoogleTTSConfigured()) return 'google';
if (process.env.ELEVENLABS_API_KEY) return 'elevenlabs';
return 'none';
}
var ttsProvider = getTTSProvider();
console.log('🔊 TTS provider:', ttsProvider);
console.log('🔊 TTS provider:', ttsProvider +
(ttsProvider === 'google' ? ' (voice: ' + (process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F') + ')' : ''));
router.post('/text-to-speech', authMiddleware, async (req, res) => {
try {
var text = (req.body.text || '').substring(0, 5000);
if (!text) return res.status(400).json({ error: 'No text provided' });
if (ttsProvider === 'google') {
var buffer = await synthesizeWithGoogleTTS(text);
res.set('Content-Type', 'audio/mpeg');
return res.send(buffer);
}
if (ttsProvider === 'litellm') {
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
var ttsModel = process.env.LITELLM_TTS_MODEL || 'vertex_ai/text-to-speech';
var base = process.env.LITELLM_API_BASE.replace(/\/+$/, '');
// Model must be the model_name alias from LiteLLM model_list (e.g. tts-1, tts-1-hd)
// NOT the underlying vertex_ai/... path — LiteLLM routes by alias
var ttsModel = process.env.LITELLM_TTS_MODEL || 'tts-1';
var ttsVoice = process.env.LITELLM_TTS_VOICE || 'en-US-Journey-F';
var base = process.env.LITELLM_API_BASE.replace(/\/+$/, '');
var ttsHeaders = { 'Content-Type': 'application/json' };
if (process.env.LITELLM_API_KEY) ttsHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
// Send only model + input — voice is configured inside LiteLLM per model
var ttsResp = await axios.post(base + '/v1/audio/speech', {
model: ttsModel,
input: text
}, {
headers: ttsHeaders,
responseType: 'arraybuffer',
timeout: 60000
});
var ttsResp = await axios.post(base + '/v1/audio/speech',
{ model: ttsModel, voice: ttsVoice, input: text },
{ headers: ttsHeaders, responseType: 'arraybuffer', timeout: 60000 }
);
res.set('Content-Type', 'audio/mpeg');
return res.send(Buffer.from(ttsResp.data));
}
if (ttsProvider === 'elevenlabs') {
if (!process.env.ELEVENLABS_API_KEY) return res.status(400).json({ error: 'ElevenLabs not configured' });
const response = await axios({
var resp = await axios({
method: 'POST',
url: 'https://api.elevenlabs.io/v1/text-to-speech/pNInz6obpgDQGcFmaJgB',
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY, 'Content-Type': 'application/json' },
@ -55,10 +64,10 @@ router.post('/text-to-speech', authMiddleware, async (req, res) => {
responseType: 'arraybuffer'
});
res.set('Content-Type', 'audio/mpeg');
return res.send(Buffer.from(response.data));
return res.send(Buffer.from(resp.data));
}
res.status(400).json({ error: 'TTS not configured. Set LITELLM_API_BASE or ELEVENLABS_API_KEY.' });
res.status(400).json({ error: 'TTS not configured. Set GOOGLE_VERTEX_PROJECT, LITELLM_API_BASE, or ELEVENLABS_API_KEY.' });
} catch (err) {
var detail = err.response && err.response.data
? JSON.stringify(err.response.data).substring(0, 500)

45
src/utils/transcribeGoogle.js Normal file
View file

@ -0,0 +1,45 @@
// ============================================================
// TRANSCRIBE GOOGLE — Vertex AI / Gemini audio transcription
// Sends audio inline to Gemini which returns spoken text.
// Supports: gemini-2.0-flash (default), gemini-2.5-flash
// Requires: GOOGLE_VERTEX_PROJECT, @google-cloud/vertexai installed
// ============================================================
async function transcribeWithGemini(buffer, mimeType) {
var VertexModule = require('@google-cloud/vertexai');
var project = process.env.GOOGLE_VERTEX_PROJECT;
var location = process.env.GOOGLE_VERTEX_LOCATION || 'us-central1';
if (!project) throw new Error('GOOGLE_VERTEX_PROJECT not set');
var vertex = new VertexModule.VertexAI({ project: project, location: location });
var model = vertex.getGenerativeModel({
model: process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash'
});
var base64 = buffer.toString('base64');
var safeMime = mimeType || 'audio/webm';
var result = await model.generateContent({
contents: [{
role: 'user',
parts: [
{ inlineData: { mimeType: safeMime, data: base64 } },
{ text: 'Transcribe this audio. Output the spoken words only, exactly as heard. No commentary, no formatting, no explanation.' }
]
}]
});
var text = '';
if (result.response && result.response.candidates && result.response.candidates[0]) {
var parts = result.response.candidates[0].content.parts || [];
parts.forEach(function(p) { if (p.text) text += p.text; });
}
return text.trim();
}
function isGoogleSTTConfigured() {
if (!process.env.GOOGLE_VERTEX_PROJECT) return false;
try { require('@google-cloud/vertexai'); return true; } catch(e) { return false; }
}
module.exports = { transcribeWithGemini, isGoogleSTTConfigured };

47
src/utils/ttsGoogle.js Normal file
View file

@ -0,0 +1,47 @@
// ============================================================
// TTS GOOGLE — Google Cloud Text-to-Speech (direct, no LiteLLM)
// Uses google-auth-library (transitive dep of @google-cloud/vertexai)
// Voices: en-US-Journey-F (female), en-US-Journey-D (male),
// en-US-Studio-O, en-US-Neural2-C, etc.
// Requires: GOOGLE_VERTEX_PROJECT, ADC or GOOGLE_APPLICATION_CREDENTIALS
// ============================================================
async function synthesizeWithGoogleTTS(text) {
var GoogleAuth = require('google-auth-library').GoogleAuth;
var axios = require('axios');
var auth = new GoogleAuth({
scopes: ['https://www.googleapis.com/auth/cloud-platform']
});
var client = await auth.getClient();
var tokenData = await client.getAccessToken();
var token = tokenData.token;
var voice = process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
var langCode = voice.substring(0, 5); // e.g. en-US
var resp = await axios.post(
'https://texttospeech.googleapis.com/v1/text:synthesize',
{
input: { text: text },
voice: { languageCode: langCode, name: voice },
audioConfig: { audioEncoding: 'MP3' }
},
{
headers: {
'Authorization': 'Bearer ' + token,
'Content-Type': 'application/json'
},
timeout: 30000
}
);
return Buffer.from(resp.data.audioContent, 'base64');
}
function isGoogleTTSConfigured() {
if (!process.env.GOOGLE_VERTEX_PROJECT) return false;
try { require('google-auth-library'); return true; } catch(e) { return false; }
}
module.exports = { synthesizeWithGoogleTTS, isGoogleTTSConfigured };