diff --git a/.env.example b/.env.example index 50dba5d..e8e9079 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,15 @@ OPENROUTER_API_KEY=sk-or-v1-your-key # GOOGLE_VERTEX_LOCATION=us-central1 # GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json # (Or use default credentials if running on GCE/GKE/Cloud Run) +# +# Google STT — Gemini inline audio (auto-detected when GOOGLE_VERTEX_PROJECT set) +# TRANSCRIBE_PROVIDER=google +# GOOGLE_STT_MODEL=gemini-2.0-flash # or gemini-2.5-flash for better accuracy +# +# Google TTS — Google Cloud Text-to-Speech (auto-detected when GOOGLE_VERTEX_PROJECT set) +# TTS_PROVIDER=google +# GOOGLE_TTS_VOICE=en-US-Journey-F # female | en-US-Journey-D = male +# Other options: en-US-Studio-O, en-US-Neural2-C, en-US-Neural2-J # Option 5: LiteLLM Proxy (self-hosted, routes to any provider) # AI_PROVIDER=litellm @@ -35,12 +44,13 @@ OPENROUTER_API_KEY=sk-or-v1-your-key # # LiteLLM Speech-to-Text (auto-enabled when LITELLM_API_BASE is set) # TRANSCRIBE_PROVIDER=litellm -# LITELLM_STT_MODEL=vertex_ai/chirp # or: vertex_ai/chirp_2 for newer model +# LiteLLM STT: NOT supported for Vertex AI Chirp via LiteLLM proxy. +# Use TRANSCRIBE_PROVIDER=google for Vertex AI / Gemini STT. # -# LiteLLM Text-to-Speech (auto-enabled when LITELLM_API_BASE is set) -# TTS_PROVIDER=litellm -# LITELLM_TTS_MODEL=vertex_ai/text-to-speech # voice is set inside LiteLLM per model -# LITELLM_TTS_VOICE is ignored for Vertex AI — voice is configured in LiteLLM model entry +# LiteLLM TTS: WORKS — use the model_name alias from your LiteLLM model_list +# TTS_PROVIDER=litellm (auto-detected when LITELLM_API_BASE set) +# LITELLM_TTS_MODEL=tts-1 # model_name alias in LiteLLM config +# LITELLM_TTS_VOICE=en-US-Journey-F # passed to LiteLLM, supports Google Cloud voice names # ============================================================ # TRANSCRIPTION (speech-to-text) diff --git a/src/routes/transcribe.js b/src/routes/transcribe.js index 209cbac..f8a93f7 100644 --- a/src/routes/transcribe.js +++ b/src/routes/transcribe.js @@ -4,32 +4,33 @@ const multer = require('multer'); const { whisperClient, litellmClient } = require('../utils/ai'); const { transcribeWithAWS, isAWSTranscribeConfigured } = require('../utils/transcribeAWS'); const { transcribeWithLocal, isLocalWhisperConfigured } = require('../utils/transcribeLocal'); +const { transcribeWithGemini, isGoogleSTTConfigured } = require('../utils/transcribeGoogle'); const { authMiddleware } = require('../middleware/auth'); const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } }); -// Determine which provider to use: -// TRANSCRIBE_PROVIDER=litellm → LiteLLM proxy (any Whisper-compatible model) -// TRANSCRIBE_PROVIDER=aws → Amazon Transcribe -// TRANSCRIBE_PROVIDER=local → local whisper.cpp / faster-whisper -// TRANSCRIBE_PROVIDER=openai → OpenAI Whisper directly -// (auto) LITELLM_API_BASE set → litellm -// (auto) AWS Bedrock region set → aws -// (default) → openai whisper +// Provider priority (auto-detect): +// TRANSCRIBE_PROVIDER=google → Vertex AI / Gemini (direct, HIPAA eligible) +// TRANSCRIBE_PROVIDER=aws → Amazon Transcribe (HIPAA eligible) +// TRANSCRIBE_PROVIDER=local → local whisper.cpp / faster-whisper +// TRANSCRIBE_PROVIDER=openai → OpenAI Whisper (direct) +// TRANSCRIBE_PROVIDER=litellm→ LiteLLM proxy (explicit only — audio routing bugs) +// Auto: google > aws > openai function getTranscribeProvider() { var env = process.env.TRANSCRIBE_PROVIDER; - if (env === 'litellm') return 'litellm'; - if (env === 'local') return 'local'; + if (env === 'google') return 'google'; if (env === 'aws') return 'aws'; + if (env === 'local') return 'local'; if (env === 'openai') return 'openai'; - // Auto-detect priority: AWS > LiteLLM > OpenAI - // LiteLLM's Vertex AI audio proxy has routing bugs — only use it if explicitly set + if (env === 'litellm') return 'litellm'; + // Auto-detect + if (isGoogleSTTConfigured()) return 'google'; if (isAWSTranscribeConfigured()) return 'aws'; - if (process.env.LITELLM_API_BASE && litellmClient) return 'litellm'; return 'openai'; } function isTranscribeAvailable() { + if (isGoogleSTTConfigured()) return true; if (isLocalWhisperConfigured()) return true; if (isAWSTranscribeConfigured()) return true; if (litellmClient) return true; @@ -38,11 +39,13 @@ function isTranscribeAvailable() { } var provider = getTranscribeProvider(); -var medical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true'; +var medical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true'; var available = isTranscribeAvailable(); -console.log('🎙️ Transcribe provider:', provider + (provider === 'aws' && medical ? ' (Medical)' : '') + (available ? '' : ' (NOT CONFIGURED — browser speech only)')); +console.log('🎙️ Transcribe provider:', provider + + (provider === 'aws' && medical ? ' (Medical)' : '') + + (provider === 'google' ? ' (model: ' + (process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash') + ')' : '') + + (available ? '' : ' (NOT CONFIGURED — browser speech only)')); -// Status endpoint — frontend checks this to decide whether to upload audio router.get('/transcribe/status', authMiddleware, (req, res) => { res.json({ available: available, provider: available ? provider : 'none' }); }); @@ -51,9 +54,15 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r try { if (!req.file) return res.status(400).json({ error: 'No audio' }); var startTime = Date.now(); - var fileSize = req.file.size; + var fileSize = req.file.size; console.log('[Transcribe] Received ' + (fileSize / 1024).toFixed(0) + 'KB audio (' + (req.file.mimetype || 'unknown') + ') via ' + provider); + if (provider === 'google') { + var text = await transcribeWithGemini(req.file.buffer, req.file.mimetype || 'audio/webm'); + console.log('[Transcribe] Google/Gemini done in ' + (Date.now() - startTime) + 'ms'); + return res.json({ success: true, text: text, provider: 'google-gemini', duration: Date.now() - startTime }); + } + if (provider === 'local') { var text = await transcribeWithLocal(req.file.buffer, req.file.mimetype || 'audio/webm'); console.log('[Transcribe] Local done in ' + (Date.now() - startTime) + 'ms'); @@ -61,9 +70,7 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r } if (provider === 'aws') { - if (!isAWSTranscribeConfigured()) { - return res.status(400).json({ error: 'AWS Transcribe not configured. Set AWS_BEDROCK_REGION.' }); - } + if (!isAWSTranscribeConfigured()) return res.status(400).json({ error: 'AWS Transcribe not configured.' }); var text = await transcribeWithAWS(req.file.buffer, req.file.mimetype || 'audio/webm'); console.log('[Transcribe] AWS done in ' + (Date.now() - startTime) + 'ms'); return res.json({ success: true, text: text, provider: 'aws-transcribe', duration: Date.now() - startTime }); @@ -73,22 +80,16 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' }); var sttModel = process.env.LITELLM_STT_MODEL || 'vertex_ai/chirp'; var FormData = require('form-data'); - var axios = require('axios'); + var axios = require('axios'); var form = new FormData(); - form.append('file', req.file.buffer, { - filename: 'audio.webm', - contentType: req.file.mimetype || 'audio/webm' - }); + form.append('file', req.file.buffer, { filename: 'audio.webm', contentType: req.file.mimetype || 'audio/webm' }); form.append('model', sttModel); - // Note: Vertex AI Chirp via LiteLLM does not support 'prompt' or 'response_format' - // Only send parameters Chirp actually accepts var sttHeaders = Object.assign({}, form.getHeaders()); if (process.env.LITELLM_API_KEY) sttHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY; - var base = process.env.LITELLM_API_BASE.replace(/\/+$/, ''); - var sttResp = await axios.post(base + '/v1/audio/transcriptions', form, { - headers: sttHeaders, - timeout: 120000 - }); + var sttResp = await require('axios').post( + process.env.LITELLM_API_BASE.replace(/\/+$/, '') + '/v1/audio/transcriptions', + form, { headers: sttHeaders, timeout: 120000 } + ); var text = (sttResp.data && sttResp.data.text) ? sttResp.data.text : String(sttResp.data || ''); console.log('[Transcribe] LiteLLM (' + sttModel + ') done in ' + (Date.now() - startTime) + 'ms'); return res.json({ success: true, text: text, provider: 'litellm-stt', duration: Date.now() - startTime }); @@ -96,7 +97,7 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r // OpenAI Whisper (direct) if (!whisperClient) return res.status(400).json({ error: 'Whisper not configured. Set OPENAI_API_KEY.' }); - var file = new File([req.file.buffer], 'audio.webm', { type: req.file.mimetype || 'audio/webm' }); + var file = new File([req.file.buffer], 'audio.webm', { type: req.file.mimetype || 'audio/webm' }); var result = await whisperClient.audio.transcriptions.create({ file, model: 'whisper-1', language: 'en', response_format: 'text', @@ -105,8 +106,8 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r var text = typeof result === 'string' ? result : result.text; console.log('[Transcribe] Whisper done in ' + (Date.now() - startTime) + 'ms'); res.json({ success: true, text: text, provider: 'openai-whisper', duration: Date.now() - startTime }); + } catch (err) { - // Log full LiteLLM/upstream error body so we can diagnose var detail = err.response && err.response.data ? JSON.stringify(err.response.data).substring(0, 500) : err.message; diff --git a/src/routes/tts.js b/src/routes/tts.js index 89e8e2e..17f61b4 100644 --- a/src/routes/tts.js +++ b/src/routes/tts.js @@ -1,53 +1,62 @@ const express = require('express'); -const router = express.Router(); -const axios = require('axios'); +const router = express.Router(); +const axios = require('axios'); +const { synthesizeWithGoogleTTS, isGoogleTTSConfigured } = require('../utils/ttsGoogle'); const { authMiddleware } = require('../middleware/auth'); -// TTS_PROVIDER=litellm → LiteLLM proxy (Vertex AI TTS, OpenAI TTS, etc.) +// Provider priority (auto-detect): +// TTS_PROVIDER=google → Google Cloud TTS direct (HIPAA eligible) +// TTS_PROVIDER=litellm → LiteLLM proxy // TTS_PROVIDER=elevenlabs → ElevenLabs (not HIPAA) -// (auto) LITELLM_API_BASE set → litellm -// (auto) ELEVENLABS_API_KEY set → elevenlabs +// Auto: google > litellm > elevenlabs function getTTSProvider() { var env = process.env.TTS_PROVIDER; + if (env === 'google') return 'google'; if (env === 'litellm') return 'litellm'; if (env === 'elevenlabs') return 'elevenlabs'; - if (process.env.LITELLM_API_BASE) return 'litellm'; + // Auto-detect: litellm > google > elevenlabs + // LiteLLM Vertex TTS works correctly via alias (tts-1) + if (process.env.LITELLM_API_BASE) return 'litellm'; + if (isGoogleTTSConfigured()) return 'google'; if (process.env.ELEVENLABS_API_KEY) return 'elevenlabs'; return 'none'; } var ttsProvider = getTTSProvider(); -console.log('🔊 TTS provider:', ttsProvider); +console.log('🔊 TTS provider:', ttsProvider + + (ttsProvider === 'google' ? ' (voice: ' + (process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F') + ')' : '')); router.post('/text-to-speech', authMiddleware, async (req, res) => { try { var text = (req.body.text || '').substring(0, 5000); if (!text) return res.status(400).json({ error: 'No text provided' }); + if (ttsProvider === 'google') { + var buffer = await synthesizeWithGoogleTTS(text); + res.set('Content-Type', 'audio/mpeg'); + return res.send(buffer); + } + if (ttsProvider === 'litellm') { if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' }); - var ttsModel = process.env.LITELLM_TTS_MODEL || 'vertex_ai/text-to-speech'; - var base = process.env.LITELLM_API_BASE.replace(/\/+$/, ''); + // Model must be the model_name alias from LiteLLM model_list (e.g. tts-1, tts-1-hd) + // NOT the underlying vertex_ai/... path — LiteLLM routes by alias + var ttsModel = process.env.LITELLM_TTS_MODEL || 'tts-1'; + var ttsVoice = process.env.LITELLM_TTS_VOICE || 'en-US-Journey-F'; + var base = process.env.LITELLM_API_BASE.replace(/\/+$/, ''); var ttsHeaders = { 'Content-Type': 'application/json' }; if (process.env.LITELLM_API_KEY) ttsHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY; - - // Send only model + input — voice is configured inside LiteLLM per model - var ttsResp = await axios.post(base + '/v1/audio/speech', { - model: ttsModel, - input: text - }, { - headers: ttsHeaders, - responseType: 'arraybuffer', - timeout: 60000 - }); - + var ttsResp = await axios.post(base + '/v1/audio/speech', + { model: ttsModel, voice: ttsVoice, input: text }, + { headers: ttsHeaders, responseType: 'arraybuffer', timeout: 60000 } + ); res.set('Content-Type', 'audio/mpeg'); return res.send(Buffer.from(ttsResp.data)); } if (ttsProvider === 'elevenlabs') { if (!process.env.ELEVENLABS_API_KEY) return res.status(400).json({ error: 'ElevenLabs not configured' }); - const response = await axios({ + var resp = await axios({ method: 'POST', url: 'https://api.elevenlabs.io/v1/text-to-speech/pNInz6obpgDQGcFmaJgB', headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY, 'Content-Type': 'application/json' }, @@ -55,10 +64,10 @@ router.post('/text-to-speech', authMiddleware, async (req, res) => { responseType: 'arraybuffer' }); res.set('Content-Type', 'audio/mpeg'); - return res.send(Buffer.from(response.data)); + return res.send(Buffer.from(resp.data)); } - res.status(400).json({ error: 'TTS not configured. Set LITELLM_API_BASE or ELEVENLABS_API_KEY.' }); + res.status(400).json({ error: 'TTS not configured. Set GOOGLE_VERTEX_PROJECT, LITELLM_API_BASE, or ELEVENLABS_API_KEY.' }); } catch (err) { var detail = err.response && err.response.data ? JSON.stringify(err.response.data).substring(0, 500) diff --git a/src/utils/transcribeGoogle.js b/src/utils/transcribeGoogle.js new file mode 100644 index 0000000..5d9319e --- /dev/null +++ b/src/utils/transcribeGoogle.js @@ -0,0 +1,45 @@ +// ============================================================ +// TRANSCRIBE GOOGLE — Vertex AI / Gemini audio transcription +// Sends audio inline to Gemini which returns spoken text. +// Supports: gemini-2.0-flash (default), gemini-2.5-flash +// Requires: GOOGLE_VERTEX_PROJECT, @google-cloud/vertexai installed +// ============================================================ + +async function transcribeWithGemini(buffer, mimeType) { + var VertexModule = require('@google-cloud/vertexai'); + var project = process.env.GOOGLE_VERTEX_PROJECT; + var location = process.env.GOOGLE_VERTEX_LOCATION || 'us-central1'; + if (!project) throw new Error('GOOGLE_VERTEX_PROJECT not set'); + + var vertex = new VertexModule.VertexAI({ project: project, location: location }); + var model = vertex.getGenerativeModel({ + model: process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash' + }); + + var base64 = buffer.toString('base64'); + var safeMime = mimeType || 'audio/webm'; + + var result = await model.generateContent({ + contents: [{ + role: 'user', + parts: [ + { inlineData: { mimeType: safeMime, data: base64 } }, + { text: 'Transcribe this audio. Output the spoken words only, exactly as heard. No commentary, no formatting, no explanation.' } + ] + }] + }); + + var text = ''; + if (result.response && result.response.candidates && result.response.candidates[0]) { + var parts = result.response.candidates[0].content.parts || []; + parts.forEach(function(p) { if (p.text) text += p.text; }); + } + return text.trim(); +} + +function isGoogleSTTConfigured() { + if (!process.env.GOOGLE_VERTEX_PROJECT) return false; + try { require('@google-cloud/vertexai'); return true; } catch(e) { return false; } +} + +module.exports = { transcribeWithGemini, isGoogleSTTConfigured }; diff --git a/src/utils/ttsGoogle.js b/src/utils/ttsGoogle.js new file mode 100644 index 0000000..cd0e99f --- /dev/null +++ b/src/utils/ttsGoogle.js @@ -0,0 +1,47 @@ +// ============================================================ +// TTS GOOGLE — Google Cloud Text-to-Speech (direct, no LiteLLM) +// Uses google-auth-library (transitive dep of @google-cloud/vertexai) +// Voices: en-US-Journey-F (female), en-US-Journey-D (male), +// en-US-Studio-O, en-US-Neural2-C, etc. +// Requires: GOOGLE_VERTEX_PROJECT, ADC or GOOGLE_APPLICATION_CREDENTIALS +// ============================================================ + +async function synthesizeWithGoogleTTS(text) { + var GoogleAuth = require('google-auth-library').GoogleAuth; + var axios = require('axios'); + + var auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] + }); + + var client = await auth.getClient(); + var tokenData = await client.getAccessToken(); + var token = tokenData.token; + var voice = process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F'; + var langCode = voice.substring(0, 5); // e.g. en-US + + var resp = await axios.post( + 'https://texttospeech.googleapis.com/v1/text:synthesize', + { + input: { text: text }, + voice: { languageCode: langCode, name: voice }, + audioConfig: { audioEncoding: 'MP3' } + }, + { + headers: { + 'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/json' + }, + timeout: 30000 + } + ); + + return Buffer.from(resp.data.audioContent, 'base64'); +} + +function isGoogleTTSConfigured() { + if (!process.env.GOOGLE_VERTEX_PROJECT) return false; + try { require('google-auth-library'); return true; } catch(e) { return false; } +} + +module.exports = { synthesizeWithGoogleTTS, isGoogleTTSConfigured };