simplify speech and embeddings through litellm

This commit is contained in:
Daniel 2026-05-09 05:09:02 +02:00
parent 79037fa775
commit 1b3ea569b7
24 changed files with 184 additions and 1242 deletions

View file

@ -15,6 +15,7 @@ services:
TTS_PROVIDER: litellm
LITELLM_TTS_MODEL: local-kokoro-tts
LITELLM_TTS_VOICE: sherpa/kokoro:am_adam
LITELLM_TTS_VOICES: sherpa/kokoro:am_adam,sherpa/kokoro:am_michael,sherpa/kokoro:af_bella,sherpa/kokoro:af_nicole,sherpa/kokoro:bf_emma,sherpa/kokoro:bm_lewis
volumes:
- scribe-logs:/app/data/logs
- clinical-assistant-mcp-data:/app/mcp-data:ro

View file

@ -591,7 +591,7 @@ Check whether speech-to-text transcription is available and which provider is co
```json
{
"available": true,
"provider": "google | aws | litellm | openai | local | none"
"provider": "litellm | none"
}
```

View file

@ -48,8 +48,8 @@ src/
logger.js # audit/api/access + Loki shipper
errors.js # generic 500 responder
models.js, prompts.js, ai.js # AI provider + model + prompt management
embeddings.js # Vertex / LiteLLM / OpenAI embeddings
transcribe*.js, tts*.js # STT / TTS provider clients
embeddings.js # LiteLLM embeddings
transcribe.js, tts.js # LiteLLM STT / TTS routes
routes/ # Express routers (auth, hpi, soap, patient education, …)
public/ # SPA
@ -163,4 +163,4 @@ The clinical assistant can call an external MCP-backed retrieval service. Ped-AI
## Speech
Browser Whisper and browser-local Whisper model downloads are removed from runtime. Speech-to-text routes through configured server-side providers. Browser-native Web Speech remains available only when explicitly enabled by user settings and browser support.
Browser Whisper and browser-local Whisper model downloads are removed from runtime. Speech-to-text routes through LiteLLM; upstream provider choice belongs in LiteLLM config. Browser-native Web Speech remains available only when explicitly enabled by user settings and browser support.

View file

@ -31,38 +31,31 @@ keys):
|---|---|
| `AI_PROVIDER` | `openrouter` / `bedrock` / `azure` / `vertex` / `litellm`. If unset, the startup loader uses configured credentials and the last initialized provider in Bedrock → Azure → Vertex → LiteLLM order wins; otherwise OpenRouter is the default. |
| `OPENROUTER_API_KEY` | OpenRouter key (not HIPAA-eligible). |
| `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | Bedrock / Transcribe / Transcribe-Medical. |
| `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | Bedrock chat provider. |
| `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY`, `AZURE_DEPLOYMENT_NAME`, `AZURE_OPENAI_API_VERSION` | Azure OpenAI. |
| `GOOGLE_VERTEX_PROJECT`, `GOOGLE_VERTEX_LOCATION`, `GOOGLE_APPLICATION_CREDENTIALS` | Vertex AI + Gemini chat/STT/TTS. |
| `GOOGLE_VERTEX_PROJECT`, `GOOGLE_VERTEX_LOCATION`, `GOOGLE_APPLICATION_CREDENTIALS` | Vertex AI chat provider. |
| `LITELLM_API_BASE`, `LITELLM_API_KEY` | OpenAI-compatible AI gateway (Bifrost, LiteLLM, or similar). |
### Speech-to-text
| Variable | Purpose |
|---|---|
| `TRANSCRIBE_PROVIDER` | `google`, `aws`, `local`, `openai`, `litellm`. Auto-detects if unset. |
| `OPENAI_API_KEY` | OpenAI Whisper. |
| `GOOGLE_STT_MODEL` | Gemini model used as STT (default `gemini-2.0-flash`). |
| `AWS_TRANSCRIBE_MEDICAL` | `true` enables Transcribe Medical. |
| `AWS_TRANSCRIBE_SPECIALTY` | `PRIMARYCARE` / `CARDIOLOGY` / `NEUROLOGY` / `ONCOLOGY` / `RADIOLOGY` / `UROLOGY`. |
| `WHISPER_BINARY`, `WHISPER_MODEL_SIZE`, `WHISPER_MODEL_PATH`, `WHISPER_LANGUAGE`, `WHISPER_THREADS` | Local whisper.cpp / faster-whisper. |
| `TRANSCRIBE_PROVIDER` | Use `litellm`; auto mode uses LiteLLM when configured. |
| `LITELLM_STT_MODEL` | Model name for LiteLLM-routed STT. |
### Text-to-speech
| Variable | Purpose |
|---|---|
| `GOOGLE_TTS_VOICE` | Google Cloud TTS voice (e.g. `en-US-Journey-F`). |
| `ELEVENLABS_API_KEY` | ElevenLabs (not HIPAA-compliant). |
| `LITELLM_TTS_MODEL`, `LITELLM_TTS_VOICE` | LiteLLM-routed TTS. |
| `LITELLM_TTS_MODEL`, `LITELLM_TTS_VOICE` | LiteLLM-routed TTS model and default voice. |
| `LITELLM_TTS_VOICES` | Comma-separated LiteLLM-compatible voices exposed in voice search and user preferences. |
### Embeddings
| Variable | Purpose |
|---|---|
| `EMBEDDING_MODEL` | Embedding model name (default `vertex_ai/text-embedding-005`). |
| `EMBEDDING_DIMENSIONS` | Vector dimensions (default 768). |
| `VERTEX_PROJECT`, `GOOGLE_CLOUD_PROJECT` | Direct Vertex embedding project. Embedding utility also works through LiteLLM when `LITELLM_API_BASE` is set. |
| `EMBEDDING_MODEL` | LiteLLM embedding model name (default `openai-text-embedding-3-large`). |
| `EMBEDDING_DIMENSIONS` | Vector dimensions (default 3072). |
### Email (SMTP)

View file

@ -37,9 +37,9 @@ src/
fileType.js magic-byte upload verifier
errors.js generic 500 responder
logger.js audit + api + access + Loki shipper
embeddings.js Vertex / LiteLLM / OpenAI embeddings
embeddings.js LiteLLM embeddings
notify.js ntfy push
transcribe*.js, tts*.js STT / TTS provider clients
transcribe.js, tts.js LiteLLM STT / TTS routes
routes/ Express routers for auth, AI workflows, education, logs, and user data
public/
@ -272,8 +272,8 @@ docker exec -w /app pediatric-ai-scribe npm run migrate:new -- add_my_table
| `sickVisit.js` | `/api` | Auth | Sick visit |
| `milestones.js` | `/api` | Auth | Developmental milestone narratives |
| `refine.js` | `/api` | Auth | Refine / shorten / clarify |
| `transcribe.js` | `/api` | Auth | STT (5 providers) |
| `tts.js` | `/api` | Auth | TTS (3 providers) |
| `transcribe.js` | `/api` | Auth | LiteLLM STT |
| `tts.js` | `/api` | Auth | LiteLLM TTS |
| `encounters.js` | `/api` | Auth | Save / load / optimistic-lock encounters |
| `memories.js` | `/api` | Auth | Templates + prompt preferences |
| `audioBackups.js` | `/api` | Auth | Encrypted audio retry store |

View file

@ -10,7 +10,7 @@ This guide explains how to set up and use the new vector-based semantic search f
- **Semantic** (`/api/learning/search/semantic`) - AI-powered vector similarity
- **Hybrid** (`/api/learning/search/hybrid`) - Combines both for best results
- **Auto-embedding** - Content is automatically vectorized when created/updated
- **HIPAA-compliant** - Uses Vertex AI embeddings (BAA available)
- **Gateway-routed** - Uses LiteLLM embeddings so provider policy stays in one place
## Prerequisites
@ -37,37 +37,22 @@ postgres:
# ... rest of your config
```
### 2. Configure Embedding Provider
### 2. Configure LiteLLM Embeddings
Add to your `.env` file:
```bash
# Option 1: Vertex AI (HIPAA-eligible, recommended)
EMBEDDING_MODEL=vertex_ai/text-embedding-005
EMBEDDING_DIMENSIONS=768
VERTEX_PROJECT=your-gcp-project-id
VERTEX_LOCATION=us-central1
GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
# Option 2: LiteLLM Proxy (routes to any provider)
LITELLM_API_BASE=http://localhost:4000
LITELLM_API_KEY=your-key
EMBEDDING_MODEL=text-embedding-005 # LiteLLM will route to configured provider
# Option 3: OpenAI (NOT HIPAA-eligible, fallback only)
OPENAI_API_KEY=sk-your-key
# Uses text-embedding-3-small and truncates to the 768-dimension column
EMBEDDING_MODEL=openai-text-embedding-3-large
EMBEDDING_DIMENSIONS=3072
```
## Available Vertex AI Embedding Models
## Available Embedding Models
Tested and working via LiteLLM:
The Admin embedding search reads LiteLLM `/model/info` and only shows models with `model_info.mode = "embedding"`. Do not add app-side built-in Vertex/OpenAI embedding lists; configure those choices in LiteLLM.
| Model | Dimensions | Use Case | HIPAA |
|-------|-----------|----------|-------|
| **vertex_ai/text-embedding-005** | 768 | English + code (recommended) | Yes |
| **vertex_ai/gemini-embedding-001** | 768-3072 | Multilingual + code, best quality | Yes |
| **vertex_ai/text-multilingual-embedding-002** | 768 | Multilingual focus | Yes |
The local LiteLLM instance currently exposes examples such as `openai-text-embedding-3-large`, `openai-text-embedding-3-small`, and Mistral embedding models. Dimensions are read from LiteLLM metadata when available.
## Setup Steps
@ -113,8 +98,8 @@ Response:
"total": 50,
"withEmbeddings": 50,
"missing": 0,
"model": "vertex_ai/text-embedding-005",
"dimensions": 768
"model": "openai-text-embedding-3-large",
"dimensions": 3072
}
```
@ -148,8 +133,8 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
1. **Content Creation/Update**:
- Text is extracted from `title`, `subject`, and `body` (HTML stripped)
- Sent to embedding model (Vertex AI)
- Returns 768-dimensional vector
- Sent to the configured LiteLLM embedding model
- Returns an embedding vector
- Stored in `learning_content.embedding` column
2. **Semantic Search**:
@ -166,11 +151,7 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
## Cost Estimate
**Google Vertex AI pricing:**
- text-embedding-005: $0.025 per 1M characters
- Average article: 10,000 chars = $0.00025
- 1,000 articles: ~**$0.25 one-time**
- Search queries: ~$0.0000125 per query
Embedding cost depends on the upstream configured in LiteLLM.
## Troubleshooting
@ -179,14 +160,12 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
- For Docker: Use `pgvector/pgvector:pg16` image
### "Embeddings not configured"
- Verify `.env` has `VERTEX_PROJECT` / `GOOGLE_CLOUD_PROJECT`, `LITELLM_API_BASE`, or `OPENAI_API_KEY`
- Check service account credentials: `GOOGLE_APPLICATION_CREDENTIALS`
- Verify `.env` has `LITELLM_API_BASE`
- Test: `curl http://localhost:3000/api/admin/learning/embeddings/status`
### "Embedding generation failed"
- Check logs for API errors
- Verify Vertex AI API is enabled in GCP
- Verify service account has `aiplatform.endpoints.predict` permission
- Verify LiteLLM `/model/info` shows the selected model with `mode: embedding`
- Check content isn't empty (skips empty bodies)
### "No results from semantic search"
@ -196,7 +175,7 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
## Performance
- **Embedding generation**: ~500ms per article (Vertex AI)
- **Embedding generation**: latency depends on the LiteLLM upstream
- **Search latency**:
- Keyword: 10-50ms
- Semantic: 20-100ms (with IVFFLAT index)
@ -205,7 +184,7 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
## Security And Compliance
- **HIPAA-eligible**: Vertex AI supports BAA (Business Associate Agreement)
- **Compliance**: controlled by the upstream provider configured in LiteLLM
- **Data retention**: Embeddings stored in your database only
- **No PHI**: Only article content (not patient data) is embedded
- **Encryption**: TLS in transit, at-rest encryption via PostgreSQL

View file

@ -16,7 +16,7 @@ The bedside tools include a per-user phone extension and pager directory. Entrie
## Speech
Final transcription is server-side. Configure Google/Gemini, AWS Transcribe, LiteLLM, or OpenAI Whisper according to your deployment requirements.
Final transcription is server-side through LiteLLM. Configure upstream STT providers in LiteLLM rather than in Ped-AI.
Browser-native Web Speech is only an explicit opt-in preview path. It is not the final clinical transcript and may use browser-vendor cloud services.
@ -24,7 +24,7 @@ Browser Whisper and browser-local model workers are removed. Do not expect a pre
## Text To Speech
The voice preview button calls the configured TTS provider and plays the returned audio in the browser. If preview is silent, check that a voice is selected, a provider is configured, the user is authenticated, and browser autoplay has not blocked playback.
The voice preview button calls LiteLLM TTS and plays the returned audio in the browser. If preview is silent, check that a LiteLLM voice is selected, the gateway is configured, the user is authenticated, and browser autoplay has not blocked playback.
## Learning Hub

View file

@ -54,11 +54,10 @@ as low-priority style/template context. `custom` memories and legacy
`POST /api/transcribe` accepts one audio file up to 25 MB. Provider selection:
- explicit `TRANSCRIBE_PROVIDER=google|aws|local|openai|litellm`, or
- auto mode: Google/Gemini, then AWS Transcribe, then direct OpenAI Whisper.
- explicit `TRANSCRIBE_PROVIDER=litellm`, or
- auto mode when `LITELLM_API_BASE` is configured.
LiteLLM and local STT are explicit choices. Browser Whisper/browser-local model
downloads are not part of the runtime.
Direct Google, AWS, local Whisper, and OpenAI Whisper branches are not part of the runtime. Browser Whisper/browser-local model downloads are also absent.
## Browser Web Speech
@ -75,9 +74,9 @@ store the failed recording.
## Text-To-Speech
`POST /api/text-to-speech` returns `audio/mpeg` and marks the provider in
`X-TTS-Provider`. Supported providers include Google Cloud TTS, LiteLLM-routed
TTS, and ElevenLabs when configured.
`POST /api/text-to-speech` returns audio from LiteLLM and marks the LiteLLM
model in `X-TTS-Provider`. Voices are LiteLLM-compatible strings configured by
`LITELLM_TTS_VOICES`.
## Post-Note Helpers

View file

@ -2,16 +2,13 @@
## Transcription
`POST /api/transcribe` accepts `multipart/form-data` with one audio file up to 25 MB. The provider is selected by `TRANSCRIBE_PROVIDER`, or auto-detected from available credentials.
`POST /api/transcribe` accepts `multipart/form-data` with one audio file up to 25 MB. Server STT is routed through LiteLLM.
Provider priority in auto mode is Google/Gemini, then AWS Transcribe, then direct OpenAI Whisper. Set `TRANSCRIBE_PROVIDER=litellm` or `TRANSCRIBE_PROVIDER=local` explicitly for those paths.
Set `TRANSCRIBE_PROVIDER=litellm`, `LITELLM_API_BASE`, and `LITELLM_STT_MODEL`. Auto mode also uses LiteLLM when the gateway is configured.
| Provider | Notes | HIPAA posture |
|---|---|---|
| Google/Gemini | Uses the configured Vertex/Gemini STT model. | Eligible with the correct Google Cloud agreement. |
| AWS Transcribe | Supports standard and Medical mode. | Eligible with the correct AWS agreement. |
| LiteLLM | Sends audio through the configured LiteLLM `/audio/transcriptions` backend. | Depends on the selected upstream. |
| OpenAI Whisper | Uses `whisper-1` directly when `OPENAI_API_KEY` is configured. | Not HIPAA eligible unless your own agreement says otherwise. |
Browser Whisper and browser-local Whisper workers are not part of the runtime. Do not add browser model downloads or Transformers.js STT back into the public app.
@ -21,13 +18,13 @@ Browser-native Web Speech can show interim text when the user explicitly enables
## Text To Speech
`POST /api/text-to-speech` returns `audio/mpeg`. The `X-TTS-Provider` response header identifies the provider used. Requests are limited to 5000 characters.
`POST /api/text-to-speech` returns audio from LiteLLM `/audio/speech`. The `X-TTS-Provider` response header identifies the LiteLLM model used. Requests are limited to 5000 characters.
| Provider | Notes |
|---|---|
| Google Cloud TTS | Uses Google Cloud voices when configured. |
| LiteLLM | Uses `LITELLM_TTS_MODEL` and `LITELLM_TTS_VOICE`. |
| ElevenLabs | Available when configured; not HIPAA eligible by default. |
The admin/user voice pickers read available LiteLLM-compatible voices from `LITELLM_TTS_VOICES`.
## Audio Backup

View file

@ -1,21 +1,17 @@
# Transcription Options
Ped-AI currently supports server-side transcription plus an explicit browser Web Speech preview option. Browser Whisper was removed and should not be offered in settings, documentation, public workers, or model download scripts.
Ped-AI currently supports server-side transcription through LiteLLM plus an explicit browser Web Speech preview option. Browser Whisper was removed and should not be offered in settings, documentation, public workers, or model download scripts.
## Recommended Clinical Setup
Use a server-side provider covered by your compliance requirements.
Route STT through LiteLLM and configure the compliant upstream in LiteLLM.
| Need | Recommended provider |
|---|---|
| HIPAA-eligible cloud STT | Google/Gemini through Vertex AI or AWS Transcribe with a BAA. |
| OpenAI-compatible routing | LiteLLM with a compliant upstream. |
| Direct OpenAI Whisper | Only when acceptable for your deployment. |
| Server STT | LiteLLM with a compliant upstream. |
| Real-time draft preview | Browser Web Speech only with explicit user opt-in and privacy warning. |
Auto-detect mode currently checks Google/Gemini first, then AWS Transcribe,
then direct OpenAI Whisper. Use `TRANSCRIBE_PROVIDER=litellm` or
`TRANSCRIBE_PROVIDER=local` when you want those providers.
Auto-detect uses LiteLLM when `LITELLM_API_BASE` is configured. Direct Google, AWS, local Whisper, and OpenAI Whisper branches are not part of the app runtime.
## Configuration
@ -23,26 +19,7 @@ then direct OpenAI Whisper. Use `TRANSCRIBE_PROVIDER=litellm` or
TRANSCRIBE_PROVIDER=litellm
LITELLM_API_BASE=https://your-litellm.example/v1
LITELLM_API_KEY=<key>
LITELLM_STT_MODEL=whisper-1
```
Other provider examples:
```env
# Google/Gemini
TRANSCRIBE_PROVIDER=google
GOOGLE_VERTEX_PROJECT=your-project-id
GOOGLE_STT_MODEL=gemini-2.0-flash
# AWS Transcribe
TRANSCRIBE_PROVIDER=aws
AWS_BEDROCK_REGION=us-east-1
AWS_TRANSCRIBE_MEDICAL=true
AWS_TRANSCRIBE_SPECIALTY=PRIMARYCARE
# Direct OpenAI Whisper
TRANSCRIBE_PROVIDER=openai
OPENAI_API_KEY=<key>
LITELLM_STT_MODEL=local-parakeet-v3
```
## Failure Handling

View file

@ -9,7 +9,7 @@ var { authMiddleware, adminMiddleware } = require('../middleware/auth');
var PROMPTS = require('../utils/prompts');
var logger = require('../utils/logger');
var { gatewayUrl } = require('../utils/errors');
var { GOOGLE_TTS_VOICES, getTTSEnvProvider, getLiteLLMTTSDiscoveryItems, getLiteLLMTTSModels, getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider');
var { getTTSEnvProvider, getLiteLLMTTSDiscoveryItems, getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider');
var { getLiteLLMHeaders } = require('../utils/litellm');
var { getSTTDependencies, getLiteLLMSTTModels, getSTTModelLists, getSTTProvider } = require('../utils/sttProvider');
var { getLiteLLMEmbeddingModels } = require('../utils/embeddings');
@ -461,12 +461,11 @@ router.post('/config/image-models/test', async function(req, res) {
// ── GET TTS provider status, voice list, and DB overrides ────────────────
router.get('/config/tts', async function(req, res) {
try {
var { isGoogleTTSConfigured } = require('../utils/ttsGoogle');
var envProvider = getTTSEnvProvider();
var activeProvider = getTTSProvider();
var dbVoice = await db.getSetting('tts.voice') || '';
var dbModel = await db.getSetting('tts.model') || '';
var envVoice = process.env.GOOGLE_TTS_VOICE || process.env.LITELLM_TTS_VOICE || '';
var envVoice = process.env.LITELLM_TTS_VOICE || '';
var envModel = process.env.LITELLM_TTS_MODEL || '';
res.json({
success: true,
@ -479,9 +478,7 @@ router.get('/config/tts', async function(req, res) {
envVoice: envVoice,
envModel: envModel,
configured: {
google: false,
litellm: !!process.env.LITELLM_API_BASE,
elevenlabs: false
litellm: !!process.env.LITELLM_API_BASE
},
voices: getTTSVoiceLists()
});
@ -497,28 +494,6 @@ router.get('/config/tts/discover', async function(req, res) {
var provider = getTTSProvider();
if (provider === 'google') {
try {
var GoogleAuth = require('google-auth-library').GoogleAuth;
var auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] });
var client = await auth.getClient();
var tokenData = await client.getAccessToken();
var gResp = await axios.get('https://texttospeech.googleapis.com/v1/voices?languageCode=en', {
headers: { 'Authorization': 'Bearer ' + tokenData.token }, timeout: 10000
});
if (gResp.data && gResp.data.voices) {
gResp.data.voices.forEach(function(v) {
discovered.push({ id: v.name, name: v.name + ' (' + (v.ssmlGender || 'NEUTRAL') + ')', gender: v.ssmlGender, source: 'google-api' });
});
}
} catch (e) {
logger.warn('Google TTS voice API failed: ' + e.message + ' — using built-in list');
GOOGLE_TTS_VOICES.forEach(function(v) {
discovered.push({ id: v, name: v, source: 'google-builtin' });
});
}
}
if (provider === 'litellm' && process.env.LITELLM_API_BASE) {
var dbVoice = await db.getSetting('tts.voice') || '';
var dbModel = await db.getSetting('tts.model') || '';
@ -535,19 +510,6 @@ router.get('/config/tts/discover', async function(req, res) {
});
}
if (provider === 'elevenlabs' && process.env.ELEVENLABS_API_KEY) {
try {
var elResp = await axios.get('https://api.elevenlabs.io/v1/voices', {
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY }, timeout: 10000
});
if (elResp.data && elResp.data.voices) {
elResp.data.voices.forEach(function(v) {
discovered.push({ id: v.voice_id, name: v.name, source: 'elevenlabs-api' });
});
}
} catch (e) { logger.warn('ElevenLabs voice list failed: ' + e.message); }
}
if (search) {
discovered = discovered.filter(function(d) {
return d.id.toLowerCase().indexOf(search) !== -1 || d.name.toLowerCase().indexOf(search) !== -1;
@ -562,53 +524,24 @@ router.post('/config/tts/test', async function(req, res) {
try {
var text = ((req.body.text || 'Hello, this is a TTS test for Pediatric AI Scribe.')).substring(0, 500);
var voice = req.body.voice;
var { synthesizeWithGoogleTTS } = require('../utils/ttsGoogle');
var axios = require('axios');
var provider = getTTSProvider();
if (provider === 'none') return res.json({ success: false, error: 'No TTS provider configured' });
if (provider !== 'litellm') return res.json({ success: false, error: 'TTS is configured for LiteLLM only' });
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
var buffer, usedVoice = voice;
if (provider === 'google') {
usedVoice = voice || process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
buffer = await synthesizeWithGoogleTTS(text, usedVoice);
} else if (provider === 'litellm') {
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
var db = require('../db/database');
var adminModel = await db.getSetting('tts.model') || '';
var adminVoice = await db.getSetting('tts.voice') || '';
if (!adminModel && adminVoice) {
try {
var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, '');
var modelInfoResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 });
if (getLiteLLMTTSModels(modelInfoResp.data && modelInfoResp.data.data).indexOf(adminVoice) !== -1) {
adminModel = adminVoice;
adminVoice = '';
}
} catch (e) { logger.warn('LiteLLM TTS mode lookup failed: ' + e.message); }
}
var resolvedVoice = voice || adminVoice || '';
var rawModel = adminModel || process.env.LITELLM_TTS_MODEL || 'tts-1';
var ttsModel = rawModel;
usedVoice = resolvedVoice || process.env.LITELLM_TTS_VOICE || 'alloy';
// base URL handled by gatewayUrl() helper
var ttsResp = await axios.post(gatewayUrl('/audio/speech'),
{ model: ttsModel, voice: usedVoice, input: text },
{ headers: getLiteLLMHeaders('application/json'), responseType: 'arraybuffer', timeout: 60000 }
);
buffer = Buffer.from(ttsResp.data);
} else if (provider === 'elevenlabs') {
if (!process.env.ELEVENLABS_API_KEY) return res.json({ success: false, error: 'ElevenLabs not configured' });
usedVoice = voice || 'pNInz6obpgDQGcFmaJgB';
var elResp = await axios({
method: 'POST',
url: 'https://api.elevenlabs.io/v1/text-to-speech/' + usedVoice,
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY, 'Content-Type': 'application/json' },
data: { text: text, model_id: 'eleven_turbo_v2_5', voice_settings: { stability: 0.5, similarity_boost: 0.75 } },
responseType: 'arraybuffer'
});
buffer = Buffer.from(elResp.data);
}
var adminModel = await db.getSetting('tts.model') || '';
var adminVoice = await db.getSetting('tts.voice') || '';
var ttsModel = adminModel || process.env.LITELLM_TTS_MODEL || '';
var usedVoice = voice || adminVoice || process.env.LITELLM_TTS_VOICE || '';
if (!ttsModel) return res.json({ success: false, error: 'No LiteLLM TTS model configured' });
var ttsResp = await axios.post(gatewayUrl('/audio/speech'),
{ model: ttsModel, voice: usedVoice, input: text },
{ headers: getLiteLLMHeaders('application/json'), responseType: 'arraybuffer', timeout: 60000 }
);
var buffer = Buffer.from(ttsResp.data);
res.json({ success: true, audio: buffer.toString('base64'), provider: provider, voice: usedVoice });
} catch (e) {
var detail = e.response && e.response.data
@ -625,7 +558,7 @@ router.get('/config/stt', async function(req, res) {
var configured = getSTTDependencies();
var activeProvider = getSTTProvider(configured);
var dbModel = await db.getSetting('stt.model') || '';
var envModel = process.env.GOOGLE_STT_MODEL || process.env.LITELLM_STT_MODEL || '';
var envModel = process.env.LITELLM_STT_MODEL || '';
res.json({
success: true,
provider: activeProvider,
@ -644,36 +577,10 @@ router.get('/config/stt/discover', async function(req, res) {
try {
var search = (req.query.q || '').toLowerCase().trim();
var axios = require('axios');
var { whisperClient } = require('../utils/ai');
var discovered = [];
var provider = getSTTProvider();
if (provider === 'google') {
['gemini-2.5-flash','gemini-2.0-flash','gemini-2.0-flash-lite','gemini-1.5-pro','gemini-1.5-flash'].forEach(function(m) {
discovered.push({ id: m, name: m, source: 'google-builtin' });
});
}
if (provider === 'aws') {
['standard','medical'].forEach(function(m) {
discovered.push({ id: m, name: 'AWS Transcribe ' + m.charAt(0).toUpperCase() + m.slice(1), source: 'aws-builtin' });
});
}
if (provider === 'openai' && whisperClient) {
try {
var oModels = await whisperClient.models.list();
oModels.data.forEach(function(m) {
if (m.id.toLowerCase().indexOf('whisper') !== -1 || m.id.toLowerCase().indexOf('transcri') !== -1) {
discovered.push({ id: m.id, name: m.id, source: 'openai-api' });
}
});
} catch(e) {
discovered.push({ id: 'whisper-1', name: 'whisper-1', source: 'openai-builtin' });
}
}
if (provider === 'litellm' && process.env.LITELLM_API_BASE) {
try {
var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, '');
@ -689,14 +596,6 @@ router.get('/config/stt/discover', async function(req, res) {
}
}
if (provider === 'local') {
var localModel = process.env.WHISPER_MODEL_SIZE || 'base';
discovered.push({ id: localModel, name: 'Local Whisper: ' + localModel + ' (configured)', source: 'local-env' });
['tiny','base','small','medium','large'].forEach(function(m) {
if (m !== localModel) discovered.push({ id: m, name: 'Local Whisper: ' + m, source: 'local-builtin' });
});
}
if (search) {
discovered = discovered.filter(function(d) {
return d.id.toLowerCase().indexOf(search) !== -1 || d.name.toLowerCase().indexOf(search) !== -1;
@ -714,68 +613,27 @@ router.post('/config/stt/test', async function(req, res) {
var audioBuffer = Buffer.from(audioBase64, 'base64');
var mime = mimeType || 'audio/webm';
var { transcribeWithGemini } = require('../utils/transcribeGoogle');
var { transcribeWithAWS } = require('../utils/transcribeAWS');
var { transcribeWithLocal } = require('../utils/transcribeLocal');
var { whisperClient } = require('../utils/ai');
var axios = require('axios');
var provider = getSTTProvider();
if (provider === 'none') return res.json({ success: false, error: 'No STT provider configured' });
if (provider !== 'litellm') return res.json({ success: false, error: 'STT is configured for LiteLLM only' });
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
var start = Date.now();
var text = '';
if (provider === 'google') {
var model = process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash';
text = await transcribeWithGemini(audioBuffer, mime, model);
} else if (provider === 'aws') {
text = await transcribeWithAWS(audioBuffer, mime);
} else if (provider === 'local') {
text = await transcribeWithLocal(audioBuffer, mime);
} else if (provider === 'litellm') {
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
var db = require('../db/database');
var adminSttModel = await db.getSetting('stt.model') || '';
var sttModel = adminSttModel || process.env.LITELLM_STT_MODEL || 'gemini-2.0-flash';
var adminSttModel = await db.getSetting('stt.model') || '';
var sttModel = adminSttModel || process.env.LITELLM_STT_MODEL || '';
if (!sttModel) return res.json({ success: false, error: 'No LiteLLM STT model configured' });
var isTranscriptionModel = false;
try {
var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, '');
var modelInfoResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 });
isTranscriptionModel = getLiteLLMSTTModels(modelInfoResp.data && modelInfoResp.data.data).indexOf(sttModel) !== -1;
} catch (e) { logger.warn('LiteLLM STT mode lookup failed: ' + e.message); }
if (isTranscriptionModel) {
var ext = mime.split('/')[1] || 'webm';
var file = new File([audioBuffer], 'audio.' + ext, { type: mime });
var form = new FormData();
form.append('file', file);
form.append('model', sttModel);
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
method: 'POST', headers: getLiteLLMHeaders(), body: form
}).then(function(r) { return r.json().then(function(d) { return { data: d }; }); });
text = sttResp.data && sttResp.data.text ? sttResp.data.text : '';
} else {
var sttResp = await axios.post(gatewayUrl('/chat/completions'), {
model: sttModel,
messages: [{ role: 'user', content: [
{ type: 'input_audio', input_audio: { data: audioBase64, format: mime.split('/')[1] || 'webm' } },
{ type: 'text', text: 'Transcribe this audio. Output the spoken words only.' }
]}]
}, { headers: getLiteLLMHeaders('application/json'), timeout: 60000 });
if (sttResp.data && sttResp.data.choices && sttResp.data.choices[0]) {
text = (sttResp.data.choices[0].message && sttResp.data.choices[0].message.content) || '';
}
}
} else if (provider === 'openai') {
if (!whisperClient) return res.json({ success: false, error: 'Whisper not configured' });
var file = new File([audioBuffer], 'audio.webm', { type: mime });
var wResult = await whisperClient.audio.transcriptions.create({
file: file, model: 'whisper-1', language: 'en', response_format: 'text'
});
text = typeof wResult === 'string' ? wResult : (wResult.text || '');
}
var ext = mime.split('/')[1] || 'webm';
var file = new File([audioBuffer], 'audio.' + ext, { type: mime });
var form = new FormData();
form.append('file', file);
form.append('model', sttModel);
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
method: 'POST', headers: getLiteLLMHeaders(), body: form
}).then(function(r) { return r.json().then(function(d) { return { data: d }; }); });
text = sttResp.data && sttResp.data.text ? sttResp.data.text : '';
res.json({ success: true, text: text.trim(), provider: provider, duration: Date.now() - start });
} catch (e) {
@ -794,8 +652,6 @@ router.get('/config/embeddings', async function(req, res) {
var provider = 'none';
if (process.env.LITELLM_API_BASE) provider = 'litellm';
else if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT || process.env.GOOGLE_VERTEX_PROJECT) provider = 'vertex';
else if (process.env.OPENAI_API_KEY) provider = 'openai';
res.json({
success: true,
@ -807,14 +663,7 @@ router.get('/config/embeddings', async function(req, res) {
dbDimensions: dbDims,
envModel: envModel,
envDimensions: envDims,
models: [
{ id: 'vertex_ai/text-embedding-005', name: 'Vertex text-embedding-005', dims: 768, tag: 'RECOMMENDED' },
{ id: 'vertex_ai/gemini-embedding-001', name: 'Vertex gemini-embedding-001', dims: 3072, tag: 'HIGH-DIM' },
{ id: 'vertex_ai/text-multilingual-embedding-002', name: 'Vertex multilingual-embedding-002', dims: 768, tag: 'MULTILINGUAL' },
{ id: 'text-embedding-3-small', name: 'OpenAI text-embedding-3-small', dims: 1536, tag: 'OPENAI' },
{ id: 'text-embedding-3-large', name: 'OpenAI text-embedding-3-large', dims: 3072, tag: 'OPENAI' },
{ id: 'text-embedding-ada-002', name: 'OpenAI text-embedding-ada-002', dims: 1536, tag: 'OPENAI' }
]
models: []
});
} catch (e) { res.status(500).json({ error: 'Request failed' }); }
});
@ -828,15 +677,6 @@ router.get('/config/embeddings/discover', async function(req, res) {
var provider = 'none';
if (process.env.LITELLM_API_BASE) provider = 'litellm';
else if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT || process.env.GOOGLE_VERTEX_PROJECT) provider = 'vertex';
else if (process.env.OPENAI_API_KEY) provider = 'openai';
var knownVertex = [
{ id: 'vertex_ai/text-embedding-005', name: 'Vertex text-embedding-005', dims: 768 },
{ id: 'vertex_ai/gemini-embedding-001', name: 'Vertex gemini-embedding-001', dims: 3072 },
{ id: 'vertex_ai/text-multilingual-embedding-002', name: 'Vertex multilingual-embedding-002', dims: 768 },
{ id: 'text-embedding-005', name: 'text-embedding-005 (direct)', dims: 768 }
];
if (provider === 'litellm') {
try {
@ -846,37 +686,6 @@ router.get('/config/embeddings/discover', async function(req, res) {
discovered.push({ id: m.id, name: m.name, dims: m.dims, source: 'gateway-api' });
});
} catch(e) { logger.warn('LiteLLM embedding metadata discovery failed: ' + e.message); }
if (discovered.length === 0) {
knownVertex.forEach(function(m) {
if (!discovered.find(function(d) { return d.id === m.id; })) {
discovered.push(Object.assign({ source: 'builtin' }, m));
}
});
}
}
if (provider === 'vertex') {
knownVertex.forEach(function(m) {
discovered.push(Object.assign({ source: 'vertex-builtin' }, m));
});
}
if (provider === 'openai') {
try {
var OpenAI = require('openai').OpenAI;
var oc = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
var oMods = await oc.models.list();
oMods.data.forEach(function(m) {
if (m.id.toLowerCase().indexOf('embedding') !== -1) {
var dims = m.id === 'text-embedding-3-large' ? 3072 : m.id === 'text-embedding-3-small' ? 1536 : 1536;
discovered.push({ id: m.id, name: m.id, dims: dims, source: 'openai-api' });
}
});
} catch(e) {
[{ id: 'text-embedding-3-small', dims: 1536 }, { id: 'text-embedding-3-large', dims: 3072 }, { id: 'text-embedding-ada-002', dims: 1536 }].forEach(function(m) {
discovered.push({ id: m.id, name: m.id, dims: m.dims, source: 'openai-builtin' });
});
}
}
if (search) {

View file

@ -1,52 +1,30 @@
const express = require('express');
const router = express.Router();
const multer = require('multer');
const { whisperClient, litellmClient } = require('../utils/ai');
const { transcribeWithAWS, isAWSTranscribeConfigured } = require('../utils/transcribeAWS');
const { transcribeWithLocal, isLocalWhisperConfigured } = require('../utils/transcribeLocal');
const { transcribeWithGemini, isGoogleSTTConfigured } = require('../utils/transcribeGoogle');
const { litellmClient } = require('../utils/ai');
const { authMiddleware } = require('../middleware/auth');
var logger = require('../utils/logger');
var { gatewayUrl } = require('../utils/errors');
var { getLiteLLMHeaders } = require('../utils/litellm');
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } });
// Provider priority (auto-detect):
// TRANSCRIBE_PROVIDER=google → Vertex AI / Gemini (direct, HIPAA eligible)
// TRANSCRIBE_PROVIDER=aws → Amazon Transcribe (HIPAA eligible)
// TRANSCRIBE_PROVIDER=local → local whisper.cpp / faster-whisper
// TRANSCRIBE_PROVIDER=openai → OpenAI Whisper (direct)
// TRANSCRIBE_PROVIDER=litellm → LiteLLM proxy at /v1/audio/transcriptions
// Auto: google > aws > openai
// STT is intentionally routed only through LiteLLM.
function getTranscribeProvider() {
var env = process.env.TRANSCRIBE_PROVIDER;
if (env === 'google') return 'google';
if (env === 'aws') return 'aws';
if (env === 'local') return 'local';
if (env === 'openai') return 'openai';
if (env === 'litellm') return 'litellm';
// Auto-detect
if (isGoogleSTTConfigured()) return 'google';
if (isAWSTranscribeConfigured()) return 'aws';
return 'openai';
if (litellmClient) return 'litellm';
return 'none';
}
function isTranscribeAvailable() {
if (isGoogleSTTConfigured()) return true;
if (isLocalWhisperConfigured()) return true;
if (isAWSTranscribeConfigured()) return true;
if (litellmClient) return true;
if (whisperClient) return true;
return false;
}
var provider = getTranscribeProvider();
var medical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true';
var available = isTranscribeAvailable();
console.log('🎙️ Transcribe provider:', provider +
(provider === 'aws' && medical ? ' (Medical)' : '') +
(provider === 'google' ? ' (model: ' + (process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash') + ')' : '') +
(available ? '' : ' (NOT CONFIGURED — browser speech only)'));
console.log('🎙️ Transcribe provider:', provider + (available ? '' : ' (NOT CONFIGURED — browser speech only)'));
router.get('/transcribe/status', authMiddleware, (req, res) => {
res.json({ available: available, provider: available ? provider : 'none' });
@ -66,68 +44,29 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
console.log('[Transcribe] Received ' + (fileSize / 1024).toFixed(0) + 'KB audio (' + (req.file.mimetype || 'unknown') + ') via ' + provider + (userModel ? ' (user model: ' + userModel + ')' : ''));
if (provider === 'google') {
var model = userModel || adminSttModel || process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash';
var text = await transcribeWithGemini(req.file.buffer, req.file.mimetype || 'audio/webm', model);
console.log('[Transcribe] Google/' + model + ' done in ' + (Date.now() - startTime) + 'ms');
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via google', req, { category: 'clinical' });
return res.json({ success: true, text: text, provider: 'google-' + model, duration: Date.now() - startTime });
}
if (provider !== 'litellm' || !process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
var sttModel = userModel || adminSttModel || process.env.LITELLM_STT_MODEL || '';
if (!sttModel) return res.status(400).json({ error: 'No LiteLLM STT model configured.' });
var mimeType = req.file.mimetype || 'audio/webm';
var ext = mimeType.split('/')[1] || 'webm';
if (provider === 'local') {
var text = await transcribeWithLocal(req.file.buffer, req.file.mimetype || 'audio/webm');
console.log('[Transcribe] Local done in ' + (Date.now() - startTime) + 'ms');
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via local', req, { category: 'clinical' });
return res.json({ success: true, text: text, provider: 'local-whisper', duration: Date.now() - startTime });
}
var file = new File([req.file.buffer], 'audio.' + ext, { type: mimeType });
var form = new FormData();
form.append('file', file);
form.append('model', sttModel);
if (provider === 'aws') {
if (!isAWSTranscribeConfigured()) return res.status(400).json({ error: 'AWS Transcribe not configured.' });
var text = await transcribeWithAWS(req.file.buffer, req.file.mimetype || 'audio/webm');
console.log('[Transcribe] AWS done in ' + (Date.now() - startTime) + 'ms');
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via aws', req, { category: 'clinical' });
return res.json({ success: true, text: text, provider: 'aws-transcribe', duration: Date.now() - startTime });
}
if (provider === 'litellm') {
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
var sttModel = userModel || adminSttModel || process.env.LITELLM_STT_MODEL || 'whisper-1';
var mimeType = req.file.mimetype || 'audio/webm';
var ext = mimeType.split('/')[1] || 'webm';
var file = new File([req.file.buffer], 'audio.' + ext, { type: mimeType });
var form = new FormData();
form.append('file', file);
form.append('model', sttModel);
var fetchHeaders = {};
if (process.env.LITELLM_API_KEY) fetchHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
method: 'POST', headers: fetchHeaders, body: form,
});
if (!sttResp.ok) {
var errBody = await sttResp.text();
throw new Error('LiteLLM /audio/transcriptions ' + sttResp.status + ': ' + errBody.substring(0, 500));
}
var data = await sttResp.json();
var text = (data && data.text) ? String(data.text).trim() : '';
console.log('[Transcribe] LiteLLM/' + sttModel + ' done in ' + (Date.now() - startTime) + 'ms');
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via litellm', req, { category: 'clinical' });
return res.json({ success: true, text: text, provider: 'litellm/' + sttModel, duration: Date.now() - startTime });
}
// OpenAI Whisper (direct)
if (!whisperClient) return res.status(400).json({ error: 'Whisper not configured. Set OPENAI_API_KEY.' });
var file = new File([req.file.buffer], 'audio.webm', { type: req.file.mimetype || 'audio/webm' });
var result = await whisperClient.audio.transcriptions.create({
file, model: 'whisper-1', language: 'en',
response_format: 'text',
prompt: 'Medical patient encounter. Pediatric. Clinical terms, diagnoses, medications.'
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
method: 'POST', headers: getLiteLLMHeaders(), body: form,
});
var text = typeof result === 'string' ? result : result.text;
console.log('[Transcribe] Whisper done in ' + (Date.now() - startTime) + 'ms');
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via openai', req, { category: 'clinical' });
res.json({ success: true, text: text, provider: 'openai-whisper', duration: Date.now() - startTime });
if (!sttResp.ok) {
var errBody = await sttResp.text();
throw new Error('LiteLLM /audio/transcriptions ' + sttResp.status + ': ' + errBody.substring(0, 500));
}
var data = await sttResp.json();
var text = (data && data.text) ? String(data.text).trim() : '';
console.log('[Transcribe] LiteLLM/' + sttModel + ' done in ' + (Date.now() - startTime) + 'ms');
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via litellm', req, { category: 'clinical' });
return res.json({ success: true, text: text, provider: 'litellm/' + sttModel, duration: Date.now() - startTime });
} catch (err) {
var detail = err.response && err.response.data

View file

@ -1,26 +1,15 @@
const express = require('express');
const router = express.Router();
const axios = require('axios');
const { synthesizeWithGoogleTTS } = require('../utils/ttsGoogle');
const { authMiddleware } = require('../middleware/auth');
var logger = require('../utils/logger');
var { gatewayUrl } = require('../utils/errors');
var { getTTSProvider } = require('../utils/ttsProvider');
var { getLiteLLMHeaders } = require('../utils/litellm');
// Provider priority (auto-detect):
// TTS_PROVIDER=litellm → LiteLLM proxy at /v1/audio/speech
// TTS_PROVIDER=google → Google Cloud TTS direct (HIPAA eligible, opt-in)
// TTS_PROVIDER=elevenlabs → ElevenLabs direct (not HIPAA, opt-in)
// Auto: litellm > google > elevenlabs
//
// Design: the LiteLLM branch is a pure passthrough — model + voice strings
// go straight into the OpenAI-compatible /v1/audio/speech body. No regex
// dispatch on model/voice names; the LiteLLM config decides which provider
// (OpenAI, ElevenLabs, Vertex, Azure) actually serves the request.
// TTS is intentionally routed only through LiteLLM. Provider-specific voice
// routing belongs in LiteLLM config, not this app.
var ttsProvider = getTTSProvider();
console.log('🔊 TTS provider:', ttsProvider +
(ttsProvider === 'google' ? ' (voice: ' + (process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F') + ')' : ''));
console.log('🔊 TTS provider:', ttsProvider);
router.post('/text-to-speech', authMiddleware, async (req, res) => {
try {
@ -35,51 +24,27 @@ router.post('/text-to-speech', authMiddleware, async (req, res) => {
var adminModel = await db.getSetting('tts.model') || '';
var resolvedVoice = userVoice || adminVoice || '';
if (ttsProvider === 'litellm') {
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
var ttsModel = adminModel || process.env.LITELLM_TTS_MODEL || 'tts-1';
var ttsVoice = resolvedVoice || process.env.LITELLM_TTS_VOICE || 'alloy';
var ttsResp = await fetch(gatewayUrl('/audio/speech'), {
method: 'POST',
headers: getLiteLLMHeaders('application/json'),
body: JSON.stringify({ model: ttsModel, input: text, voice: ttsVoice })
});
if (!ttsResp.ok) {
var errBody = await ttsResp.text();
throw new Error('LiteLLM /audio/speech ' + ttsResp.status + ': ' + errBody.substring(0, 500));
}
var audioBuf = Buffer.from(await ttsResp.arrayBuffer());
res.set('Content-Type', ttsResp.headers.get('content-type') || 'audio/mpeg');
res.set('X-TTS-Provider', 'litellm/' + ttsModel);
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
return res.send(audioBuf);
if (ttsProvider !== 'litellm' || !process.env.LITELLM_API_BASE) {
return res.status(400).json({ error: 'TTS not configured. Set LITELLM_API_BASE.' });
}
var ttsModel = adminModel || process.env.LITELLM_TTS_MODEL || '';
var ttsVoice = resolvedVoice || process.env.LITELLM_TTS_VOICE || '';
if (!ttsModel) return res.status(400).json({ error: 'No LiteLLM TTS model configured.' });
if (ttsProvider === 'google') {
var voice = resolvedVoice || process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
var buffer = await synthesizeWithGoogleTTS(text, voice);
res.set('Content-Type', 'audio/mpeg');
res.set('X-TTS-Provider', 'google-tts/' + voice);
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
return res.send(buffer);
var ttsResp = await fetch(gatewayUrl('/audio/speech'), {
method: 'POST',
headers: getLiteLLMHeaders('application/json'),
body: JSON.stringify({ model: ttsModel, input: text, voice: ttsVoice })
});
if (!ttsResp.ok) {
var errBody = await ttsResp.text();
throw new Error('LiteLLM /audio/speech ' + ttsResp.status + ': ' + errBody.substring(0, 500));
}
if (ttsProvider === 'elevenlabs') {
if (!process.env.ELEVENLABS_API_KEY) return res.status(400).json({ error: 'ElevenLabs not configured' });
var resp = await axios({
method: 'POST',
url: 'https://api.elevenlabs.io/v1/text-to-speech/pNInz6obpgDQGcFmaJgB',
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY, 'Content-Type': 'application/json' },
data: { text: text, model_id: 'eleven_turbo_v2_5', voice_settings: { stability: 0.5, similarity_boost: 0.75 } },
responseType: 'arraybuffer'
});
res.set('Content-Type', 'audio/mpeg');
res.set('X-TTS-Provider', 'elevenlabs');
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
return res.send(Buffer.from(resp.data));
}
res.status(400).json({ error: 'TTS not configured. Set LITELLM_API_BASE, GOOGLE_VERTEX_PROJECT, or ELEVENLABS_API_KEY.' });
var audioBuf = Buffer.from(await ttsResp.arrayBuffer());
res.set('Content-Type', ttsResp.headers.get('content-type') || 'audio/mpeg');
res.set('X-TTS-Provider', 'litellm/' + ttsModel);
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
return res.send(audioBuf);
} catch (err) {
var detail = err.response && err.response.data
? JSON.stringify(err.response.data).substring(0, 500)

View file

@ -6,6 +6,8 @@ var express = require('express');
var router = express.Router();
var db = require('../db/database');
var { authMiddleware } = require('../middleware/auth');
var { getSTTModelLists, getSTTProvider } = require('../utils/sttProvider');
var { getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider');
router.use(authMiddleware);
@ -44,103 +46,11 @@ router.post('/preferences', async function(req, res) {
// Get available STT models and TTS voices
router.get('/preferences/options', async function(req, res) {
try {
var provider = process.env.TRANSCRIBE_PROVIDER;
var ttsProvider = process.env.TTS_PROVIDER;
var provider = getSTTProvider();
var ttsProvider = getTTSProvider();
// Auto-detect providers if not explicitly set
if (!provider) {
if (process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT) provider = 'google';
else if (process.env.AWS_BEDROCK_REGION) provider = 'aws';
else if (process.env.LITELLM_API_BASE) provider = 'litellm';
else if (process.env.OPENAI_API_KEY) provider = 'openai';
}
if (!ttsProvider) {
if (process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT) ttsProvider = 'google';
else if (process.env.LITELLM_API_BASE) ttsProvider = 'litellm';
else if (process.env.OPENAI_API_KEY) ttsProvider = 'openai';
else if (process.env.ELEVENLABS_API_KEY) ttsProvider = 'elevenlabs';
}
// STT Models
var sttModels = [];
if (provider === 'google') {
sttModels = [
{ value: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash (Experimental, fastest)' },
{ value: 'gemini-2.0-flash', label: 'Gemini 2.0 Flash (Fast, accurate)' },
{ value: 'gemini-1.5-flash', label: 'Gemini 1.5 Flash (Stable)' },
{ value: 'gemini-1.5-pro', label: 'Gemini 1.5 Pro (Best quality, slower)' }
];
} else if (provider === 'litellm') {
sttModels = [
{ value: 'local-whisper-large-v3-turbo', label: 'Local Whisper Large v3 Turbo' },
{ value: 'local-whisper-large-v3', label: 'Local Whisper Large v3' },
{ value: 'local-parakeet-v3', label: 'Local Parakeet v3' },
{ value: 'local-whisper-1', label: 'Local Whisper-1 Alias' },
{ value: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash Exp' },
{ value: 'gemini-2.0-flash', label: 'Gemini 2.0 Flash' },
{ value: 'gemini-1.5-flash', label: 'Gemini 1.5 Flash' },
{ value: 'gemini-1.5-pro', label: 'Gemini 1.5 Pro' }
];
} else if (provider === 'openai') {
sttModels = [
{ value: 'whisper-1', label: 'Whisper-1 (OpenAI standard)' }
];
} else if (provider === 'aws') {
sttModels = [
{ value: 'default', label: 'AWS Transcribe (Standard)' },
{ value: 'medical', label: 'AWS Transcribe Medical' }
];
} else if (provider === 'local') {
sttModels = [
{ value: 'tiny', label: 'Whisper Tiny (fastest)' },
{ value: 'base', label: 'Whisper Base' },
{ value: 'small', label: 'Whisper Small' },
{ value: 'medium', label: 'Whisper Medium' },
{ value: 'large', label: 'Whisper Large (best quality)' }
];
}
// TTS Voices
var ttsVoices = [];
if (ttsProvider === 'google') {
ttsVoices = [
{ value: 'en-US-Journey-F', label: 'Journey (Female, natural)' },
{ value: 'en-US-Journey-D', label: 'Journey (Male, natural)' },
{ value: 'en-US-Studio-O', label: 'Studio O (Female, expressive)' },
{ value: 'en-US-Studio-M', label: 'Studio M (Male, expressive)' },
{ value: 'en-US-Neural2-A', label: 'Neural2 A (Male, standard)' },
{ value: 'en-US-Neural2-C', label: 'Neural2 C (Female, standard)' },
{ value: 'en-US-Neural2-D', label: 'Neural2 D (Male, standard)' },
{ value: 'en-US-Neural2-E', label: 'Neural2 E (Female, standard)' },
{ value: 'en-US-Neural2-F', label: 'Neural2 F (Female, standard)' },
{ value: 'en-US-Neural2-G', label: 'Neural2 G (Female, standard)' },
{ value: 'en-US-Neural2-H', label: 'Neural2 H (Female, standard)' },
{ value: 'en-US-Neural2-I', label: 'Neural2 I (Male, standard)' },
{ value: 'en-US-Neural2-J', label: 'Neural2 J (Male, standard)' }
];
} else if (ttsProvider === 'litellm' || ttsProvider === 'openai') {
ttsVoices = [
{ value: 'alloy', label: 'Alloy (Neutral)' },
{ value: 'echo', label: 'Echo (Male)' },
{ value: 'fable', label: 'Fable (British Male)' },
{ value: 'onyx', label: 'Onyx (Deep Male)' },
{ value: 'nova', label: 'Nova (Female)' },
{ value: 'shimmer', label: 'Shimmer (Soft Female)' }
];
} else if (ttsProvider === 'elevenlabs') {
ttsVoices = [
{ value: 'adam', label: 'Adam (Male, deep)' },
{ value: 'rachel', label: 'Rachel (Female, calm)' },
{ value: 'domi', label: 'Domi (Female, strong)' },
{ value: 'bella', label: 'Bella (Female, soft)' },
{ value: 'antoni', label: 'Antoni (Male, deep)' },
{ value: 'elli', label: 'Elli (Female, young)' },
{ value: 'josh', label: 'Josh (Male, narration)' },
{ value: 'arnold', label: 'Arnold (Male, crisp)' },
{ value: 'sam', label: 'Sam (Male, raspy)' }
];
}
var sttModels = getSTTModelLists().litellm.map(function(model) { return { value: model, label: model }; });
var ttsVoices = getTTSVoiceLists().litellm.map(function(voice) { return { value: voice, label: voice }; });
res.json({
success: true,

View file

@ -110,17 +110,6 @@ if (process.env.LITELLM_API_BASE) {
}
}
// ============================================================
// WHISPER CLIENT (always OpenAI, separate from text AI)
// ============================================================
var whisperClient = null;
if (process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY.startsWith('sk-')) {
whisperClient = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
console.log('✅ Whisper: configured');
} else {
console.log('⚠️ Whisper: not configured (set OPENAI_API_KEY)');
}
// Force provider from env if explicitly set
if (process.env.AI_PROVIDER) {
activeProvider = process.env.AI_PROVIDER;
@ -694,4 +683,4 @@ async function discoverModels() {
return discovered;
}
module.exports = { callAI, callAIStream, whisperClient, activeProvider, discoverModels, vertexClient, litellmClient };
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient };

View file

@ -1,18 +1,13 @@
// ============================================================
// EMBEDDINGS UTILITY — Generate & search with Vertex AI embeddings
// Supports: Vertex AI (direct), LiteLLM proxy, OpenAI fallback
// EMBEDDINGS UTILITY — Generate & search through LiteLLM embeddings
// ============================================================
var axios = require('axios');
var { gatewayUrl } = require('./errors');
var { getLiteLLMHeaders } = require('./litellm');
// Vertex AI embedding models (via LiteLLM or direct)
// gemini-embedding-001: 768 dims, multilingual + code, best quality
// text-embedding-005: 768 dims, English + code optimized
// text-multilingual-embedding-002: 768 dims, multilingual focus
var DEFAULT_MODEL = 'vertex_ai/text-embedding-005';
var DEFAULT_DIMS = 768;
var DEFAULT_MODEL = 'openai-text-embedding-3-large';
var DEFAULT_DIMS = 3072;
/**
* Generate embedding for text using configured provider
@ -40,22 +35,11 @@ async function generateEmbedding(text, opts) {
throw new Error('Empty text provided for embedding');
}
// Try LiteLLM first if configured
if (process.env.LITELLM_API_BASE) {
return await generateEmbeddingLiteLLM(truncated, model, dimensions);
}
// Try Vertex AI direct if configured
if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT) {
return await generateEmbeddingVertexDirect(truncated, model, dimensions);
}
// Fallback to OpenAI if configured
if (process.env.OPENAI_API_KEY) {
return await generateEmbeddingOpenAI(truncated, model, dimensions);
}
throw new Error('No embedding provider configured. Set LITELLM_API_BASE, VERTEX_PROJECT, or OPENAI_API_KEY');
throw new Error('No embedding provider configured. Set LITELLM_API_BASE');
}
/**
@ -68,8 +52,7 @@ async function generateEmbeddingLiteLLM(text, model, dimensions) {
input: text
};
// Only include dimensions if model supports it (some models have fixed dims)
if (dimensions && model.includes('text-embedding-005')) {
if (dimensions) {
payload.dimensions = dimensions;
}
@ -89,80 +72,6 @@ async function generateEmbeddingLiteLLM(text, model, dimensions) {
}
}
/**
* Generate embedding via Vertex AI direct (using @google-cloud/vertexai)
*/
async function generateEmbeddingVertexDirect(text, model, dimensions) {
try {
var { VertexAI } = require('@google-cloud/vertexai');
var project = process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT;
var location = process.env.VERTEX_LOCATION || 'us-central1';
if (!project) {
throw new Error('VERTEX_PROJECT or GOOGLE_CLOUD_PROJECT not set');
}
var vertexAI = new VertexAI({ project: project, location: location });
// Extract model name (strip vertex_ai/ prefix if present)
var modelName = model.replace(/^vertex_ai\//, '');
// For text-embedding-005, we can specify output dimensions
var request = {
instances: [{ content: text }]
};
if (dimensions && modelName.includes('text-embedding-005')) {
request.parameters = { outputDimensionality: dimensions };
}
// Use predictText API for embeddings
var predictionClient = vertexAI.preview.getPredictionServiceClient();
var endpoint = `projects/${project}/locations/${location}/publishers/google/models/${modelName}`;
var [response] = await predictionClient.predict({
endpoint: endpoint,
instances: [{ content: text }],
parameters: request.parameters || {}
});
if (!response || !response.predictions || !response.predictions[0]) {
throw new Error('Invalid response from Vertex AI');
}
var prediction = response.predictions[0];
return prediction.embeddings?.values || prediction.values || prediction;
} catch (err) {
console.error('[Embeddings] Vertex AI direct error:', err.message);
throw new Error('Vertex AI embedding failed: ' + err.message);
}
}
/**
* Generate embedding via OpenAI (fallback)
*/
async function generateEmbeddingOpenAI(text, model, dimensions) {
try {
var openai = require('openai');
var client = new openai.OpenAI({ apiKey: process.env.OPENAI_API_KEY });
// Use OpenAI's text-embedding-3-small model (1536 dims by default)
var embModel = 'text-embedding-3-small';
var response = await client.embeddings.create({
model: embModel,
input: text,
dimensions: dimensions || 768 // OpenAI supports custom dimensions
});
return response.data[0].embedding;
} catch (err) {
console.error('[Embeddings] OpenAI error:', err.message);
throw new Error('OpenAI embedding failed: ' + err.message);
}
}
/**
* Search for similar content using cosine similarity
* @param {string} queryText - Search query
@ -271,12 +180,7 @@ function getLiteLLMEmbeddingModels(models) {
* Check if embeddings are available (provider configured)
*/
function isEmbeddingsAvailable() {
return !!(
process.env.LITELLM_API_BASE ||
process.env.VERTEX_PROJECT ||
process.env.GOOGLE_CLOUD_PROJECT ||
process.env.OPENAI_API_KEY
);
return !!process.env.LITELLM_API_BASE;
}
module.exports = {

View file

@ -1,43 +1,24 @@
const GOOGLE_STT_MODELS = ['gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.0-flash-lite'];
const AWS_STT_MODES = ['standard', 'medical'];
const OPENAI_STT_MODELS = ['whisper-1'];
const LITELLM_STT_MODELS = ['local-whisper-large-v3-turbo', 'local-whisper-large-v3', 'local-parakeet-v3', 'local-whisper-1', 'gemini-2.0-flash', 'gemini-2.5-flash'];
const LOCAL_WHISPER_MODELS = ['tiny', 'base', 'small', 'medium', 'large'];
function getSTTDependencies() {
var { isGoogleSTTConfigured } = require('./transcribeGoogle');
var { isAWSTranscribeConfigured } = require('./transcribeAWS');
var { isLocalWhisperConfigured } = require('./transcribeLocal');
var { whisperClient, litellmClient } = require('./ai');
var { litellmClient } = require('./ai');
return {
google: isGoogleSTTConfigured(),
aws: isAWSTranscribeConfigured(),
local: isLocalWhisperConfigured(),
litellm: !!litellmClient,
openai: !!whisperClient
litellm: !!litellmClient
};
}
function getSTTProvider(deps) {
var env = process.env.TRANSCRIBE_PROVIDER || 'auto';
if (env !== 'auto') return env;
if (env === 'litellm') return 'litellm';
var configured = deps || getSTTDependencies();
if (configured.google) return 'google';
if (configured.aws) return 'aws';
if (configured.local) return 'local';
if (configured.litellm) return 'litellm';
if (configured.openai) return 'openai';
return 'none';
}
function getSTTModelLists() {
return {
google: GOOGLE_STT_MODELS,
aws: AWS_STT_MODES,
openai: OPENAI_STT_MODELS,
litellm: LITELLM_STT_MODELS,
local: LOCAL_WHISPER_MODELS
litellm: LITELLM_STT_MODELS
};
}
@ -53,11 +34,7 @@ function getLiteLLMSTTModels(models) {
}
module.exports = {
AWS_STT_MODES,
GOOGLE_STT_MODELS,
LITELLM_STT_MODELS,
LOCAL_WHISPER_MODELS,
OPENAI_STT_MODELS,
getSTTDependencies,
getLiteLLMSTTModels,
getSTTModelLists,

View file

@ -1,211 +0,0 @@
// ============================================================
// TRANSCRIBE-AWS.JS — Amazon Transcribe Streaming
// No S3 required. Audio streams directly to AWS.
//
// Audio conversion: ffmpeg converts browser WebM/Opus → PCM 16kHz
// which is the most reliable format for AWS Transcribe. If ffmpeg
// is not installed, falls back to sending ogg-opus directly.
//
// Env vars:
// TRANSCRIBE_PROVIDER=aws — use this instead of Whisper
// AWS_TRANSCRIBE_MEDICAL=true — use Transcribe Medical (better
// accuracy for clinical dictation)
// AWS_TRANSCRIBE_SPECIALTY=PRIMARYCARE — medical specialty
// Options: PRIMARYCARE, CARDIOLOGY, NEUROLOGY, ONCOLOGY,
// RADIOLOGY, UROLOGY
// AWS_BEDROCK_REGION — reused for Transcribe region
// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — reused credentials
// ============================================================
const { spawn } = require('child_process');
const CHUNK_SIZE = 8192; // 8 KB per audio chunk (AWS limit ~32KB per event frame)
// Convert any browser audio (WebM, OGG, etc.) to raw PCM s16le 16kHz mono
// using ffmpeg. Returns a Buffer of raw PCM bytes.
// Throws if ffmpeg is not installed.
function convertToPCM(inputBuffer) {
return new Promise(function(resolve, reject) {
var ff = spawn('ffmpeg', [
'-i', 'pipe:0', // read from stdin
'-f', 's16le', // raw signed 16-bit little-endian PCM
'-ar', '16000', // 16 kHz — ideal for speech recognition
'-ac', '1', // mono
'-acodec', 'pcm_s16le',
'pipe:1' // write to stdout
]);
var out = [];
var errBuf = [];
ff.stdout.on('data', function(d) { out.push(d); });
ff.stderr.on('data', function(d) { errBuf.push(d); }); // ffmpeg logs to stderr, not an error
ff.on('close', function(code) {
if (code !== 0) {
reject(new Error('ffmpeg conversion failed (code ' + code + '): ' + Buffer.concat(errBuf).toString().slice(-200)));
return;
}
resolve(Buffer.concat(out));
});
ff.on('error', function(err) {
reject(new Error('ffmpeg not found. Install ffmpeg on the server: ' + err.message));
});
ff.stdin.write(inputBuffer);
ff.stdin.end();
});
}
async function* makeAudioStream(buffer) {
// 8KB chunks — larger sizes cause AWS SDK deserialization errors
// ("to see the raw response, inspect the hidden field {error}.$response").
// No artificial delay between chunks; yield with a microtask break
// to avoid blocking the event loop without adding real latency.
for (var i = 0; i < buffer.length; i += CHUNK_SIZE) {
yield { AudioEvent: { AudioChunk: buffer.slice(i, i + CHUNK_SIZE) } };
// Microtask break every 16 chunks (~128KB) to keep event loop responsive
if ((i / CHUNK_SIZE) % 16 === 15) {
await new Promise(function(r) { setImmediate(r); });
}
}
}
function logTranscribeError(label, err) {
console.error('[Transcribe] ' + label + ':', err.message);
if (err.name) console.error('[Transcribe] name:', err.name);
if (err.$metadata) console.error('[Transcribe] metadata:', JSON.stringify(err.$metadata));
if (err.cause) console.error('[Transcribe] cause:', err.cause.message || JSON.stringify(err.cause));
if (err.$response) {
try {
var resp = err.$response;
console.error('[Transcribe] status:', resp.statusCode);
if (resp.headers) console.error('[Transcribe] headers:', JSON.stringify(resp.headers));
if (resp.body) console.error('[Transcribe] body:', typeof resp.body === 'string' ? resp.body.slice(0, 500) : '(stream)');
} catch (e) { /* ignore logging errors */ }
}
}
async function transcribeWithAWS(audioBuffer, mimeType) {
var startTime = Date.now();
var TranscribeModule = require('@aws-sdk/client-transcribe-streaming');
var TranscribeStreamingClient = TranscribeModule.TranscribeStreamingClient;
var region = process.env.AWS_BEDROCK_REGION || process.env.AWS_REGION || 'us-east-1';
var credentials = process.env.AWS_ACCESS_KEY_ID ? {
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
} : undefined;
// Try ffmpeg conversion to PCM (most reliable with AWS Transcribe).
// Falls back to ogg-opus passthrough if ffmpeg is not installed.
var audioToSend = audioBuffer;
var mediaEncoding = 'ogg-opus';
var sampleRate = 48000;
try {
var ffStart = Date.now();
audioToSend = await convertToPCM(audioBuffer);
mediaEncoding = 'pcm';
sampleRate = 16000;
console.log('[Transcribe] ffmpeg: ' + (Date.now() - ffStart) + 'ms — ' + audioBuffer.length + ' → ' + audioToSend.length + ' bytes');
} catch (ffErr) {
console.warn('[Transcribe] ffmpeg unavailable, sending ogg-opus directly:', ffErr.message);
if (mimeType && mimeType.indexOf('wav') !== -1) {
mediaEncoding = 'pcm';
sampleRate = 16000;
}
}
// Minimum ~0.5s of PCM audio at 16kHz mono 16-bit = 16000 bytes
if (mediaEncoding === 'pcm' && audioToSend.length < 16000) {
console.warn('[Transcribe] Audio too short (' + audioToSend.length + ' bytes), skipping');
return '';
}
var useMedical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true';
var specialty = process.env.AWS_TRANSCRIBE_SPECIALTY || 'PRIMARYCARE';
var transcript = '';
// Helper to collect transcript from a result stream
async function collectTranscript(resultStream) {
var text = '';
for await (var event of resultStream) {
if (event.TranscriptEvent && event.TranscriptEvent.Transcript) {
var results = event.TranscriptEvent.Transcript.Results || [];
for (var i = 0; i < results.length; i++) {
var r = results[i];
if (!r.IsPartial && r.Alternatives && r.Alternatives[0]) {
text += r.Alternatives[0].Transcript + ' ';
}
}
}
}
return text;
}
// Try Medical first if enabled, with fallback to Standard
var streamStart = Date.now();
if (useMedical) {
try {
var client = new TranscribeStreamingClient({ region: region, credentials: credentials });
var medCmd = new TranscribeModule.StartMedicalStreamTranscriptionCommand({
LanguageCode: 'en-US',
MediaSampleRateHertz: sampleRate,
MediaEncoding: mediaEncoding,
Specialty: specialty,
Type: 'DICTATION',
AudioStream: makeAudioStream(audioToSend)
});
var medResp = await client.send(medCmd);
transcript = await collectTranscript(medResp.TranscriptResultStream);
console.log('[Transcribe] Medical OK — ' + (Date.now() - streamStart) + 'ms, ' + transcript.length + ' chars');
} catch (medErr) {
logTranscribeError('Medical failed', medErr);
console.warn('[Transcribe] Falling back to standard...');
streamStart = Date.now();
try {
var client2 = new TranscribeStreamingClient({ region: region, credentials: credentials });
var stdCmd = new TranscribeModule.StartStreamTranscriptionCommand({
LanguageCode: 'en-US',
MediaSampleRateHertz: sampleRate,
MediaEncoding: mediaEncoding,
AudioStream: makeAudioStream(audioToSend)
});
var stdResp = await client2.send(stdCmd);
transcript = await collectTranscript(stdResp.TranscriptResultStream);
console.log('[Transcribe] Standard fallback OK — ' + (Date.now() - streamStart) + 'ms, ' + transcript.length + ' chars');
} catch (stdErr) {
logTranscribeError('Standard also failed', stdErr);
throw stdErr;
}
}
} else {
var client = new TranscribeStreamingClient({ region: region, credentials: credentials });
var cmd = new TranscribeModule.StartStreamTranscriptionCommand({
LanguageCode: 'en-US',
MediaSampleRateHertz: sampleRate,
MediaEncoding: mediaEncoding,
AudioStream: makeAudioStream(audioToSend)
});
var resp = await client.send(cmd);
transcript = await collectTranscript(resp.TranscriptResultStream);
console.log('[Transcribe] Standard OK — ' + (Date.now() - streamStart) + 'ms, ' + transcript.length + ' chars');
}
console.log('[Transcribe] Total: ' + (Date.now() - startTime) + 'ms');
return transcript.trim();
}
// Check if AWS Transcribe is available and configured
function isAWSTranscribeConfigured() {
try {
require('@aws-sdk/client-transcribe-streaming');
return !!(process.env.AWS_BEDROCK_REGION || process.env.AWS_REGION);
} catch (e) {
return false;
}
}
module.exports = { transcribeWithAWS, isAWSTranscribeConfigured };

View file

@ -1,45 +0,0 @@
// ============================================================
// TRANSCRIBE GOOGLE — Vertex AI / Gemini audio transcription
// Sends audio inline to Gemini which returns spoken text.
// Supports: gemini-2.0-flash (default), gemini-2.5-flash
// Requires: GOOGLE_VERTEX_PROJECT, @google-cloud/vertexai installed
// ============================================================
async function transcribeWithGemini(buffer, mimeType, modelName) {
var VertexModule = require('@google-cloud/vertexai');
var project = process.env.GOOGLE_VERTEX_PROJECT;
var location = process.env.GOOGLE_VERTEX_LOCATION || 'us-central1';
if (!project) throw new Error('GOOGLE_VERTEX_PROJECT not set');
var vertex = new VertexModule.VertexAI({ project: project, location: location });
var model = vertex.getGenerativeModel({
model: modelName || process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash'
});
var base64 = buffer.toString('base64');
var safeMime = mimeType || 'audio/webm';
var result = await model.generateContent({
contents: [{
role: 'user',
parts: [
{ inlineData: { mimeType: safeMime, data: base64 } },
{ text: 'Transcribe this audio. Output the spoken words only, exactly as heard. No commentary, no formatting, no explanation.' }
]
}]
});
var text = '';
if (result.response && result.response.candidates && result.response.candidates[0]) {
var parts = result.response.candidates[0].content.parts || [];
parts.forEach(function(p) { if (p.text) text += p.text; });
}
return text.trim();
}
function isGoogleSTTConfigured() {
if (!process.env.GOOGLE_VERTEX_PROJECT) return false;
try { require('@google-cloud/vertexai'); return true; } catch(e) { return false; }
}
module.exports = { transcribeWithGemini, isGoogleSTTConfigured };

View file

@ -1,178 +0,0 @@
// ============================================================
// LOCAL WHISPER TRANSCRIPTION — uses whisper.cpp or faster-whisper
// ============================================================
// Requires: whisper.cpp binary or faster-whisper-cli installed on the host.
// Set TRANSCRIBE_PROVIDER=local and optionally WHISPER_MODEL_SIZE, WHISPER_BINARY.
var { execFile } = require('child_process');
var fs = require('fs');
var path = require('path');
var os = require('os');
var crypto = require('crypto');
var logger = require('./logger');
var WHISPER_BINARY = process.env.WHISPER_BINARY || 'whisper-cpp';
var WHISPER_MODEL_SIZE = process.env.WHISPER_MODEL_SIZE || 'small';
var WHISPER_MODEL_PATH = process.env.WHISPER_MODEL_PATH || '';
var WHISPER_LANGUAGE = process.env.WHISPER_LANGUAGE || 'en';
var WHISPER_THREADS = process.env.WHISPER_THREADS || String(Math.max(2, os.cpus().length - 1));
/**
* Check if local Whisper is available
*/
function isLocalWhisperConfigured() {
return process.env.TRANSCRIBE_PROVIDER === 'local';
}
/**
* Detect which whisper binary is available
* Supports: whisper-cpp, whisper, faster-whisper
*/
function detectBinary(callback) {
var candidates = [WHISPER_BINARY, 'whisper-cpp', 'whisper', 'faster-whisper'];
var tried = 0;
function tryNext() {
if (tried >= candidates.length) return callback(null);
var bin = candidates[tried++];
execFile(bin, ['--help'], { timeout: 5000 }, function(err) {
if (!err || (err.code !== 'ENOENT' && err.code !== 127)) return callback(bin);
tryNext();
});
}
tryNext();
}
/**
* Convert audio buffer to WAV using ffmpeg (whisper.cpp needs 16kHz mono WAV)
*/
function convertToWav(inputPath, callback) {
var wavPath = inputPath + '.wav';
execFile('ffmpeg', [
'-i', inputPath,
'-ar', '16000',
'-ac', '1',
'-f', 'wav',
'-y', wavPath
], { timeout: 30000 }, function(err) {
if (err) return callback(err, null);
callback(null, wavPath);
});
}
/**
* Transcribe audio buffer using local Whisper
* @param {Buffer} audioBuffer - Audio data
* @param {string} mimeType - MIME type of the audio
* @returns {Promise<string>} Transcribed text
*/
function transcribeWithLocal(audioBuffer, mimeType) {
return new Promise(function(resolve, reject) {
// Write buffer to temp file
var ext = '.webm';
if (mimeType && mimeType.includes('wav')) ext = '.wav';
else if (mimeType && mimeType.includes('mp3')) ext = '.mp3';
else if (mimeType && mimeType.includes('ogg')) ext = '.ogg';
else if (mimeType && mimeType.includes('mp4')) ext = '.mp4';
var tmpDir = os.tmpdir();
var tmpFile = path.join(tmpDir, 'whisper_' + crypto.randomBytes(8).toString('hex') + ext);
fs.writeFile(tmpFile, audioBuffer, function(err) {
if (err) return reject(new Error('Failed to write temp audio: ' + err.message));
// Convert to WAV for whisper.cpp compatibility
convertToWav(tmpFile, function(convErr, wavPath) {
var audioPath = convErr ? tmpFile : wavPath;
detectBinary(function(binary) {
if (!binary) {
cleanup(tmpFile, wavPath);
return reject(new Error(
'Local Whisper not found. Install whisper.cpp (https://github.com/ggerganov/whisper.cpp) ' +
'or faster-whisper (pip install faster-whisper). Set WHISPER_BINARY env var if using a custom path.'
));
}
var args = buildArgs(binary, audioPath);
logger.info('[LocalWhisper] Running: ' + binary + ' ' + args.join(' '));
execFile(binary, args, { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, function(execErr, stdout, stderr) {
cleanup(tmpFile, wavPath);
if (execErr) {
logger.error('[LocalWhisper] Error:', execErr.message);
if (stderr) logger.error('[LocalWhisper] stderr:', stderr.substring(0, 500));
return reject(new Error('Local Whisper transcription failed: ' + execErr.message));
}
// Parse output — whisper.cpp outputs timestamped lines like [00:00:00.000 --> 00:00:05.000] text
var text = parseOutput(stdout);
if (!text || !text.trim()) {
return reject(new Error('Local Whisper returned empty transcription'));
}
resolve(text.trim());
});
});
});
});
});
}
/**
* Build command-line args based on the detected binary
*/
function buildArgs(binary, audioPath) {
if (binary.includes('faster-whisper')) {
// faster-whisper CLI
var args = ['--model', WHISPER_MODEL_SIZE, '--language', WHISPER_LANGUAGE];
if (WHISPER_THREADS) args.push('--threads', WHISPER_THREADS);
args.push(audioPath);
return args;
}
// whisper.cpp style
var args = ['-f', audioPath, '-l', WHISPER_LANGUAGE, '-t', WHISPER_THREADS, '--no-timestamps'];
if (WHISPER_MODEL_PATH) {
args.push('-m', WHISPER_MODEL_PATH);
} else {
// whisper.cpp uses model names like ggml-small.bin
args.push('-m', 'models/ggml-' + WHISPER_MODEL_SIZE + '.bin');
}
// Medical vocabulary hint via initial prompt
args.push('--prompt', 'Medical patient encounter. Pediatric. Clinical terms, diagnoses, medications.');
return args;
}
/**
* Parse whisper output into clean text
*/
function parseOutput(stdout) {
if (!stdout) return '';
// whisper.cpp with --no-timestamps outputs plain text
// whisper.cpp with timestamps: [00:00:00.000 --> 00:00:05.000] Hello world
var lines = stdout.split('\n');
var text = [];
for (var i = 0; i < lines.length; i++) {
var line = lines[i];
// Strip timestamp prefix if present
var match = line.match(/^\[[\d:.]+\s*-->\s*[\d:.]+\]\s*(.*)$/);
if (match) {
text.push(match[1].trim());
} else if (line.trim() && !line.startsWith('whisper_') && !line.startsWith('main:')) {
// Plain text line (skip whisper.cpp log lines)
text.push(line.trim());
}
}
return text.join(' ');
}
/**
* Clean up temp files
*/
function cleanup(tmpFile, wavPath) {
try { fs.unlinkSync(tmpFile); } catch(e) {}
if (wavPath) { try { fs.unlinkSync(wavPath); } catch(e) {} }
}
module.exports = { transcribeWithLocal, isLocalWhisperConfigured };

View file

@ -1,47 +0,0 @@
// ============================================================
// TTS GOOGLE — Google Cloud Text-to-Speech (direct, no LiteLLM)
// Uses google-auth-library (transitive dep of @google-cloud/vertexai)
// Voices: en-US-Journey-F (female), en-US-Journey-D (male),
// en-US-Studio-O, en-US-Neural2-C, etc.
// Requires: GOOGLE_VERTEX_PROJECT, ADC or GOOGLE_APPLICATION_CREDENTIALS
// ============================================================
async function synthesizeWithGoogleTTS(text, voiceName) {
var GoogleAuth = require('google-auth-library').GoogleAuth;
var axios = require('axios');
var auth = new GoogleAuth({
scopes: ['https://www.googleapis.com/auth/cloud-platform']
});
var client = await auth.getClient();
var tokenData = await client.getAccessToken();
var token = tokenData.token;
var voice = voiceName || process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
var langCode = voice.substring(0, 5); // e.g. en-US
var resp = await axios.post(
'https://texttospeech.googleapis.com/v1/text:synthesize',
{
input: { text: text },
voice: { languageCode: langCode, name: voice },
audioConfig: { audioEncoding: 'MP3' }
},
{
headers: {
'Authorization': 'Bearer ' + token,
'Content-Type': 'application/json'
},
timeout: 30000
}
);
return Buffer.from(resp.data.audioContent, 'base64');
}
function isGoogleTTSConfigured() {
if (!process.env.GOOGLE_VERTEX_PROJECT) return false;
try { require('google-auth-library'); return true; } catch(e) { return false; }
}
module.exports = { synthesizeWithGoogleTTS, isGoogleTTSConfigured };

View file

@ -1,18 +1,16 @@
const { isGoogleTTSConfigured } = require('./ttsGoogle');
const { getLiteLLMHeaders } = require('./litellm');
const GOOGLE_TTS_VOICES = ['en-US-Journey-F','en-US-Journey-D','en-US-Studio-O','en-US-Neural2-A','en-US-Neural2-C','en-US-Neural2-D','en-US-Neural2-E','en-US-Neural2-F','en-US-Neural2-G','en-US-Neural2-H','en-US-Neural2-I','en-US-Neural2-J'];
const OPENAI_TTS_VOICES = ['alloy','echo','fable','onyx','nova','shimmer','ash','sage','coral'];
const ELEVENLABS_DEFAULT_VOICES = ['pNInz6obpgDQGcFmaJgB'];
function parseList(value) {
return String(value || '')
.split(',')
.map(function(item) { return item.trim(); })
.filter(Boolean);
}
function getTTSProvider() {
var env = process.env.TTS_PROVIDER;
if (env === 'google') return 'google';
if (env === 'litellm') return 'litellm';
if (env === 'elevenlabs') return 'elevenlabs';
if (process.env.LITELLM_API_BASE) return 'litellm';
if (isGoogleTTSConfigured()) return 'google';
if (process.env.ELEVENLABS_API_KEY) return 'elevenlabs';
return 'none';
}
@ -22,9 +20,7 @@ function getTTSEnvProvider() {
function getTTSVoiceLists() {
return {
google: GOOGLE_TTS_VOICES,
litellm: OPENAI_TTS_VOICES,
elevenlabs: ELEVENLABS_DEFAULT_VOICES
litellm: parseList(process.env.LITELLM_TTS_VOICES)
};
}
@ -57,16 +53,13 @@ function getLiteLLMTTSDiscoveryItems(models, opts) {
if (opts.currentVoice) {
pushUniqueTTSItem(items, { id: opts.currentVoice, name: opts.currentVoice, source: 'configured-voice', kind: 'voice' });
}
OPENAI_TTS_VOICES.forEach(function(voice) {
pushUniqueTTSItem(items, { id: voice, name: voice, source: 'openai-compatible', kind: 'voice' });
getTTSVoiceLists().litellm.forEach(function(voice) {
pushUniqueTTSItem(items, { id: voice, name: voice, source: 'configured-voice-list', kind: 'voice' });
});
return items;
}
module.exports = {
ELEVENLABS_DEFAULT_VOICES,
GOOGLE_TTS_VOICES,
OPENAI_TTS_VOICES,
getTTSEnvProvider,
getLiteLLMTTSDiscoveryItems,
getLiteLLMHeaders,

View file

@ -24,14 +24,12 @@ test('STT provider respects explicit provider override', () => {
});
});
test('STT provider auto-detects in existing priority order', () => {
test('STT provider only auto-detects LiteLLM', () => {
const sttProvider = require('../src/utils/sttProvider');
withEnv({}, () => {
assert.equal(sttProvider.getSTTProvider({ google: true, aws: true, local: true, litellm: true, openai: true }), 'google');
assert.equal(sttProvider.getSTTProvider({ aws: true, local: true, litellm: true, openai: true }), 'aws');
assert.equal(sttProvider.getSTTProvider({ local: true, litellm: true, openai: true }), 'local');
assert.equal(sttProvider.getSTTProvider({ google: true, aws: true, local: true, litellm: true, openai: true }), 'litellm');
assert.equal(sttProvider.getSTTProvider({ litellm: true, openai: true }), 'litellm');
assert.equal(sttProvider.getSTTProvider({ openai: true }), 'openai');
assert.equal(sttProvider.getSTTProvider({ openai: true }), 'none');
assert.equal(sttProvider.getSTTProvider({}), 'none');
});
});
@ -39,11 +37,7 @@ test('STT provider auto-detects in existing priority order', () => {
test('STT model lists preserve admin defaults', () => {
const sttProvider = require('../src/utils/sttProvider');
assert.deepEqual(sttProvider.getSTTModelLists(), {
google: ['gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.0-flash-lite'],
aws: ['standard', 'medical'],
openai: ['whisper-1'],
litellm: ['local-whisper-large-v3-turbo', 'local-whisper-large-v3', 'local-parakeet-v3', 'local-whisper-1', 'gemini-2.0-flash', 'gemini-2.5-flash'],
local: ['tiny', 'base', 'small', 'medium', 'large']
litellm: ['local-whisper-large-v3-turbo', 'local-whisper-large-v3', 'local-parakeet-v3', 'local-whisper-1', 'gemini-2.0-flash', 'gemini-2.5-flash']
});
});

View file

@ -1,7 +1,7 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const ENV_KEYS = ['TTS_PROVIDER', 'LITELLM_API_BASE', 'LITELLM_API_KEY', 'ELEVENLABS_API_KEY', 'GOOGLE_VERTEX_PROJECT'];
const ENV_KEYS = ['TTS_PROVIDER', 'LITELLM_API_BASE', 'LITELLM_API_KEY', 'LITELLM_TTS_VOICES'];
function withEnv(overrides, fn) {
const previous = {};
@ -25,27 +25,22 @@ test('TTS provider defaults to LiteLLM when gateway is configured', () => {
});
});
test('TTS provider respects explicit provider override', () => {
test('TTS provider ignores non-LiteLLM provider overrides', () => {
const ttsProvider = require('../src/utils/ttsProvider');
withEnv({ TTS_PROVIDER: 'elevenlabs', LITELLM_API_BASE: 'https://llm.example.com' }, () => {
assert.equal(ttsProvider.getTTSEnvProvider(), 'elevenlabs');
assert.equal(ttsProvider.getTTSProvider(), 'elevenlabs');
assert.equal(ttsProvider.getTTSProvider(), 'litellm');
});
});
test('LiteLLM TTS voice list stays OpenAI-compatible', () => {
test('LiteLLM TTS voice list comes from configured voice catalog', () => {
const ttsProvider = require('../src/utils/ttsProvider');
assert.deepEqual(ttsProvider.getTTSVoiceLists().litellm, [
'alloy',
'echo',
'fable',
'onyx',
'nova',
'shimmer',
'ash',
'sage',
'coral'
]);
withEnv({ LITELLM_TTS_VOICES: 'sherpa/kokoro:am_adam, sherpa/kokoro:af_bella' }, () => {
assert.deepEqual(ttsProvider.getTTSVoiceLists().litellm, [
'sherpa/kokoro:am_adam',
'sherpa/kokoro:af_bella'
]);
});
});
test('TTS model discovery uses audio_speech metadata only', () => {
@ -79,20 +74,22 @@ test('LiteLLM TTS model extraction filters gateway model objects', () => {
test('LiteLLM TTS discovery includes metadata models and configured fallbacks', () => {
const ttsProvider = require('../src/utils/ttsProvider');
const items = ttsProvider.getLiteLLMTTSDiscoveryItems([
{ model_name: 'local-kokoro-tts', model_info: { mode: 'audio_speech' } },
{ model_name: 'not-tts-by-name-only' },
{ model_name: 'local-parakeet-v3', model_info: { mode: 'audio_transcription' } }
], {
currentModel: 'local-kokoro-tts',
currentVoice: 'sherpa/kokoro:am_adam'
withEnv({ LITELLM_TTS_VOICES: 'sherpa/kokoro:am_adam,sherpa/kokoro:af_bella' }, () => {
const items = ttsProvider.getLiteLLMTTSDiscoveryItems([
{ model_name: 'local-kokoro-tts', model_info: { mode: 'audio_speech' } },
{ model_name: 'not-tts-by-name-only' },
{ model_name: 'local-parakeet-v3', model_info: { mode: 'audio_transcription' } }
], {
currentModel: 'local-kokoro-tts',
currentVoice: 'sherpa/kokoro:am_adam'
});
assert.deepEqual(items.slice(0, 3), [
{ id: 'local-kokoro-tts', name: 'local-kokoro-tts', source: 'gateway-api', kind: 'model' },
{ id: 'sherpa/kokoro:am_adam', name: 'sherpa/kokoro:am_adam', source: 'configured-voice', kind: 'voice' },
{ id: 'sherpa/kokoro:af_bella', name: 'sherpa/kokoro:af_bella', source: 'configured-voice-list', kind: 'voice' }
]);
assert.equal(items.some(function(item) { return item.id === 'not-tts-by-name-only'; }), false);
});
assert.deepEqual(items.slice(0, 3), [
{ id: 'local-kokoro-tts', name: 'local-kokoro-tts', source: 'gateway-api', kind: 'model' },
{ id: 'sherpa/kokoro:am_adam', name: 'sherpa/kokoro:am_adam', source: 'configured-voice', kind: 'voice' },
{ id: 'alloy', name: 'alloy', source: 'openai-compatible', kind: 'voice' }
]);
assert.equal(items.some(function(item) { return item.id === 'not-tts-by-name-only'; }), false);
});
test('LiteLLM TTS discovery still shows configured model if metadata lookup fails', () => {