simplify speech and embeddings through litellm
This commit is contained in:
parent
79037fa775
commit
1b3ea569b7
24 changed files with 184 additions and 1242 deletions
|
|
@ -15,6 +15,7 @@ services:
|
|||
TTS_PROVIDER: litellm
|
||||
LITELLM_TTS_MODEL: local-kokoro-tts
|
||||
LITELLM_TTS_VOICE: sherpa/kokoro:am_adam
|
||||
LITELLM_TTS_VOICES: sherpa/kokoro:am_adam,sherpa/kokoro:am_michael,sherpa/kokoro:af_bella,sherpa/kokoro:af_nicole,sherpa/kokoro:bf_emma,sherpa/kokoro:bm_lewis
|
||||
volumes:
|
||||
- scribe-logs:/app/data/logs
|
||||
- clinical-assistant-mcp-data:/app/mcp-data:ro
|
||||
|
|
|
|||
|
|
@ -591,7 +591,7 @@ Check whether speech-to-text transcription is available and which provider is co
|
|||
```json
|
||||
{
|
||||
"available": true,
|
||||
"provider": "google | aws | litellm | openai | local | none"
|
||||
"provider": "litellm | none"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -48,8 +48,8 @@ src/
|
|||
logger.js # audit/api/access + Loki shipper
|
||||
errors.js # generic 500 responder
|
||||
models.js, prompts.js, ai.js # AI provider + model + prompt management
|
||||
embeddings.js # Vertex / LiteLLM / OpenAI embeddings
|
||||
transcribe*.js, tts*.js # STT / TTS provider clients
|
||||
embeddings.js # LiteLLM embeddings
|
||||
transcribe.js, tts.js # LiteLLM STT / TTS routes
|
||||
routes/ # Express routers (auth, hpi, soap, patient education, …)
|
||||
|
||||
public/ # SPA
|
||||
|
|
@ -163,4 +163,4 @@ The clinical assistant can call an external MCP-backed retrieval service. Ped-AI
|
|||
|
||||
## Speech
|
||||
|
||||
Browser Whisper and browser-local Whisper model downloads are removed from runtime. Speech-to-text routes through configured server-side providers. Browser-native Web Speech remains available only when explicitly enabled by user settings and browser support.
|
||||
Browser Whisper and browser-local Whisper model downloads are removed from runtime. Speech-to-text routes through LiteLLM; upstream provider choice belongs in LiteLLM config. Browser-native Web Speech remains available only when explicitly enabled by user settings and browser support.
|
||||
|
|
|
|||
|
|
@ -31,38 +31,31 @@ keys):
|
|||
|---|---|
|
||||
| `AI_PROVIDER` | `openrouter` / `bedrock` / `azure` / `vertex` / `litellm`. If unset, the startup loader uses configured credentials and the last initialized provider in Bedrock → Azure → Vertex → LiteLLM order wins; otherwise OpenRouter is the default. |
|
||||
| `OPENROUTER_API_KEY` | OpenRouter key (not HIPAA-eligible). |
|
||||
| `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | Bedrock / Transcribe / Transcribe-Medical. |
|
||||
| `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | Bedrock chat provider. |
|
||||
| `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY`, `AZURE_DEPLOYMENT_NAME`, `AZURE_OPENAI_API_VERSION` | Azure OpenAI. |
|
||||
| `GOOGLE_VERTEX_PROJECT`, `GOOGLE_VERTEX_LOCATION`, `GOOGLE_APPLICATION_CREDENTIALS` | Vertex AI + Gemini chat/STT/TTS. |
|
||||
| `GOOGLE_VERTEX_PROJECT`, `GOOGLE_VERTEX_LOCATION`, `GOOGLE_APPLICATION_CREDENTIALS` | Vertex AI chat provider. |
|
||||
| `LITELLM_API_BASE`, `LITELLM_API_KEY` | OpenAI-compatible AI gateway (Bifrost, LiteLLM, or similar). |
|
||||
|
||||
### Speech-to-text
|
||||
|
||||
| Variable | Purpose |
|
||||
|---|---|
|
||||
| `TRANSCRIBE_PROVIDER` | `google`, `aws`, `local`, `openai`, `litellm`. Auto-detects if unset. |
|
||||
| `OPENAI_API_KEY` | OpenAI Whisper. |
|
||||
| `GOOGLE_STT_MODEL` | Gemini model used as STT (default `gemini-2.0-flash`). |
|
||||
| `AWS_TRANSCRIBE_MEDICAL` | `true` enables Transcribe Medical. |
|
||||
| `AWS_TRANSCRIBE_SPECIALTY` | `PRIMARYCARE` / `CARDIOLOGY` / `NEUROLOGY` / `ONCOLOGY` / `RADIOLOGY` / `UROLOGY`. |
|
||||
| `WHISPER_BINARY`, `WHISPER_MODEL_SIZE`, `WHISPER_MODEL_PATH`, `WHISPER_LANGUAGE`, `WHISPER_THREADS` | Local whisper.cpp / faster-whisper. |
|
||||
| `TRANSCRIBE_PROVIDER` | Use `litellm`; auto mode uses LiteLLM when configured. |
|
||||
| `LITELLM_STT_MODEL` | Model name for LiteLLM-routed STT. |
|
||||
|
||||
### Text-to-speech
|
||||
|
||||
| Variable | Purpose |
|
||||
|---|---|
|
||||
| `GOOGLE_TTS_VOICE` | Google Cloud TTS voice (e.g. `en-US-Journey-F`). |
|
||||
| `ELEVENLABS_API_KEY` | ElevenLabs (not HIPAA-compliant). |
|
||||
| `LITELLM_TTS_MODEL`, `LITELLM_TTS_VOICE` | LiteLLM-routed TTS. |
|
||||
| `LITELLM_TTS_MODEL`, `LITELLM_TTS_VOICE` | LiteLLM-routed TTS model and default voice. |
|
||||
| `LITELLM_TTS_VOICES` | Comma-separated LiteLLM-compatible voices exposed in voice search and user preferences. |
|
||||
|
||||
### Embeddings
|
||||
|
||||
| Variable | Purpose |
|
||||
|---|---|
|
||||
| `EMBEDDING_MODEL` | Embedding model name (default `vertex_ai/text-embedding-005`). |
|
||||
| `EMBEDDING_DIMENSIONS` | Vector dimensions (default 768). |
|
||||
| `VERTEX_PROJECT`, `GOOGLE_CLOUD_PROJECT` | Direct Vertex embedding project. Embedding utility also works through LiteLLM when `LITELLM_API_BASE` is set. |
|
||||
| `EMBEDDING_MODEL` | LiteLLM embedding model name (default `openai-text-embedding-3-large`). |
|
||||
| `EMBEDDING_DIMENSIONS` | Vector dimensions (default 3072). |
|
||||
|
||||
### Email (SMTP)
|
||||
|
||||
|
|
|
|||
|
|
@ -37,9 +37,9 @@ src/
|
|||
fileType.js magic-byte upload verifier
|
||||
errors.js generic 500 responder
|
||||
logger.js audit + api + access + Loki shipper
|
||||
embeddings.js Vertex / LiteLLM / OpenAI embeddings
|
||||
embeddings.js LiteLLM embeddings
|
||||
notify.js ntfy push
|
||||
transcribe*.js, tts*.js STT / TTS provider clients
|
||||
transcribe.js, tts.js LiteLLM STT / TTS routes
|
||||
routes/ Express routers for auth, AI workflows, education, logs, and user data
|
||||
|
||||
public/
|
||||
|
|
@ -272,8 +272,8 @@ docker exec -w /app pediatric-ai-scribe npm run migrate:new -- add_my_table
|
|||
| `sickVisit.js` | `/api` | Auth | Sick visit |
|
||||
| `milestones.js` | `/api` | Auth | Developmental milestone narratives |
|
||||
| `refine.js` | `/api` | Auth | Refine / shorten / clarify |
|
||||
| `transcribe.js` | `/api` | Auth | STT (5 providers) |
|
||||
| `tts.js` | `/api` | Auth | TTS (3 providers) |
|
||||
| `transcribe.js` | `/api` | Auth | LiteLLM STT |
|
||||
| `tts.js` | `/api` | Auth | LiteLLM TTS |
|
||||
| `encounters.js` | `/api` | Auth | Save / load / optimistic-lock encounters |
|
||||
| `memories.js` | `/api` | Auth | Templates + prompt preferences |
|
||||
| `audioBackups.js` | `/api` | Auth | Encrypted audio retry store |
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ This guide explains how to set up and use the new vector-based semantic search f
|
|||
- **Semantic** (`/api/learning/search/semantic`) - AI-powered vector similarity
|
||||
- **Hybrid** (`/api/learning/search/hybrid`) - Combines both for best results
|
||||
- **Auto-embedding** - Content is automatically vectorized when created/updated
|
||||
- **HIPAA-compliant** - Uses Vertex AI embeddings (BAA available)
|
||||
- **Gateway-routed** - Uses LiteLLM embeddings so provider policy stays in one place
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
|
@ -37,37 +37,22 @@ postgres:
|
|||
# ... rest of your config
|
||||
```
|
||||
|
||||
### 2. Configure Embedding Provider
|
||||
### 2. Configure LiteLLM Embeddings
|
||||
|
||||
Add to your `.env` file:
|
||||
|
||||
```bash
|
||||
# Option 1: Vertex AI (HIPAA-eligible, recommended)
|
||||
EMBEDDING_MODEL=vertex_ai/text-embedding-005
|
||||
EMBEDDING_DIMENSIONS=768
|
||||
VERTEX_PROJECT=your-gcp-project-id
|
||||
VERTEX_LOCATION=us-central1
|
||||
GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
|
||||
|
||||
# Option 2: LiteLLM Proxy (routes to any provider)
|
||||
LITELLM_API_BASE=http://localhost:4000
|
||||
LITELLM_API_KEY=your-key
|
||||
EMBEDDING_MODEL=text-embedding-005 # LiteLLM will route to configured provider
|
||||
|
||||
# Option 3: OpenAI (NOT HIPAA-eligible, fallback only)
|
||||
OPENAI_API_KEY=sk-your-key
|
||||
# Uses text-embedding-3-small and truncates to the 768-dimension column
|
||||
EMBEDDING_MODEL=openai-text-embedding-3-large
|
||||
EMBEDDING_DIMENSIONS=3072
|
||||
```
|
||||
|
||||
## Available Vertex AI Embedding Models
|
||||
## Available Embedding Models
|
||||
|
||||
Tested and working via LiteLLM:
|
||||
The Admin embedding search reads LiteLLM `/model/info` and only shows models with `model_info.mode = "embedding"`. Do not add app-side built-in Vertex/OpenAI embedding lists; configure those choices in LiteLLM.
|
||||
|
||||
| Model | Dimensions | Use Case | HIPAA |
|
||||
|-------|-----------|----------|-------|
|
||||
| **vertex_ai/text-embedding-005** | 768 | English + code (recommended) | Yes |
|
||||
| **vertex_ai/gemini-embedding-001** | 768-3072 | Multilingual + code, best quality | Yes |
|
||||
| **vertex_ai/text-multilingual-embedding-002** | 768 | Multilingual focus | Yes |
|
||||
The local LiteLLM instance currently exposes examples such as `openai-text-embedding-3-large`, `openai-text-embedding-3-small`, and Mistral embedding models. Dimensions are read from LiteLLM metadata when available.
|
||||
|
||||
## Setup Steps
|
||||
|
||||
|
|
@ -113,8 +98,8 @@ Response:
|
|||
"total": 50,
|
||||
"withEmbeddings": 50,
|
||||
"missing": 0,
|
||||
"model": "vertex_ai/text-embedding-005",
|
||||
"dimensions": 768
|
||||
"model": "openai-text-embedding-3-large",
|
||||
"dimensions": 3072
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -148,8 +133,8 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
|
|||
|
||||
1. **Content Creation/Update**:
|
||||
- Text is extracted from `title`, `subject`, and `body` (HTML stripped)
|
||||
- Sent to embedding model (Vertex AI)
|
||||
- Returns 768-dimensional vector
|
||||
- Sent to the configured LiteLLM embedding model
|
||||
- Returns an embedding vector
|
||||
- Stored in `learning_content.embedding` column
|
||||
|
||||
2. **Semantic Search**:
|
||||
|
|
@ -166,11 +151,7 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
|
|||
|
||||
## Cost Estimate
|
||||
|
||||
**Google Vertex AI pricing:**
|
||||
- text-embedding-005: $0.025 per 1M characters
|
||||
- Average article: 10,000 chars = $0.00025
|
||||
- 1,000 articles: ~**$0.25 one-time**
|
||||
- Search queries: ~$0.0000125 per query
|
||||
Embedding cost depends on the upstream configured in LiteLLM.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
|
@ -179,14 +160,12 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
|
|||
- For Docker: Use `pgvector/pgvector:pg16` image
|
||||
|
||||
### "Embeddings not configured"
|
||||
- Verify `.env` has `VERTEX_PROJECT` / `GOOGLE_CLOUD_PROJECT`, `LITELLM_API_BASE`, or `OPENAI_API_KEY`
|
||||
- Check service account credentials: `GOOGLE_APPLICATION_CREDENTIALS`
|
||||
- Verify `.env` has `LITELLM_API_BASE`
|
||||
- Test: `curl http://localhost:3000/api/admin/learning/embeddings/status`
|
||||
|
||||
### "Embedding generation failed"
|
||||
- Check logs for API errors
|
||||
- Verify Vertex AI API is enabled in GCP
|
||||
- Verify service account has `aiplatform.endpoints.predict` permission
|
||||
- Verify LiteLLM `/model/info` shows the selected model with `mode: embedding`
|
||||
- Check content isn't empty (skips empty bodies)
|
||||
|
||||
### "No results from semantic search"
|
||||
|
|
@ -196,7 +175,7 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
|
|||
|
||||
## Performance
|
||||
|
||||
- **Embedding generation**: ~500ms per article (Vertex AI)
|
||||
- **Embedding generation**: latency depends on the LiteLLM upstream
|
||||
- **Search latency**:
|
||||
- Keyword: 10-50ms
|
||||
- Semantic: 20-100ms (with IVFFLAT index)
|
||||
|
|
@ -205,7 +184,7 @@ Combines keyword + semantic for best results. Automatically deduplicates and ran
|
|||
|
||||
## Security And Compliance
|
||||
|
||||
- **HIPAA-eligible**: Vertex AI supports BAA (Business Associate Agreement)
|
||||
- **Compliance**: controlled by the upstream provider configured in LiteLLM
|
||||
- **Data retention**: Embeddings stored in your database only
|
||||
- **No PHI**: Only article content (not patient data) is embedded
|
||||
- **Encryption**: TLS in transit, at-rest encryption via PostgreSQL
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ The bedside tools include a per-user phone extension and pager directory. Entrie
|
|||
|
||||
## Speech
|
||||
|
||||
Final transcription is server-side. Configure Google/Gemini, AWS Transcribe, LiteLLM, or OpenAI Whisper according to your deployment requirements.
|
||||
Final transcription is server-side through LiteLLM. Configure upstream STT providers in LiteLLM rather than in Ped-AI.
|
||||
|
||||
Browser-native Web Speech is only an explicit opt-in preview path. It is not the final clinical transcript and may use browser-vendor cloud services.
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ Browser Whisper and browser-local model workers are removed. Do not expect a pre
|
|||
|
||||
## Text To Speech
|
||||
|
||||
The voice preview button calls the configured TTS provider and plays the returned audio in the browser. If preview is silent, check that a voice is selected, a provider is configured, the user is authenticated, and browser autoplay has not blocked playback.
|
||||
The voice preview button calls LiteLLM TTS and plays the returned audio in the browser. If preview is silent, check that a LiteLLM voice is selected, the gateway is configured, the user is authenticated, and browser autoplay has not blocked playback.
|
||||
|
||||
## Learning Hub
|
||||
|
||||
|
|
|
|||
|
|
@ -54,11 +54,10 @@ as low-priority style/template context. `custom` memories and legacy
|
|||
|
||||
`POST /api/transcribe` accepts one audio file up to 25 MB. Provider selection:
|
||||
|
||||
- explicit `TRANSCRIBE_PROVIDER=google|aws|local|openai|litellm`, or
|
||||
- auto mode: Google/Gemini, then AWS Transcribe, then direct OpenAI Whisper.
|
||||
- explicit `TRANSCRIBE_PROVIDER=litellm`, or
|
||||
- auto mode when `LITELLM_API_BASE` is configured.
|
||||
|
||||
LiteLLM and local STT are explicit choices. Browser Whisper/browser-local model
|
||||
downloads are not part of the runtime.
|
||||
Direct Google, AWS, local Whisper, and OpenAI Whisper branches are not part of the runtime. Browser Whisper/browser-local model downloads are also absent.
|
||||
|
||||
## Browser Web Speech
|
||||
|
||||
|
|
@ -75,9 +74,9 @@ store the failed recording.
|
|||
|
||||
## Text-To-Speech
|
||||
|
||||
`POST /api/text-to-speech` returns `audio/mpeg` and marks the provider in
|
||||
`X-TTS-Provider`. Supported providers include Google Cloud TTS, LiteLLM-routed
|
||||
TTS, and ElevenLabs when configured.
|
||||
`POST /api/text-to-speech` returns audio from LiteLLM and marks the LiteLLM
|
||||
model in `X-TTS-Provider`. Voices are LiteLLM-compatible strings configured by
|
||||
`LITELLM_TTS_VOICES`.
|
||||
|
||||
## Post-Note Helpers
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,13 @@
|
|||
|
||||
## Transcription
|
||||
|
||||
`POST /api/transcribe` accepts `multipart/form-data` with one audio file up to 25 MB. The provider is selected by `TRANSCRIBE_PROVIDER`, or auto-detected from available credentials.
|
||||
`POST /api/transcribe` accepts `multipart/form-data` with one audio file up to 25 MB. Server STT is routed through LiteLLM.
|
||||
|
||||
Provider priority in auto mode is Google/Gemini, then AWS Transcribe, then direct OpenAI Whisper. Set `TRANSCRIBE_PROVIDER=litellm` or `TRANSCRIBE_PROVIDER=local` explicitly for those paths.
|
||||
Set `TRANSCRIBE_PROVIDER=litellm`, `LITELLM_API_BASE`, and `LITELLM_STT_MODEL`. Auto mode also uses LiteLLM when the gateway is configured.
|
||||
|
||||
| Provider | Notes | HIPAA posture |
|
||||
|---|---|---|
|
||||
| Google/Gemini | Uses the configured Vertex/Gemini STT model. | Eligible with the correct Google Cloud agreement. |
|
||||
| AWS Transcribe | Supports standard and Medical mode. | Eligible with the correct AWS agreement. |
|
||||
| LiteLLM | Sends audio through the configured LiteLLM `/audio/transcriptions` backend. | Depends on the selected upstream. |
|
||||
| OpenAI Whisper | Uses `whisper-1` directly when `OPENAI_API_KEY` is configured. | Not HIPAA eligible unless your own agreement says otherwise. |
|
||||
|
||||
Browser Whisper and browser-local Whisper workers are not part of the runtime. Do not add browser model downloads or Transformers.js STT back into the public app.
|
||||
|
||||
|
|
@ -21,13 +18,13 @@ Browser-native Web Speech can show interim text when the user explicitly enables
|
|||
|
||||
## Text To Speech
|
||||
|
||||
`POST /api/text-to-speech` returns `audio/mpeg`. The `X-TTS-Provider` response header identifies the provider used. Requests are limited to 5000 characters.
|
||||
`POST /api/text-to-speech` returns audio from LiteLLM `/audio/speech`. The `X-TTS-Provider` response header identifies the LiteLLM model used. Requests are limited to 5000 characters.
|
||||
|
||||
| Provider | Notes |
|
||||
|---|---|
|
||||
| Google Cloud TTS | Uses Google Cloud voices when configured. |
|
||||
| LiteLLM | Uses `LITELLM_TTS_MODEL` and `LITELLM_TTS_VOICE`. |
|
||||
| ElevenLabs | Available when configured; not HIPAA eligible by default. |
|
||||
|
||||
The admin/user voice pickers read available LiteLLM-compatible voices from `LITELLM_TTS_VOICES`.
|
||||
|
||||
## Audio Backup
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,17 @@
|
|||
# Transcription Options
|
||||
|
||||
Ped-AI currently supports server-side transcription plus an explicit browser Web Speech preview option. Browser Whisper was removed and should not be offered in settings, documentation, public workers, or model download scripts.
|
||||
Ped-AI currently supports server-side transcription through LiteLLM plus an explicit browser Web Speech preview option. Browser Whisper was removed and should not be offered in settings, documentation, public workers, or model download scripts.
|
||||
|
||||
## Recommended Clinical Setup
|
||||
|
||||
Use a server-side provider covered by your compliance requirements.
|
||||
Route STT through LiteLLM and configure the compliant upstream in LiteLLM.
|
||||
|
||||
| Need | Recommended provider |
|
||||
|---|---|
|
||||
| HIPAA-eligible cloud STT | Google/Gemini through Vertex AI or AWS Transcribe with a BAA. |
|
||||
| OpenAI-compatible routing | LiteLLM with a compliant upstream. |
|
||||
| Direct OpenAI Whisper | Only when acceptable for your deployment. |
|
||||
| Server STT | LiteLLM with a compliant upstream. |
|
||||
| Real-time draft preview | Browser Web Speech only with explicit user opt-in and privacy warning. |
|
||||
|
||||
Auto-detect mode currently checks Google/Gemini first, then AWS Transcribe,
|
||||
then direct OpenAI Whisper. Use `TRANSCRIBE_PROVIDER=litellm` or
|
||||
`TRANSCRIBE_PROVIDER=local` when you want those providers.
|
||||
Auto-detect uses LiteLLM when `LITELLM_API_BASE` is configured. Direct Google, AWS, local Whisper, and OpenAI Whisper branches are not part of the app runtime.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
|
@ -23,26 +19,7 @@ then direct OpenAI Whisper. Use `TRANSCRIBE_PROVIDER=litellm` or
|
|||
TRANSCRIBE_PROVIDER=litellm
|
||||
LITELLM_API_BASE=https://your-litellm.example/v1
|
||||
LITELLM_API_KEY=<key>
|
||||
LITELLM_STT_MODEL=whisper-1
|
||||
```
|
||||
|
||||
Other provider examples:
|
||||
|
||||
```env
|
||||
# Google/Gemini
|
||||
TRANSCRIBE_PROVIDER=google
|
||||
GOOGLE_VERTEX_PROJECT=your-project-id
|
||||
GOOGLE_STT_MODEL=gemini-2.0-flash
|
||||
|
||||
# AWS Transcribe
|
||||
TRANSCRIBE_PROVIDER=aws
|
||||
AWS_BEDROCK_REGION=us-east-1
|
||||
AWS_TRANSCRIBE_MEDICAL=true
|
||||
AWS_TRANSCRIBE_SPECIALTY=PRIMARYCARE
|
||||
|
||||
# Direct OpenAI Whisper
|
||||
TRANSCRIBE_PROVIDER=openai
|
||||
OPENAI_API_KEY=<key>
|
||||
LITELLM_STT_MODEL=local-parakeet-v3
|
||||
```
|
||||
|
||||
## Failure Handling
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ var { authMiddleware, adminMiddleware } = require('../middleware/auth');
|
|||
var PROMPTS = require('../utils/prompts');
|
||||
var logger = require('../utils/logger');
|
||||
var { gatewayUrl } = require('../utils/errors');
|
||||
var { GOOGLE_TTS_VOICES, getTTSEnvProvider, getLiteLLMTTSDiscoveryItems, getLiteLLMTTSModels, getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider');
|
||||
var { getTTSEnvProvider, getLiteLLMTTSDiscoveryItems, getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider');
|
||||
var { getLiteLLMHeaders } = require('../utils/litellm');
|
||||
var { getSTTDependencies, getLiteLLMSTTModels, getSTTModelLists, getSTTProvider } = require('../utils/sttProvider');
|
||||
var { getLiteLLMEmbeddingModels } = require('../utils/embeddings');
|
||||
|
|
@ -461,12 +461,11 @@ router.post('/config/image-models/test', async function(req, res) {
|
|||
// ── GET TTS provider status, voice list, and DB overrides ────────────────
|
||||
router.get('/config/tts', async function(req, res) {
|
||||
try {
|
||||
var { isGoogleTTSConfigured } = require('../utils/ttsGoogle');
|
||||
var envProvider = getTTSEnvProvider();
|
||||
var activeProvider = getTTSProvider();
|
||||
var dbVoice = await db.getSetting('tts.voice') || '';
|
||||
var dbModel = await db.getSetting('tts.model') || '';
|
||||
var envVoice = process.env.GOOGLE_TTS_VOICE || process.env.LITELLM_TTS_VOICE || '';
|
||||
var envVoice = process.env.LITELLM_TTS_VOICE || '';
|
||||
var envModel = process.env.LITELLM_TTS_MODEL || '';
|
||||
res.json({
|
||||
success: true,
|
||||
|
|
@ -479,9 +478,7 @@ router.get('/config/tts', async function(req, res) {
|
|||
envVoice: envVoice,
|
||||
envModel: envModel,
|
||||
configured: {
|
||||
google: false,
|
||||
litellm: !!process.env.LITELLM_API_BASE,
|
||||
elevenlabs: false
|
||||
litellm: !!process.env.LITELLM_API_BASE
|
||||
},
|
||||
voices: getTTSVoiceLists()
|
||||
});
|
||||
|
|
@ -497,28 +494,6 @@ router.get('/config/tts/discover', async function(req, res) {
|
|||
|
||||
var provider = getTTSProvider();
|
||||
|
||||
if (provider === 'google') {
|
||||
try {
|
||||
var GoogleAuth = require('google-auth-library').GoogleAuth;
|
||||
var auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] });
|
||||
var client = await auth.getClient();
|
||||
var tokenData = await client.getAccessToken();
|
||||
var gResp = await axios.get('https://texttospeech.googleapis.com/v1/voices?languageCode=en', {
|
||||
headers: { 'Authorization': 'Bearer ' + tokenData.token }, timeout: 10000
|
||||
});
|
||||
if (gResp.data && gResp.data.voices) {
|
||||
gResp.data.voices.forEach(function(v) {
|
||||
discovered.push({ id: v.name, name: v.name + ' (' + (v.ssmlGender || 'NEUTRAL') + ')', gender: v.ssmlGender, source: 'google-api' });
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
logger.warn('Google TTS voice API failed: ' + e.message + ' — using built-in list');
|
||||
GOOGLE_TTS_VOICES.forEach(function(v) {
|
||||
discovered.push({ id: v, name: v, source: 'google-builtin' });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (provider === 'litellm' && process.env.LITELLM_API_BASE) {
|
||||
var dbVoice = await db.getSetting('tts.voice') || '';
|
||||
var dbModel = await db.getSetting('tts.model') || '';
|
||||
|
|
@ -535,19 +510,6 @@ router.get('/config/tts/discover', async function(req, res) {
|
|||
});
|
||||
}
|
||||
|
||||
if (provider === 'elevenlabs' && process.env.ELEVENLABS_API_KEY) {
|
||||
try {
|
||||
var elResp = await axios.get('https://api.elevenlabs.io/v1/voices', {
|
||||
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY }, timeout: 10000
|
||||
});
|
||||
if (elResp.data && elResp.data.voices) {
|
||||
elResp.data.voices.forEach(function(v) {
|
||||
discovered.push({ id: v.voice_id, name: v.name, source: 'elevenlabs-api' });
|
||||
});
|
||||
}
|
||||
} catch (e) { logger.warn('ElevenLabs voice list failed: ' + e.message); }
|
||||
}
|
||||
|
||||
if (search) {
|
||||
discovered = discovered.filter(function(d) {
|
||||
return d.id.toLowerCase().indexOf(search) !== -1 || d.name.toLowerCase().indexOf(search) !== -1;
|
||||
|
|
@ -562,53 +524,24 @@ router.post('/config/tts/test', async function(req, res) {
|
|||
try {
|
||||
var text = ((req.body.text || 'Hello, this is a TTS test for Pediatric AI Scribe.')).substring(0, 500);
|
||||
var voice = req.body.voice;
|
||||
var { synthesizeWithGoogleTTS } = require('../utils/ttsGoogle');
|
||||
var axios = require('axios');
|
||||
|
||||
var provider = getTTSProvider();
|
||||
if (provider === 'none') return res.json({ success: false, error: 'No TTS provider configured' });
|
||||
if (provider !== 'litellm') return res.json({ success: false, error: 'TTS is configured for LiteLLM only' });
|
||||
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
|
||||
|
||||
var buffer, usedVoice = voice;
|
||||
if (provider === 'google') {
|
||||
usedVoice = voice || process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
|
||||
buffer = await synthesizeWithGoogleTTS(text, usedVoice);
|
||||
} else if (provider === 'litellm') {
|
||||
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
|
||||
var db = require('../db/database');
|
||||
var adminModel = await db.getSetting('tts.model') || '';
|
||||
var adminVoice = await db.getSetting('tts.voice') || '';
|
||||
if (!adminModel && adminVoice) {
|
||||
try {
|
||||
var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, '');
|
||||
var modelInfoResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 });
|
||||
if (getLiteLLMTTSModels(modelInfoResp.data && modelInfoResp.data.data).indexOf(adminVoice) !== -1) {
|
||||
adminModel = adminVoice;
|
||||
adminVoice = '';
|
||||
}
|
||||
} catch (e) { logger.warn('LiteLLM TTS mode lookup failed: ' + e.message); }
|
||||
}
|
||||
var resolvedVoice = voice || adminVoice || '';
|
||||
var rawModel = adminModel || process.env.LITELLM_TTS_MODEL || 'tts-1';
|
||||
var ttsModel = rawModel;
|
||||
usedVoice = resolvedVoice || process.env.LITELLM_TTS_VOICE || 'alloy';
|
||||
// base URL handled by gatewayUrl() helper
|
||||
var ttsResp = await axios.post(gatewayUrl('/audio/speech'),
|
||||
{ model: ttsModel, voice: usedVoice, input: text },
|
||||
{ headers: getLiteLLMHeaders('application/json'), responseType: 'arraybuffer', timeout: 60000 }
|
||||
);
|
||||
buffer = Buffer.from(ttsResp.data);
|
||||
} else if (provider === 'elevenlabs') {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return res.json({ success: false, error: 'ElevenLabs not configured' });
|
||||
usedVoice = voice || 'pNInz6obpgDQGcFmaJgB';
|
||||
var elResp = await axios({
|
||||
method: 'POST',
|
||||
url: 'https://api.elevenlabs.io/v1/text-to-speech/' + usedVoice,
|
||||
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY, 'Content-Type': 'application/json' },
|
||||
data: { text: text, model_id: 'eleven_turbo_v2_5', voice_settings: { stability: 0.5, similarity_boost: 0.75 } },
|
||||
responseType: 'arraybuffer'
|
||||
});
|
||||
buffer = Buffer.from(elResp.data);
|
||||
}
|
||||
var adminModel = await db.getSetting('tts.model') || '';
|
||||
var adminVoice = await db.getSetting('tts.voice') || '';
|
||||
var ttsModel = adminModel || process.env.LITELLM_TTS_MODEL || '';
|
||||
var usedVoice = voice || adminVoice || process.env.LITELLM_TTS_VOICE || '';
|
||||
if (!ttsModel) return res.json({ success: false, error: 'No LiteLLM TTS model configured' });
|
||||
|
||||
var ttsResp = await axios.post(gatewayUrl('/audio/speech'),
|
||||
{ model: ttsModel, voice: usedVoice, input: text },
|
||||
{ headers: getLiteLLMHeaders('application/json'), responseType: 'arraybuffer', timeout: 60000 }
|
||||
);
|
||||
var buffer = Buffer.from(ttsResp.data);
|
||||
res.json({ success: true, audio: buffer.toString('base64'), provider: provider, voice: usedVoice });
|
||||
} catch (e) {
|
||||
var detail = e.response && e.response.data
|
||||
|
|
@ -625,7 +558,7 @@ router.get('/config/stt', async function(req, res) {
|
|||
var configured = getSTTDependencies();
|
||||
var activeProvider = getSTTProvider(configured);
|
||||
var dbModel = await db.getSetting('stt.model') || '';
|
||||
var envModel = process.env.GOOGLE_STT_MODEL || process.env.LITELLM_STT_MODEL || '';
|
||||
var envModel = process.env.LITELLM_STT_MODEL || '';
|
||||
res.json({
|
||||
success: true,
|
||||
provider: activeProvider,
|
||||
|
|
@ -644,36 +577,10 @@ router.get('/config/stt/discover', async function(req, res) {
|
|||
try {
|
||||
var search = (req.query.q || '').toLowerCase().trim();
|
||||
var axios = require('axios');
|
||||
var { whisperClient } = require('../utils/ai');
|
||||
var discovered = [];
|
||||
|
||||
var provider = getSTTProvider();
|
||||
|
||||
if (provider === 'google') {
|
||||
['gemini-2.5-flash','gemini-2.0-flash','gemini-2.0-flash-lite','gemini-1.5-pro','gemini-1.5-flash'].forEach(function(m) {
|
||||
discovered.push({ id: m, name: m, source: 'google-builtin' });
|
||||
});
|
||||
}
|
||||
|
||||
if (provider === 'aws') {
|
||||
['standard','medical'].forEach(function(m) {
|
||||
discovered.push({ id: m, name: 'AWS Transcribe ' + m.charAt(0).toUpperCase() + m.slice(1), source: 'aws-builtin' });
|
||||
});
|
||||
}
|
||||
|
||||
if (provider === 'openai' && whisperClient) {
|
||||
try {
|
||||
var oModels = await whisperClient.models.list();
|
||||
oModels.data.forEach(function(m) {
|
||||
if (m.id.toLowerCase().indexOf('whisper') !== -1 || m.id.toLowerCase().indexOf('transcri') !== -1) {
|
||||
discovered.push({ id: m.id, name: m.id, source: 'openai-api' });
|
||||
}
|
||||
});
|
||||
} catch(e) {
|
||||
discovered.push({ id: 'whisper-1', name: 'whisper-1', source: 'openai-builtin' });
|
||||
}
|
||||
}
|
||||
|
||||
if (provider === 'litellm' && process.env.LITELLM_API_BASE) {
|
||||
try {
|
||||
var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, '');
|
||||
|
|
@ -689,14 +596,6 @@ router.get('/config/stt/discover', async function(req, res) {
|
|||
}
|
||||
}
|
||||
|
||||
if (provider === 'local') {
|
||||
var localModel = process.env.WHISPER_MODEL_SIZE || 'base';
|
||||
discovered.push({ id: localModel, name: 'Local Whisper: ' + localModel + ' (configured)', source: 'local-env' });
|
||||
['tiny','base','small','medium','large'].forEach(function(m) {
|
||||
if (m !== localModel) discovered.push({ id: m, name: 'Local Whisper: ' + m, source: 'local-builtin' });
|
||||
});
|
||||
}
|
||||
|
||||
if (search) {
|
||||
discovered = discovered.filter(function(d) {
|
||||
return d.id.toLowerCase().indexOf(search) !== -1 || d.name.toLowerCase().indexOf(search) !== -1;
|
||||
|
|
@ -714,68 +613,27 @@ router.post('/config/stt/test', async function(req, res) {
|
|||
var audioBuffer = Buffer.from(audioBase64, 'base64');
|
||||
var mime = mimeType || 'audio/webm';
|
||||
|
||||
var { transcribeWithGemini } = require('../utils/transcribeGoogle');
|
||||
var { transcribeWithAWS } = require('../utils/transcribeAWS');
|
||||
var { transcribeWithLocal } = require('../utils/transcribeLocal');
|
||||
var { whisperClient } = require('../utils/ai');
|
||||
var axios = require('axios');
|
||||
|
||||
var provider = getSTTProvider();
|
||||
if (provider === 'none') return res.json({ success: false, error: 'No STT provider configured' });
|
||||
if (provider !== 'litellm') return res.json({ success: false, error: 'STT is configured for LiteLLM only' });
|
||||
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
|
||||
|
||||
var start = Date.now();
|
||||
var text = '';
|
||||
|
||||
if (provider === 'google') {
|
||||
var model = process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash';
|
||||
text = await transcribeWithGemini(audioBuffer, mime, model);
|
||||
} else if (provider === 'aws') {
|
||||
text = await transcribeWithAWS(audioBuffer, mime);
|
||||
} else if (provider === 'local') {
|
||||
text = await transcribeWithLocal(audioBuffer, mime);
|
||||
} else if (provider === 'litellm') {
|
||||
if (!process.env.LITELLM_API_BASE) return res.json({ success: false, error: 'LITELLM_API_BASE not set' });
|
||||
var db = require('../db/database');
|
||||
var adminSttModel = await db.getSetting('stt.model') || '';
|
||||
var sttModel = adminSttModel || process.env.LITELLM_STT_MODEL || 'gemini-2.0-flash';
|
||||
var adminSttModel = await db.getSetting('stt.model') || '';
|
||||
var sttModel = adminSttModel || process.env.LITELLM_STT_MODEL || '';
|
||||
if (!sttModel) return res.json({ success: false, error: 'No LiteLLM STT model configured' });
|
||||
|
||||
var isTranscriptionModel = false;
|
||||
try {
|
||||
var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, '');
|
||||
var modelInfoResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 });
|
||||
isTranscriptionModel = getLiteLLMSTTModels(modelInfoResp.data && modelInfoResp.data.data).indexOf(sttModel) !== -1;
|
||||
} catch (e) { logger.warn('LiteLLM STT mode lookup failed: ' + e.message); }
|
||||
|
||||
if (isTranscriptionModel) {
|
||||
var ext = mime.split('/')[1] || 'webm';
|
||||
var file = new File([audioBuffer], 'audio.' + ext, { type: mime });
|
||||
var form = new FormData();
|
||||
form.append('file', file);
|
||||
form.append('model', sttModel);
|
||||
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
|
||||
method: 'POST', headers: getLiteLLMHeaders(), body: form
|
||||
}).then(function(r) { return r.json().then(function(d) { return { data: d }; }); });
|
||||
text = sttResp.data && sttResp.data.text ? sttResp.data.text : '';
|
||||
} else {
|
||||
var sttResp = await axios.post(gatewayUrl('/chat/completions'), {
|
||||
model: sttModel,
|
||||
messages: [{ role: 'user', content: [
|
||||
{ type: 'input_audio', input_audio: { data: audioBase64, format: mime.split('/')[1] || 'webm' } },
|
||||
{ type: 'text', text: 'Transcribe this audio. Output the spoken words only.' }
|
||||
]}]
|
||||
}, { headers: getLiteLLMHeaders('application/json'), timeout: 60000 });
|
||||
if (sttResp.data && sttResp.data.choices && sttResp.data.choices[0]) {
|
||||
text = (sttResp.data.choices[0].message && sttResp.data.choices[0].message.content) || '';
|
||||
}
|
||||
}
|
||||
} else if (provider === 'openai') {
|
||||
if (!whisperClient) return res.json({ success: false, error: 'Whisper not configured' });
|
||||
var file = new File([audioBuffer], 'audio.webm', { type: mime });
|
||||
var wResult = await whisperClient.audio.transcriptions.create({
|
||||
file: file, model: 'whisper-1', language: 'en', response_format: 'text'
|
||||
});
|
||||
text = typeof wResult === 'string' ? wResult : (wResult.text || '');
|
||||
}
|
||||
var ext = mime.split('/')[1] || 'webm';
|
||||
var file = new File([audioBuffer], 'audio.' + ext, { type: mime });
|
||||
var form = new FormData();
|
||||
form.append('file', file);
|
||||
form.append('model', sttModel);
|
||||
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
|
||||
method: 'POST', headers: getLiteLLMHeaders(), body: form
|
||||
}).then(function(r) { return r.json().then(function(d) { return { data: d }; }); });
|
||||
text = sttResp.data && sttResp.data.text ? sttResp.data.text : '';
|
||||
|
||||
res.json({ success: true, text: text.trim(), provider: provider, duration: Date.now() - start });
|
||||
} catch (e) {
|
||||
|
|
@ -794,8 +652,6 @@ router.get('/config/embeddings', async function(req, res) {
|
|||
|
||||
var provider = 'none';
|
||||
if (process.env.LITELLM_API_BASE) provider = 'litellm';
|
||||
else if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT || process.env.GOOGLE_VERTEX_PROJECT) provider = 'vertex';
|
||||
else if (process.env.OPENAI_API_KEY) provider = 'openai';
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
|
|
@ -807,14 +663,7 @@ router.get('/config/embeddings', async function(req, res) {
|
|||
dbDimensions: dbDims,
|
||||
envModel: envModel,
|
||||
envDimensions: envDims,
|
||||
models: [
|
||||
{ id: 'vertex_ai/text-embedding-005', name: 'Vertex text-embedding-005', dims: 768, tag: 'RECOMMENDED' },
|
||||
{ id: 'vertex_ai/gemini-embedding-001', name: 'Vertex gemini-embedding-001', dims: 3072, tag: 'HIGH-DIM' },
|
||||
{ id: 'vertex_ai/text-multilingual-embedding-002', name: 'Vertex multilingual-embedding-002', dims: 768, tag: 'MULTILINGUAL' },
|
||||
{ id: 'text-embedding-3-small', name: 'OpenAI text-embedding-3-small', dims: 1536, tag: 'OPENAI' },
|
||||
{ id: 'text-embedding-3-large', name: 'OpenAI text-embedding-3-large', dims: 3072, tag: 'OPENAI' },
|
||||
{ id: 'text-embedding-ada-002', name: 'OpenAI text-embedding-ada-002', dims: 1536, tag: 'OPENAI' }
|
||||
]
|
||||
models: []
|
||||
});
|
||||
} catch (e) { res.status(500).json({ error: 'Request failed' }); }
|
||||
});
|
||||
|
|
@ -828,15 +677,6 @@ router.get('/config/embeddings/discover', async function(req, res) {
|
|||
|
||||
var provider = 'none';
|
||||
if (process.env.LITELLM_API_BASE) provider = 'litellm';
|
||||
else if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT || process.env.GOOGLE_VERTEX_PROJECT) provider = 'vertex';
|
||||
else if (process.env.OPENAI_API_KEY) provider = 'openai';
|
||||
|
||||
var knownVertex = [
|
||||
{ id: 'vertex_ai/text-embedding-005', name: 'Vertex text-embedding-005', dims: 768 },
|
||||
{ id: 'vertex_ai/gemini-embedding-001', name: 'Vertex gemini-embedding-001', dims: 3072 },
|
||||
{ id: 'vertex_ai/text-multilingual-embedding-002', name: 'Vertex multilingual-embedding-002', dims: 768 },
|
||||
{ id: 'text-embedding-005', name: 'text-embedding-005 (direct)', dims: 768 }
|
||||
];
|
||||
|
||||
if (provider === 'litellm') {
|
||||
try {
|
||||
|
|
@ -846,37 +686,6 @@ router.get('/config/embeddings/discover', async function(req, res) {
|
|||
discovered.push({ id: m.id, name: m.name, dims: m.dims, source: 'gateway-api' });
|
||||
});
|
||||
} catch(e) { logger.warn('LiteLLM embedding metadata discovery failed: ' + e.message); }
|
||||
if (discovered.length === 0) {
|
||||
knownVertex.forEach(function(m) {
|
||||
if (!discovered.find(function(d) { return d.id === m.id; })) {
|
||||
discovered.push(Object.assign({ source: 'builtin' }, m));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (provider === 'vertex') {
|
||||
knownVertex.forEach(function(m) {
|
||||
discovered.push(Object.assign({ source: 'vertex-builtin' }, m));
|
||||
});
|
||||
}
|
||||
|
||||
if (provider === 'openai') {
|
||||
try {
|
||||
var OpenAI = require('openai').OpenAI;
|
||||
var oc = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
||||
var oMods = await oc.models.list();
|
||||
oMods.data.forEach(function(m) {
|
||||
if (m.id.toLowerCase().indexOf('embedding') !== -1) {
|
||||
var dims = m.id === 'text-embedding-3-large' ? 3072 : m.id === 'text-embedding-3-small' ? 1536 : 1536;
|
||||
discovered.push({ id: m.id, name: m.id, dims: dims, source: 'openai-api' });
|
||||
}
|
||||
});
|
||||
} catch(e) {
|
||||
[{ id: 'text-embedding-3-small', dims: 1536 }, { id: 'text-embedding-3-large', dims: 3072 }, { id: 'text-embedding-ada-002', dims: 1536 }].forEach(function(m) {
|
||||
discovered.push({ id: m.id, name: m.id, dims: m.dims, source: 'openai-builtin' });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (search) {
|
||||
|
|
|
|||
|
|
@ -1,52 +1,30 @@
|
|||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const multer = require('multer');
|
||||
const { whisperClient, litellmClient } = require('../utils/ai');
|
||||
const { transcribeWithAWS, isAWSTranscribeConfigured } = require('../utils/transcribeAWS');
|
||||
const { transcribeWithLocal, isLocalWhisperConfigured } = require('../utils/transcribeLocal');
|
||||
const { transcribeWithGemini, isGoogleSTTConfigured } = require('../utils/transcribeGoogle');
|
||||
const { litellmClient } = require('../utils/ai');
|
||||
const { authMiddleware } = require('../middleware/auth');
|
||||
var logger = require('../utils/logger');
|
||||
var { gatewayUrl } = require('../utils/errors');
|
||||
var { getLiteLLMHeaders } = require('../utils/litellm');
|
||||
|
||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } });
|
||||
|
||||
// Provider priority (auto-detect):
|
||||
// TRANSCRIBE_PROVIDER=google → Vertex AI / Gemini (direct, HIPAA eligible)
|
||||
// TRANSCRIBE_PROVIDER=aws → Amazon Transcribe (HIPAA eligible)
|
||||
// TRANSCRIBE_PROVIDER=local → local whisper.cpp / faster-whisper
|
||||
// TRANSCRIBE_PROVIDER=openai → OpenAI Whisper (direct)
|
||||
// TRANSCRIBE_PROVIDER=litellm → LiteLLM proxy at /v1/audio/transcriptions
|
||||
// Auto: google > aws > openai
|
||||
// STT is intentionally routed only through LiteLLM.
|
||||
function getTranscribeProvider() {
|
||||
var env = process.env.TRANSCRIBE_PROVIDER;
|
||||
if (env === 'google') return 'google';
|
||||
if (env === 'aws') return 'aws';
|
||||
if (env === 'local') return 'local';
|
||||
if (env === 'openai') return 'openai';
|
||||
if (env === 'litellm') return 'litellm';
|
||||
// Auto-detect
|
||||
if (isGoogleSTTConfigured()) return 'google';
|
||||
if (isAWSTranscribeConfigured()) return 'aws';
|
||||
return 'openai';
|
||||
if (litellmClient) return 'litellm';
|
||||
return 'none';
|
||||
}
|
||||
|
||||
function isTranscribeAvailable() {
|
||||
if (isGoogleSTTConfigured()) return true;
|
||||
if (isLocalWhisperConfigured()) return true;
|
||||
if (isAWSTranscribeConfigured()) return true;
|
||||
if (litellmClient) return true;
|
||||
if (whisperClient) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
var provider = getTranscribeProvider();
|
||||
var medical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true';
|
||||
var available = isTranscribeAvailable();
|
||||
console.log('🎙️ Transcribe provider:', provider +
|
||||
(provider === 'aws' && medical ? ' (Medical)' : '') +
|
||||
(provider === 'google' ? ' (model: ' + (process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash') + ')' : '') +
|
||||
(available ? '' : ' (NOT CONFIGURED — browser speech only)'));
|
||||
console.log('🎙️ Transcribe provider:', provider + (available ? '' : ' (NOT CONFIGURED — browser speech only)'));
|
||||
|
||||
router.get('/transcribe/status', authMiddleware, (req, res) => {
|
||||
res.json({ available: available, provider: available ? provider : 'none' });
|
||||
|
|
@ -66,68 +44,29 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
|
|||
|
||||
console.log('[Transcribe] Received ' + (fileSize / 1024).toFixed(0) + 'KB audio (' + (req.file.mimetype || 'unknown') + ') via ' + provider + (userModel ? ' (user model: ' + userModel + ')' : ''));
|
||||
|
||||
if (provider === 'google') {
|
||||
var model = userModel || adminSttModel || process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash';
|
||||
var text = await transcribeWithGemini(req.file.buffer, req.file.mimetype || 'audio/webm', model);
|
||||
console.log('[Transcribe] Google/' + model + ' done in ' + (Date.now() - startTime) + 'ms');
|
||||
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via google', req, { category: 'clinical' });
|
||||
return res.json({ success: true, text: text, provider: 'google-' + model, duration: Date.now() - startTime });
|
||||
}
|
||||
if (provider !== 'litellm' || !process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
|
||||
var sttModel = userModel || adminSttModel || process.env.LITELLM_STT_MODEL || '';
|
||||
if (!sttModel) return res.status(400).json({ error: 'No LiteLLM STT model configured.' });
|
||||
var mimeType = req.file.mimetype || 'audio/webm';
|
||||
var ext = mimeType.split('/')[1] || 'webm';
|
||||
|
||||
if (provider === 'local') {
|
||||
var text = await transcribeWithLocal(req.file.buffer, req.file.mimetype || 'audio/webm');
|
||||
console.log('[Transcribe] Local done in ' + (Date.now() - startTime) + 'ms');
|
||||
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via local', req, { category: 'clinical' });
|
||||
return res.json({ success: true, text: text, provider: 'local-whisper', duration: Date.now() - startTime });
|
||||
}
|
||||
var file = new File([req.file.buffer], 'audio.' + ext, { type: mimeType });
|
||||
var form = new FormData();
|
||||
form.append('file', file);
|
||||
form.append('model', sttModel);
|
||||
|
||||
if (provider === 'aws') {
|
||||
if (!isAWSTranscribeConfigured()) return res.status(400).json({ error: 'AWS Transcribe not configured.' });
|
||||
var text = await transcribeWithAWS(req.file.buffer, req.file.mimetype || 'audio/webm');
|
||||
console.log('[Transcribe] AWS done in ' + (Date.now() - startTime) + 'ms');
|
||||
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via aws', req, { category: 'clinical' });
|
||||
return res.json({ success: true, text: text, provider: 'aws-transcribe', duration: Date.now() - startTime });
|
||||
}
|
||||
|
||||
if (provider === 'litellm') {
|
||||
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
|
||||
var sttModel = userModel || adminSttModel || process.env.LITELLM_STT_MODEL || 'whisper-1';
|
||||
var mimeType = req.file.mimetype || 'audio/webm';
|
||||
var ext = mimeType.split('/')[1] || 'webm';
|
||||
|
||||
var file = new File([req.file.buffer], 'audio.' + ext, { type: mimeType });
|
||||
var form = new FormData();
|
||||
form.append('file', file);
|
||||
form.append('model', sttModel);
|
||||
var fetchHeaders = {};
|
||||
if (process.env.LITELLM_API_KEY) fetchHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
|
||||
|
||||
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
|
||||
method: 'POST', headers: fetchHeaders, body: form,
|
||||
});
|
||||
if (!sttResp.ok) {
|
||||
var errBody = await sttResp.text();
|
||||
throw new Error('LiteLLM /audio/transcriptions ' + sttResp.status + ': ' + errBody.substring(0, 500));
|
||||
}
|
||||
var data = await sttResp.json();
|
||||
var text = (data && data.text) ? String(data.text).trim() : '';
|
||||
console.log('[Transcribe] LiteLLM/' + sttModel + ' done in ' + (Date.now() - startTime) + 'ms');
|
||||
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via litellm', req, { category: 'clinical' });
|
||||
return res.json({ success: true, text: text, provider: 'litellm/' + sttModel, duration: Date.now() - startTime });
|
||||
}
|
||||
|
||||
// OpenAI Whisper (direct)
|
||||
if (!whisperClient) return res.status(400).json({ error: 'Whisper not configured. Set OPENAI_API_KEY.' });
|
||||
var file = new File([req.file.buffer], 'audio.webm', { type: req.file.mimetype || 'audio/webm' });
|
||||
var result = await whisperClient.audio.transcriptions.create({
|
||||
file, model: 'whisper-1', language: 'en',
|
||||
response_format: 'text',
|
||||
prompt: 'Medical patient encounter. Pediatric. Clinical terms, diagnoses, medications.'
|
||||
var sttResp = await fetch(gatewayUrl('/audio/transcriptions'), {
|
||||
method: 'POST', headers: getLiteLLMHeaders(), body: form,
|
||||
});
|
||||
var text = typeof result === 'string' ? result : result.text;
|
||||
console.log('[Transcribe] Whisper done in ' + (Date.now() - startTime) + 'ms');
|
||||
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via openai', req, { category: 'clinical' });
|
||||
res.json({ success: true, text: text, provider: 'openai-whisper', duration: Date.now() - startTime });
|
||||
if (!sttResp.ok) {
|
||||
var errBody = await sttResp.text();
|
||||
throw new Error('LiteLLM /audio/transcriptions ' + sttResp.status + ': ' + errBody.substring(0, 500));
|
||||
}
|
||||
var data = await sttResp.json();
|
||||
var text = (data && data.text) ? String(data.text).trim() : '';
|
||||
console.log('[Transcribe] LiteLLM/' + sttModel + ' done in ' + (Date.now() - startTime) + 'ms');
|
||||
logger.audit(req.user.id, 'transcribe', 'Transcribed audio via litellm', req, { category: 'clinical' });
|
||||
return res.json({ success: true, text: text, provider: 'litellm/' + sttModel, duration: Date.now() - startTime });
|
||||
|
||||
} catch (err) {
|
||||
var detail = err.response && err.response.data
|
||||
|
|
|
|||
|
|
@ -1,26 +1,15 @@
|
|||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const axios = require('axios');
|
||||
const { synthesizeWithGoogleTTS } = require('../utils/ttsGoogle');
|
||||
const { authMiddleware } = require('../middleware/auth');
|
||||
var logger = require('../utils/logger');
|
||||
var { gatewayUrl } = require('../utils/errors');
|
||||
var { getTTSProvider } = require('../utils/ttsProvider');
|
||||
var { getLiteLLMHeaders } = require('../utils/litellm');
|
||||
|
||||
// Provider priority (auto-detect):
|
||||
// TTS_PROVIDER=litellm → LiteLLM proxy at /v1/audio/speech
|
||||
// TTS_PROVIDER=google → Google Cloud TTS direct (HIPAA eligible, opt-in)
|
||||
// TTS_PROVIDER=elevenlabs → ElevenLabs direct (not HIPAA, opt-in)
|
||||
// Auto: litellm > google > elevenlabs
|
||||
//
|
||||
// Design: the LiteLLM branch is a pure passthrough — model + voice strings
|
||||
// go straight into the OpenAI-compatible /v1/audio/speech body. No regex
|
||||
// dispatch on model/voice names; the LiteLLM config decides which provider
|
||||
// (OpenAI, ElevenLabs, Vertex, Azure) actually serves the request.
|
||||
// TTS is intentionally routed only through LiteLLM. Provider-specific voice
|
||||
// routing belongs in LiteLLM config, not this app.
|
||||
var ttsProvider = getTTSProvider();
|
||||
console.log('🔊 TTS provider:', ttsProvider +
|
||||
(ttsProvider === 'google' ? ' (voice: ' + (process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F') + ')' : ''));
|
||||
console.log('🔊 TTS provider:', ttsProvider);
|
||||
|
||||
router.post('/text-to-speech', authMiddleware, async (req, res) => {
|
||||
try {
|
||||
|
|
@ -35,51 +24,27 @@ router.post('/text-to-speech', authMiddleware, async (req, res) => {
|
|||
var adminModel = await db.getSetting('tts.model') || '';
|
||||
var resolvedVoice = userVoice || adminVoice || '';
|
||||
|
||||
if (ttsProvider === 'litellm') {
|
||||
if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
|
||||
var ttsModel = adminModel || process.env.LITELLM_TTS_MODEL || 'tts-1';
|
||||
var ttsVoice = resolvedVoice || process.env.LITELLM_TTS_VOICE || 'alloy';
|
||||
var ttsResp = await fetch(gatewayUrl('/audio/speech'), {
|
||||
method: 'POST',
|
||||
headers: getLiteLLMHeaders('application/json'),
|
||||
body: JSON.stringify({ model: ttsModel, input: text, voice: ttsVoice })
|
||||
});
|
||||
if (!ttsResp.ok) {
|
||||
var errBody = await ttsResp.text();
|
||||
throw new Error('LiteLLM /audio/speech ' + ttsResp.status + ': ' + errBody.substring(0, 500));
|
||||
}
|
||||
var audioBuf = Buffer.from(await ttsResp.arrayBuffer());
|
||||
res.set('Content-Type', ttsResp.headers.get('content-type') || 'audio/mpeg');
|
||||
res.set('X-TTS-Provider', 'litellm/' + ttsModel);
|
||||
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
|
||||
return res.send(audioBuf);
|
||||
if (ttsProvider !== 'litellm' || !process.env.LITELLM_API_BASE) {
|
||||
return res.status(400).json({ error: 'TTS not configured. Set LITELLM_API_BASE.' });
|
||||
}
|
||||
var ttsModel = adminModel || process.env.LITELLM_TTS_MODEL || '';
|
||||
var ttsVoice = resolvedVoice || process.env.LITELLM_TTS_VOICE || '';
|
||||
if (!ttsModel) return res.status(400).json({ error: 'No LiteLLM TTS model configured.' });
|
||||
|
||||
if (ttsProvider === 'google') {
|
||||
var voice = resolvedVoice || process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
|
||||
var buffer = await synthesizeWithGoogleTTS(text, voice);
|
||||
res.set('Content-Type', 'audio/mpeg');
|
||||
res.set('X-TTS-Provider', 'google-tts/' + voice);
|
||||
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
|
||||
return res.send(buffer);
|
||||
var ttsResp = await fetch(gatewayUrl('/audio/speech'), {
|
||||
method: 'POST',
|
||||
headers: getLiteLLMHeaders('application/json'),
|
||||
body: JSON.stringify({ model: ttsModel, input: text, voice: ttsVoice })
|
||||
});
|
||||
if (!ttsResp.ok) {
|
||||
var errBody = await ttsResp.text();
|
||||
throw new Error('LiteLLM /audio/speech ' + ttsResp.status + ': ' + errBody.substring(0, 500));
|
||||
}
|
||||
|
||||
if (ttsProvider === 'elevenlabs') {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return res.status(400).json({ error: 'ElevenLabs not configured' });
|
||||
var resp = await axios({
|
||||
method: 'POST',
|
||||
url: 'https://api.elevenlabs.io/v1/text-to-speech/pNInz6obpgDQGcFmaJgB',
|
||||
headers: { 'xi-api-key': process.env.ELEVENLABS_API_KEY, 'Content-Type': 'application/json' },
|
||||
data: { text: text, model_id: 'eleven_turbo_v2_5', voice_settings: { stability: 0.5, similarity_boost: 0.75 } },
|
||||
responseType: 'arraybuffer'
|
||||
});
|
||||
res.set('Content-Type', 'audio/mpeg');
|
||||
res.set('X-TTS-Provider', 'elevenlabs');
|
||||
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
|
||||
return res.send(Buffer.from(resp.data));
|
||||
}
|
||||
|
||||
res.status(400).json({ error: 'TTS not configured. Set LITELLM_API_BASE, GOOGLE_VERTEX_PROJECT, or ELEVENLABS_API_KEY.' });
|
||||
var audioBuf = Buffer.from(await ttsResp.arrayBuffer());
|
||||
res.set('Content-Type', ttsResp.headers.get('content-type') || 'audio/mpeg');
|
||||
res.set('X-TTS-Provider', 'litellm/' + ttsModel);
|
||||
logger.audit(req.user.id, 'text_to_speech', 'TTS generated', req, { category: 'clinical' });
|
||||
return res.send(audioBuf);
|
||||
} catch (err) {
|
||||
var detail = err.response && err.response.data
|
||||
? JSON.stringify(err.response.data).substring(0, 500)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ var express = require('express');
|
|||
var router = express.Router();
|
||||
var db = require('../db/database');
|
||||
var { authMiddleware } = require('../middleware/auth');
|
||||
var { getSTTModelLists, getSTTProvider } = require('../utils/sttProvider');
|
||||
var { getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider');
|
||||
|
||||
router.use(authMiddleware);
|
||||
|
||||
|
|
@ -44,103 +46,11 @@ router.post('/preferences', async function(req, res) {
|
|||
// Get available STT models and TTS voices
|
||||
router.get('/preferences/options', async function(req, res) {
|
||||
try {
|
||||
var provider = process.env.TRANSCRIBE_PROVIDER;
|
||||
var ttsProvider = process.env.TTS_PROVIDER;
|
||||
var provider = getSTTProvider();
|
||||
var ttsProvider = getTTSProvider();
|
||||
|
||||
// Auto-detect providers if not explicitly set
|
||||
if (!provider) {
|
||||
if (process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT) provider = 'google';
|
||||
else if (process.env.AWS_BEDROCK_REGION) provider = 'aws';
|
||||
else if (process.env.LITELLM_API_BASE) provider = 'litellm';
|
||||
else if (process.env.OPENAI_API_KEY) provider = 'openai';
|
||||
}
|
||||
|
||||
if (!ttsProvider) {
|
||||
if (process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT) ttsProvider = 'google';
|
||||
else if (process.env.LITELLM_API_BASE) ttsProvider = 'litellm';
|
||||
else if (process.env.OPENAI_API_KEY) ttsProvider = 'openai';
|
||||
else if (process.env.ELEVENLABS_API_KEY) ttsProvider = 'elevenlabs';
|
||||
}
|
||||
|
||||
// STT Models
|
||||
var sttModels = [];
|
||||
if (provider === 'google') {
|
||||
sttModels = [
|
||||
{ value: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash (Experimental, fastest)' },
|
||||
{ value: 'gemini-2.0-flash', label: 'Gemini 2.0 Flash (Fast, accurate)' },
|
||||
{ value: 'gemini-1.5-flash', label: 'Gemini 1.5 Flash (Stable)' },
|
||||
{ value: 'gemini-1.5-pro', label: 'Gemini 1.5 Pro (Best quality, slower)' }
|
||||
];
|
||||
} else if (provider === 'litellm') {
|
||||
sttModels = [
|
||||
{ value: 'local-whisper-large-v3-turbo', label: 'Local Whisper Large v3 Turbo' },
|
||||
{ value: 'local-whisper-large-v3', label: 'Local Whisper Large v3' },
|
||||
{ value: 'local-parakeet-v3', label: 'Local Parakeet v3' },
|
||||
{ value: 'local-whisper-1', label: 'Local Whisper-1 Alias' },
|
||||
{ value: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash Exp' },
|
||||
{ value: 'gemini-2.0-flash', label: 'Gemini 2.0 Flash' },
|
||||
{ value: 'gemini-1.5-flash', label: 'Gemini 1.5 Flash' },
|
||||
{ value: 'gemini-1.5-pro', label: 'Gemini 1.5 Pro' }
|
||||
];
|
||||
} else if (provider === 'openai') {
|
||||
sttModels = [
|
||||
{ value: 'whisper-1', label: 'Whisper-1 (OpenAI standard)' }
|
||||
];
|
||||
} else if (provider === 'aws') {
|
||||
sttModels = [
|
||||
{ value: 'default', label: 'AWS Transcribe (Standard)' },
|
||||
{ value: 'medical', label: 'AWS Transcribe Medical' }
|
||||
];
|
||||
} else if (provider === 'local') {
|
||||
sttModels = [
|
||||
{ value: 'tiny', label: 'Whisper Tiny (fastest)' },
|
||||
{ value: 'base', label: 'Whisper Base' },
|
||||
{ value: 'small', label: 'Whisper Small' },
|
||||
{ value: 'medium', label: 'Whisper Medium' },
|
||||
{ value: 'large', label: 'Whisper Large (best quality)' }
|
||||
];
|
||||
}
|
||||
|
||||
// TTS Voices
|
||||
var ttsVoices = [];
|
||||
if (ttsProvider === 'google') {
|
||||
ttsVoices = [
|
||||
{ value: 'en-US-Journey-F', label: 'Journey (Female, natural)' },
|
||||
{ value: 'en-US-Journey-D', label: 'Journey (Male, natural)' },
|
||||
{ value: 'en-US-Studio-O', label: 'Studio O (Female, expressive)' },
|
||||
{ value: 'en-US-Studio-M', label: 'Studio M (Male, expressive)' },
|
||||
{ value: 'en-US-Neural2-A', label: 'Neural2 A (Male, standard)' },
|
||||
{ value: 'en-US-Neural2-C', label: 'Neural2 C (Female, standard)' },
|
||||
{ value: 'en-US-Neural2-D', label: 'Neural2 D (Male, standard)' },
|
||||
{ value: 'en-US-Neural2-E', label: 'Neural2 E (Female, standard)' },
|
||||
{ value: 'en-US-Neural2-F', label: 'Neural2 F (Female, standard)' },
|
||||
{ value: 'en-US-Neural2-G', label: 'Neural2 G (Female, standard)' },
|
||||
{ value: 'en-US-Neural2-H', label: 'Neural2 H (Female, standard)' },
|
||||
{ value: 'en-US-Neural2-I', label: 'Neural2 I (Male, standard)' },
|
||||
{ value: 'en-US-Neural2-J', label: 'Neural2 J (Male, standard)' }
|
||||
];
|
||||
} else if (ttsProvider === 'litellm' || ttsProvider === 'openai') {
|
||||
ttsVoices = [
|
||||
{ value: 'alloy', label: 'Alloy (Neutral)' },
|
||||
{ value: 'echo', label: 'Echo (Male)' },
|
||||
{ value: 'fable', label: 'Fable (British Male)' },
|
||||
{ value: 'onyx', label: 'Onyx (Deep Male)' },
|
||||
{ value: 'nova', label: 'Nova (Female)' },
|
||||
{ value: 'shimmer', label: 'Shimmer (Soft Female)' }
|
||||
];
|
||||
} else if (ttsProvider === 'elevenlabs') {
|
||||
ttsVoices = [
|
||||
{ value: 'adam', label: 'Adam (Male, deep)' },
|
||||
{ value: 'rachel', label: 'Rachel (Female, calm)' },
|
||||
{ value: 'domi', label: 'Domi (Female, strong)' },
|
||||
{ value: 'bella', label: 'Bella (Female, soft)' },
|
||||
{ value: 'antoni', label: 'Antoni (Male, deep)' },
|
||||
{ value: 'elli', label: 'Elli (Female, young)' },
|
||||
{ value: 'josh', label: 'Josh (Male, narration)' },
|
||||
{ value: 'arnold', label: 'Arnold (Male, crisp)' },
|
||||
{ value: 'sam', label: 'Sam (Male, raspy)' }
|
||||
];
|
||||
}
|
||||
var sttModels = getSTTModelLists().litellm.map(function(model) { return { value: model, label: model }; });
|
||||
var ttsVoices = getTTSVoiceLists().litellm.map(function(voice) { return { value: voice, label: voice }; });
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
|
|
|
|||
|
|
@ -110,17 +110,6 @@ if (process.env.LITELLM_API_BASE) {
|
|||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// WHISPER CLIENT (always OpenAI, separate from text AI)
|
||||
// ============================================================
|
||||
var whisperClient = null;
|
||||
if (process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY.startsWith('sk-')) {
|
||||
whisperClient = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
||||
console.log('✅ Whisper: configured');
|
||||
} else {
|
||||
console.log('⚠️ Whisper: not configured (set OPENAI_API_KEY)');
|
||||
}
|
||||
|
||||
// Force provider from env if explicitly set
|
||||
if (process.env.AI_PROVIDER) {
|
||||
activeProvider = process.env.AI_PROVIDER;
|
||||
|
|
@ -694,4 +683,4 @@ async function discoverModels() {
|
|||
return discovered;
|
||||
}
|
||||
|
||||
module.exports = { callAI, callAIStream, whisperClient, activeProvider, discoverModels, vertexClient, litellmClient };
|
||||
module.exports = { callAI, callAIStream, activeProvider, discoverModels, vertexClient, litellmClient };
|
||||
|
|
|
|||
|
|
@ -1,18 +1,13 @@
|
|||
// ============================================================
|
||||
// EMBEDDINGS UTILITY — Generate & search with Vertex AI embeddings
|
||||
// Supports: Vertex AI (direct), LiteLLM proxy, OpenAI fallback
|
||||
// EMBEDDINGS UTILITY — Generate & search through LiteLLM embeddings
|
||||
// ============================================================
|
||||
|
||||
var axios = require('axios');
|
||||
var { gatewayUrl } = require('./errors');
|
||||
var { getLiteLLMHeaders } = require('./litellm');
|
||||
|
||||
// Vertex AI embedding models (via LiteLLM or direct)
|
||||
// gemini-embedding-001: 768 dims, multilingual + code, best quality
|
||||
// text-embedding-005: 768 dims, English + code optimized
|
||||
// text-multilingual-embedding-002: 768 dims, multilingual focus
|
||||
var DEFAULT_MODEL = 'vertex_ai/text-embedding-005';
|
||||
var DEFAULT_DIMS = 768;
|
||||
var DEFAULT_MODEL = 'openai-text-embedding-3-large';
|
||||
var DEFAULT_DIMS = 3072;
|
||||
|
||||
/**
|
||||
* Generate embedding for text using configured provider
|
||||
|
|
@ -40,22 +35,11 @@ async function generateEmbedding(text, opts) {
|
|||
throw new Error('Empty text provided for embedding');
|
||||
}
|
||||
|
||||
// Try LiteLLM first if configured
|
||||
if (process.env.LITELLM_API_BASE) {
|
||||
return await generateEmbeddingLiteLLM(truncated, model, dimensions);
|
||||
}
|
||||
|
||||
// Try Vertex AI direct if configured
|
||||
if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT) {
|
||||
return await generateEmbeddingVertexDirect(truncated, model, dimensions);
|
||||
}
|
||||
|
||||
// Fallback to OpenAI if configured
|
||||
if (process.env.OPENAI_API_KEY) {
|
||||
return await generateEmbeddingOpenAI(truncated, model, dimensions);
|
||||
}
|
||||
|
||||
throw new Error('No embedding provider configured. Set LITELLM_API_BASE, VERTEX_PROJECT, or OPENAI_API_KEY');
|
||||
throw new Error('No embedding provider configured. Set LITELLM_API_BASE');
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -68,8 +52,7 @@ async function generateEmbeddingLiteLLM(text, model, dimensions) {
|
|||
input: text
|
||||
};
|
||||
|
||||
// Only include dimensions if model supports it (some models have fixed dims)
|
||||
if (dimensions && model.includes('text-embedding-005')) {
|
||||
if (dimensions) {
|
||||
payload.dimensions = dimensions;
|
||||
}
|
||||
|
||||
|
|
@ -89,80 +72,6 @@ async function generateEmbeddingLiteLLM(text, model, dimensions) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding via Vertex AI direct (using @google-cloud/vertexai)
|
||||
*/
|
||||
async function generateEmbeddingVertexDirect(text, model, dimensions) {
|
||||
try {
|
||||
var { VertexAI } = require('@google-cloud/vertexai');
|
||||
|
||||
var project = process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT;
|
||||
var location = process.env.VERTEX_LOCATION || 'us-central1';
|
||||
|
||||
if (!project) {
|
||||
throw new Error('VERTEX_PROJECT or GOOGLE_CLOUD_PROJECT not set');
|
||||
}
|
||||
|
||||
var vertexAI = new VertexAI({ project: project, location: location });
|
||||
|
||||
// Extract model name (strip vertex_ai/ prefix if present)
|
||||
var modelName = model.replace(/^vertex_ai\//, '');
|
||||
|
||||
// For text-embedding-005, we can specify output dimensions
|
||||
var request = {
|
||||
instances: [{ content: text }]
|
||||
};
|
||||
|
||||
if (dimensions && modelName.includes('text-embedding-005')) {
|
||||
request.parameters = { outputDimensionality: dimensions };
|
||||
}
|
||||
|
||||
// Use predictText API for embeddings
|
||||
var predictionClient = vertexAI.preview.getPredictionServiceClient();
|
||||
var endpoint = `projects/${project}/locations/${location}/publishers/google/models/${modelName}`;
|
||||
|
||||
var [response] = await predictionClient.predict({
|
||||
endpoint: endpoint,
|
||||
instances: [{ content: text }],
|
||||
parameters: request.parameters || {}
|
||||
});
|
||||
|
||||
if (!response || !response.predictions || !response.predictions[0]) {
|
||||
throw new Error('Invalid response from Vertex AI');
|
||||
}
|
||||
|
||||
var prediction = response.predictions[0];
|
||||
return prediction.embeddings?.values || prediction.values || prediction;
|
||||
|
||||
} catch (err) {
|
||||
console.error('[Embeddings] Vertex AI direct error:', err.message);
|
||||
throw new Error('Vertex AI embedding failed: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding via OpenAI (fallback)
|
||||
*/
|
||||
async function generateEmbeddingOpenAI(text, model, dimensions) {
|
||||
try {
|
||||
var openai = require('openai');
|
||||
var client = new openai.OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
||||
|
||||
// Use OpenAI's text-embedding-3-small model (1536 dims by default)
|
||||
var embModel = 'text-embedding-3-small';
|
||||
var response = await client.embeddings.create({
|
||||
model: embModel,
|
||||
input: text,
|
||||
dimensions: dimensions || 768 // OpenAI supports custom dimensions
|
||||
});
|
||||
|
||||
return response.data[0].embedding;
|
||||
} catch (err) {
|
||||
console.error('[Embeddings] OpenAI error:', err.message);
|
||||
throw new Error('OpenAI embedding failed: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for similar content using cosine similarity
|
||||
* @param {string} queryText - Search query
|
||||
|
|
@ -271,12 +180,7 @@ function getLiteLLMEmbeddingModels(models) {
|
|||
* Check if embeddings are available (provider configured)
|
||||
*/
|
||||
function isEmbeddingsAvailable() {
|
||||
return !!(
|
||||
process.env.LITELLM_API_BASE ||
|
||||
process.env.VERTEX_PROJECT ||
|
||||
process.env.GOOGLE_CLOUD_PROJECT ||
|
||||
process.env.OPENAI_API_KEY
|
||||
);
|
||||
return !!process.env.LITELLM_API_BASE;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
|
|
@ -1,43 +1,24 @@
|
|||
const GOOGLE_STT_MODELS = ['gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.0-flash-lite'];
|
||||
const AWS_STT_MODES = ['standard', 'medical'];
|
||||
const OPENAI_STT_MODELS = ['whisper-1'];
|
||||
const LITELLM_STT_MODELS = ['local-whisper-large-v3-turbo', 'local-whisper-large-v3', 'local-parakeet-v3', 'local-whisper-1', 'gemini-2.0-flash', 'gemini-2.5-flash'];
|
||||
const LOCAL_WHISPER_MODELS = ['tiny', 'base', 'small', 'medium', 'large'];
|
||||
|
||||
function getSTTDependencies() {
|
||||
var { isGoogleSTTConfigured } = require('./transcribeGoogle');
|
||||
var { isAWSTranscribeConfigured } = require('./transcribeAWS');
|
||||
var { isLocalWhisperConfigured } = require('./transcribeLocal');
|
||||
var { whisperClient, litellmClient } = require('./ai');
|
||||
var { litellmClient } = require('./ai');
|
||||
return {
|
||||
google: isGoogleSTTConfigured(),
|
||||
aws: isAWSTranscribeConfigured(),
|
||||
local: isLocalWhisperConfigured(),
|
||||
litellm: !!litellmClient,
|
||||
openai: !!whisperClient
|
||||
litellm: !!litellmClient
|
||||
};
|
||||
}
|
||||
|
||||
function getSTTProvider(deps) {
|
||||
var env = process.env.TRANSCRIBE_PROVIDER || 'auto';
|
||||
if (env !== 'auto') return env;
|
||||
if (env === 'litellm') return 'litellm';
|
||||
|
||||
var configured = deps || getSTTDependencies();
|
||||
if (configured.google) return 'google';
|
||||
if (configured.aws) return 'aws';
|
||||
if (configured.local) return 'local';
|
||||
if (configured.litellm) return 'litellm';
|
||||
if (configured.openai) return 'openai';
|
||||
return 'none';
|
||||
}
|
||||
|
||||
function getSTTModelLists() {
|
||||
return {
|
||||
google: GOOGLE_STT_MODELS,
|
||||
aws: AWS_STT_MODES,
|
||||
openai: OPENAI_STT_MODELS,
|
||||
litellm: LITELLM_STT_MODELS,
|
||||
local: LOCAL_WHISPER_MODELS
|
||||
litellm: LITELLM_STT_MODELS
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -53,11 +34,7 @@ function getLiteLLMSTTModels(models) {
|
|||
}
|
||||
|
||||
module.exports = {
|
||||
AWS_STT_MODES,
|
||||
GOOGLE_STT_MODELS,
|
||||
LITELLM_STT_MODELS,
|
||||
LOCAL_WHISPER_MODELS,
|
||||
OPENAI_STT_MODELS,
|
||||
getSTTDependencies,
|
||||
getLiteLLMSTTModels,
|
||||
getSTTModelLists,
|
||||
|
|
|
|||
|
|
@ -1,211 +0,0 @@
|
|||
// ============================================================
|
||||
// TRANSCRIBE-AWS.JS — Amazon Transcribe Streaming
|
||||
// No S3 required. Audio streams directly to AWS.
|
||||
//
|
||||
// Audio conversion: ffmpeg converts browser WebM/Opus → PCM 16kHz
|
||||
// which is the most reliable format for AWS Transcribe. If ffmpeg
|
||||
// is not installed, falls back to sending ogg-opus directly.
|
||||
//
|
||||
// Env vars:
|
||||
// TRANSCRIBE_PROVIDER=aws — use this instead of Whisper
|
||||
// AWS_TRANSCRIBE_MEDICAL=true — use Transcribe Medical (better
|
||||
// accuracy for clinical dictation)
|
||||
// AWS_TRANSCRIBE_SPECIALTY=PRIMARYCARE — medical specialty
|
||||
// Options: PRIMARYCARE, CARDIOLOGY, NEUROLOGY, ONCOLOGY,
|
||||
// RADIOLOGY, UROLOGY
|
||||
// AWS_BEDROCK_REGION — reused for Transcribe region
|
||||
// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — reused credentials
|
||||
// ============================================================
|
||||
|
||||
const { spawn } = require('child_process');
|
||||
|
||||
const CHUNK_SIZE = 8192; // 8 KB per audio chunk (AWS limit ~32KB per event frame)
|
||||
|
||||
// Convert any browser audio (WebM, OGG, etc.) to raw PCM s16le 16kHz mono
|
||||
// using ffmpeg. Returns a Buffer of raw PCM bytes.
|
||||
// Throws if ffmpeg is not installed.
|
||||
function convertToPCM(inputBuffer) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
var ff = spawn('ffmpeg', [
|
||||
'-i', 'pipe:0', // read from stdin
|
||||
'-f', 's16le', // raw signed 16-bit little-endian PCM
|
||||
'-ar', '16000', // 16 kHz — ideal for speech recognition
|
||||
'-ac', '1', // mono
|
||||
'-acodec', 'pcm_s16le',
|
||||
'pipe:1' // write to stdout
|
||||
]);
|
||||
|
||||
var out = [];
|
||||
var errBuf = [];
|
||||
|
||||
ff.stdout.on('data', function(d) { out.push(d); });
|
||||
ff.stderr.on('data', function(d) { errBuf.push(d); }); // ffmpeg logs to stderr, not an error
|
||||
|
||||
ff.on('close', function(code) {
|
||||
if (code !== 0) {
|
||||
reject(new Error('ffmpeg conversion failed (code ' + code + '): ' + Buffer.concat(errBuf).toString().slice(-200)));
|
||||
return;
|
||||
}
|
||||
resolve(Buffer.concat(out));
|
||||
});
|
||||
|
||||
ff.on('error', function(err) {
|
||||
reject(new Error('ffmpeg not found. Install ffmpeg on the server: ' + err.message));
|
||||
});
|
||||
|
||||
ff.stdin.write(inputBuffer);
|
||||
ff.stdin.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function* makeAudioStream(buffer) {
|
||||
// 8KB chunks — larger sizes cause AWS SDK deserialization errors
|
||||
// ("to see the raw response, inspect the hidden field {error}.$response").
|
||||
// No artificial delay between chunks; yield with a microtask break
|
||||
// to avoid blocking the event loop without adding real latency.
|
||||
for (var i = 0; i < buffer.length; i += CHUNK_SIZE) {
|
||||
yield { AudioEvent: { AudioChunk: buffer.slice(i, i + CHUNK_SIZE) } };
|
||||
// Microtask break every 16 chunks (~128KB) to keep event loop responsive
|
||||
if ((i / CHUNK_SIZE) % 16 === 15) {
|
||||
await new Promise(function(r) { setImmediate(r); });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function logTranscribeError(label, err) {
|
||||
console.error('[Transcribe] ' + label + ':', err.message);
|
||||
if (err.name) console.error('[Transcribe] name:', err.name);
|
||||
if (err.$metadata) console.error('[Transcribe] metadata:', JSON.stringify(err.$metadata));
|
||||
if (err.cause) console.error('[Transcribe] cause:', err.cause.message || JSON.stringify(err.cause));
|
||||
if (err.$response) {
|
||||
try {
|
||||
var resp = err.$response;
|
||||
console.error('[Transcribe] status:', resp.statusCode);
|
||||
if (resp.headers) console.error('[Transcribe] headers:', JSON.stringify(resp.headers));
|
||||
if (resp.body) console.error('[Transcribe] body:', typeof resp.body === 'string' ? resp.body.slice(0, 500) : '(stream)');
|
||||
} catch (e) { /* ignore logging errors */ }
|
||||
}
|
||||
}
|
||||
|
||||
async function transcribeWithAWS(audioBuffer, mimeType) {
|
||||
var startTime = Date.now();
|
||||
var TranscribeModule = require('@aws-sdk/client-transcribe-streaming');
|
||||
var TranscribeStreamingClient = TranscribeModule.TranscribeStreamingClient;
|
||||
|
||||
var region = process.env.AWS_BEDROCK_REGION || process.env.AWS_REGION || 'us-east-1';
|
||||
var credentials = process.env.AWS_ACCESS_KEY_ID ? {
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
} : undefined;
|
||||
|
||||
// Try ffmpeg conversion to PCM (most reliable with AWS Transcribe).
|
||||
// Falls back to ogg-opus passthrough if ffmpeg is not installed.
|
||||
var audioToSend = audioBuffer;
|
||||
var mediaEncoding = 'ogg-opus';
|
||||
var sampleRate = 48000;
|
||||
|
||||
try {
|
||||
var ffStart = Date.now();
|
||||
audioToSend = await convertToPCM(audioBuffer);
|
||||
mediaEncoding = 'pcm';
|
||||
sampleRate = 16000;
|
||||
console.log('[Transcribe] ffmpeg: ' + (Date.now() - ffStart) + 'ms — ' + audioBuffer.length + ' → ' + audioToSend.length + ' bytes');
|
||||
} catch (ffErr) {
|
||||
console.warn('[Transcribe] ffmpeg unavailable, sending ogg-opus directly:', ffErr.message);
|
||||
if (mimeType && mimeType.indexOf('wav') !== -1) {
|
||||
mediaEncoding = 'pcm';
|
||||
sampleRate = 16000;
|
||||
}
|
||||
}
|
||||
|
||||
// Minimum ~0.5s of PCM audio at 16kHz mono 16-bit = 16000 bytes
|
||||
if (mediaEncoding === 'pcm' && audioToSend.length < 16000) {
|
||||
console.warn('[Transcribe] Audio too short (' + audioToSend.length + ' bytes), skipping');
|
||||
return '';
|
||||
}
|
||||
|
||||
var useMedical = process.env.AWS_TRANSCRIBE_MEDICAL === 'true';
|
||||
var specialty = process.env.AWS_TRANSCRIBE_SPECIALTY || 'PRIMARYCARE';
|
||||
var transcript = '';
|
||||
|
||||
// Helper to collect transcript from a result stream
|
||||
async function collectTranscript(resultStream) {
|
||||
var text = '';
|
||||
for await (var event of resultStream) {
|
||||
if (event.TranscriptEvent && event.TranscriptEvent.Transcript) {
|
||||
var results = event.TranscriptEvent.Transcript.Results || [];
|
||||
for (var i = 0; i < results.length; i++) {
|
||||
var r = results[i];
|
||||
if (!r.IsPartial && r.Alternatives && r.Alternatives[0]) {
|
||||
text += r.Alternatives[0].Transcript + ' ';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
// Try Medical first if enabled, with fallback to Standard
|
||||
var streamStart = Date.now();
|
||||
if (useMedical) {
|
||||
try {
|
||||
var client = new TranscribeStreamingClient({ region: region, credentials: credentials });
|
||||
var medCmd = new TranscribeModule.StartMedicalStreamTranscriptionCommand({
|
||||
LanguageCode: 'en-US',
|
||||
MediaSampleRateHertz: sampleRate,
|
||||
MediaEncoding: mediaEncoding,
|
||||
Specialty: specialty,
|
||||
Type: 'DICTATION',
|
||||
AudioStream: makeAudioStream(audioToSend)
|
||||
});
|
||||
var medResp = await client.send(medCmd);
|
||||
transcript = await collectTranscript(medResp.TranscriptResultStream);
|
||||
console.log('[Transcribe] Medical OK — ' + (Date.now() - streamStart) + 'ms, ' + transcript.length + ' chars');
|
||||
} catch (medErr) {
|
||||
logTranscribeError('Medical failed', medErr);
|
||||
console.warn('[Transcribe] Falling back to standard...');
|
||||
streamStart = Date.now();
|
||||
try {
|
||||
var client2 = new TranscribeStreamingClient({ region: region, credentials: credentials });
|
||||
var stdCmd = new TranscribeModule.StartStreamTranscriptionCommand({
|
||||
LanguageCode: 'en-US',
|
||||
MediaSampleRateHertz: sampleRate,
|
||||
MediaEncoding: mediaEncoding,
|
||||
AudioStream: makeAudioStream(audioToSend)
|
||||
});
|
||||
var stdResp = await client2.send(stdCmd);
|
||||
transcript = await collectTranscript(stdResp.TranscriptResultStream);
|
||||
console.log('[Transcribe] Standard fallback OK — ' + (Date.now() - streamStart) + 'ms, ' + transcript.length + ' chars');
|
||||
} catch (stdErr) {
|
||||
logTranscribeError('Standard also failed', stdErr);
|
||||
throw stdErr;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var client = new TranscribeStreamingClient({ region: region, credentials: credentials });
|
||||
var cmd = new TranscribeModule.StartStreamTranscriptionCommand({
|
||||
LanguageCode: 'en-US',
|
||||
MediaSampleRateHertz: sampleRate,
|
||||
MediaEncoding: mediaEncoding,
|
||||
AudioStream: makeAudioStream(audioToSend)
|
||||
});
|
||||
var resp = await client.send(cmd);
|
||||
transcript = await collectTranscript(resp.TranscriptResultStream);
|
||||
console.log('[Transcribe] Standard OK — ' + (Date.now() - streamStart) + 'ms, ' + transcript.length + ' chars');
|
||||
}
|
||||
|
||||
console.log('[Transcribe] Total: ' + (Date.now() - startTime) + 'ms');
|
||||
return transcript.trim();
|
||||
}
|
||||
|
||||
// Check if AWS Transcribe is available and configured
|
||||
function isAWSTranscribeConfigured() {
|
||||
try {
|
||||
require('@aws-sdk/client-transcribe-streaming');
|
||||
return !!(process.env.AWS_BEDROCK_REGION || process.env.AWS_REGION);
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { transcribeWithAWS, isAWSTranscribeConfigured };
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
// ============================================================
|
||||
// TRANSCRIBE GOOGLE — Vertex AI / Gemini audio transcription
|
||||
// Sends audio inline to Gemini which returns spoken text.
|
||||
// Supports: gemini-2.0-flash (default), gemini-2.5-flash
|
||||
// Requires: GOOGLE_VERTEX_PROJECT, @google-cloud/vertexai installed
|
||||
// ============================================================
|
||||
|
||||
async function transcribeWithGemini(buffer, mimeType, modelName) {
|
||||
var VertexModule = require('@google-cloud/vertexai');
|
||||
var project = process.env.GOOGLE_VERTEX_PROJECT;
|
||||
var location = process.env.GOOGLE_VERTEX_LOCATION || 'us-central1';
|
||||
if (!project) throw new Error('GOOGLE_VERTEX_PROJECT not set');
|
||||
|
||||
var vertex = new VertexModule.VertexAI({ project: project, location: location });
|
||||
var model = vertex.getGenerativeModel({
|
||||
model: modelName || process.env.GOOGLE_STT_MODEL || 'gemini-2.0-flash'
|
||||
});
|
||||
|
||||
var base64 = buffer.toString('base64');
|
||||
var safeMime = mimeType || 'audio/webm';
|
||||
|
||||
var result = await model.generateContent({
|
||||
contents: [{
|
||||
role: 'user',
|
||||
parts: [
|
||||
{ inlineData: { mimeType: safeMime, data: base64 } },
|
||||
{ text: 'Transcribe this audio. Output the spoken words only, exactly as heard. No commentary, no formatting, no explanation.' }
|
||||
]
|
||||
}]
|
||||
});
|
||||
|
||||
var text = '';
|
||||
if (result.response && result.response.candidates && result.response.candidates[0]) {
|
||||
var parts = result.response.candidates[0].content.parts || [];
|
||||
parts.forEach(function(p) { if (p.text) text += p.text; });
|
||||
}
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
function isGoogleSTTConfigured() {
|
||||
if (!process.env.GOOGLE_VERTEX_PROJECT) return false;
|
||||
try { require('@google-cloud/vertexai'); return true; } catch(e) { return false; }
|
||||
}
|
||||
|
||||
module.exports = { transcribeWithGemini, isGoogleSTTConfigured };
|
||||
|
|
@ -1,178 +0,0 @@
|
|||
// ============================================================
|
||||
// LOCAL WHISPER TRANSCRIPTION — uses whisper.cpp or faster-whisper
|
||||
// ============================================================
|
||||
// Requires: whisper.cpp binary or faster-whisper-cli installed on the host.
|
||||
// Set TRANSCRIBE_PROVIDER=local and optionally WHISPER_MODEL_SIZE, WHISPER_BINARY.
|
||||
|
||||
var { execFile } = require('child_process');
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
var os = require('os');
|
||||
var crypto = require('crypto');
|
||||
var logger = require('./logger');
|
||||
|
||||
var WHISPER_BINARY = process.env.WHISPER_BINARY || 'whisper-cpp';
|
||||
var WHISPER_MODEL_SIZE = process.env.WHISPER_MODEL_SIZE || 'small';
|
||||
var WHISPER_MODEL_PATH = process.env.WHISPER_MODEL_PATH || '';
|
||||
var WHISPER_LANGUAGE = process.env.WHISPER_LANGUAGE || 'en';
|
||||
var WHISPER_THREADS = process.env.WHISPER_THREADS || String(Math.max(2, os.cpus().length - 1));
|
||||
|
||||
/**
|
||||
* Check if local Whisper is available
|
||||
*/
|
||||
function isLocalWhisperConfigured() {
|
||||
return process.env.TRANSCRIBE_PROVIDER === 'local';
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect which whisper binary is available
|
||||
* Supports: whisper-cpp, whisper, faster-whisper
|
||||
*/
|
||||
function detectBinary(callback) {
|
||||
var candidates = [WHISPER_BINARY, 'whisper-cpp', 'whisper', 'faster-whisper'];
|
||||
var tried = 0;
|
||||
function tryNext() {
|
||||
if (tried >= candidates.length) return callback(null);
|
||||
var bin = candidates[tried++];
|
||||
execFile(bin, ['--help'], { timeout: 5000 }, function(err) {
|
||||
if (!err || (err.code !== 'ENOENT' && err.code !== 127)) return callback(bin);
|
||||
tryNext();
|
||||
});
|
||||
}
|
||||
tryNext();
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert audio buffer to WAV using ffmpeg (whisper.cpp needs 16kHz mono WAV)
|
||||
*/
|
||||
function convertToWav(inputPath, callback) {
|
||||
var wavPath = inputPath + '.wav';
|
||||
execFile('ffmpeg', [
|
||||
'-i', inputPath,
|
||||
'-ar', '16000',
|
||||
'-ac', '1',
|
||||
'-f', 'wav',
|
||||
'-y', wavPath
|
||||
], { timeout: 30000 }, function(err) {
|
||||
if (err) return callback(err, null);
|
||||
callback(null, wavPath);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe audio buffer using local Whisper
|
||||
* @param {Buffer} audioBuffer - Audio data
|
||||
* @param {string} mimeType - MIME type of the audio
|
||||
* @returns {Promise<string>} Transcribed text
|
||||
*/
|
||||
function transcribeWithLocal(audioBuffer, mimeType) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
// Write buffer to temp file
|
||||
var ext = '.webm';
|
||||
if (mimeType && mimeType.includes('wav')) ext = '.wav';
|
||||
else if (mimeType && mimeType.includes('mp3')) ext = '.mp3';
|
||||
else if (mimeType && mimeType.includes('ogg')) ext = '.ogg';
|
||||
else if (mimeType && mimeType.includes('mp4')) ext = '.mp4';
|
||||
|
||||
var tmpDir = os.tmpdir();
|
||||
var tmpFile = path.join(tmpDir, 'whisper_' + crypto.randomBytes(8).toString('hex') + ext);
|
||||
|
||||
fs.writeFile(tmpFile, audioBuffer, function(err) {
|
||||
if (err) return reject(new Error('Failed to write temp audio: ' + err.message));
|
||||
|
||||
// Convert to WAV for whisper.cpp compatibility
|
||||
convertToWav(tmpFile, function(convErr, wavPath) {
|
||||
var audioPath = convErr ? tmpFile : wavPath;
|
||||
|
||||
detectBinary(function(binary) {
|
||||
if (!binary) {
|
||||
cleanup(tmpFile, wavPath);
|
||||
return reject(new Error(
|
||||
'Local Whisper not found. Install whisper.cpp (https://github.com/ggerganov/whisper.cpp) ' +
|
||||
'or faster-whisper (pip install faster-whisper). Set WHISPER_BINARY env var if using a custom path.'
|
||||
));
|
||||
}
|
||||
|
||||
var args = buildArgs(binary, audioPath);
|
||||
logger.info('[LocalWhisper] Running: ' + binary + ' ' + args.join(' '));
|
||||
|
||||
execFile(binary, args, { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, function(execErr, stdout, stderr) {
|
||||
cleanup(tmpFile, wavPath);
|
||||
|
||||
if (execErr) {
|
||||
logger.error('[LocalWhisper] Error:', execErr.message);
|
||||
if (stderr) logger.error('[LocalWhisper] stderr:', stderr.substring(0, 500));
|
||||
return reject(new Error('Local Whisper transcription failed: ' + execErr.message));
|
||||
}
|
||||
|
||||
// Parse output — whisper.cpp outputs timestamped lines like [00:00:00.000 --> 00:00:05.000] text
|
||||
var text = parseOutput(stdout);
|
||||
if (!text || !text.trim()) {
|
||||
return reject(new Error('Local Whisper returned empty transcription'));
|
||||
}
|
||||
|
||||
resolve(text.trim());
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build command-line args based on the detected binary
|
||||
*/
|
||||
function buildArgs(binary, audioPath) {
|
||||
if (binary.includes('faster-whisper')) {
|
||||
// faster-whisper CLI
|
||||
var args = ['--model', WHISPER_MODEL_SIZE, '--language', WHISPER_LANGUAGE];
|
||||
if (WHISPER_THREADS) args.push('--threads', WHISPER_THREADS);
|
||||
args.push(audioPath);
|
||||
return args;
|
||||
}
|
||||
|
||||
// whisper.cpp style
|
||||
var args = ['-f', audioPath, '-l', WHISPER_LANGUAGE, '-t', WHISPER_THREADS, '--no-timestamps'];
|
||||
if (WHISPER_MODEL_PATH) {
|
||||
args.push('-m', WHISPER_MODEL_PATH);
|
||||
} else {
|
||||
// whisper.cpp uses model names like ggml-small.bin
|
||||
args.push('-m', 'models/ggml-' + WHISPER_MODEL_SIZE + '.bin');
|
||||
}
|
||||
// Medical vocabulary hint via initial prompt
|
||||
args.push('--prompt', 'Medical patient encounter. Pediatric. Clinical terms, diagnoses, medications.');
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse whisper output into clean text
|
||||
*/
|
||||
function parseOutput(stdout) {
|
||||
if (!stdout) return '';
|
||||
// whisper.cpp with --no-timestamps outputs plain text
|
||||
// whisper.cpp with timestamps: [00:00:00.000 --> 00:00:05.000] Hello world
|
||||
var lines = stdout.split('\n');
|
||||
var text = [];
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
var line = lines[i];
|
||||
// Strip timestamp prefix if present
|
||||
var match = line.match(/^\[[\d:.]+\s*-->\s*[\d:.]+\]\s*(.*)$/);
|
||||
if (match) {
|
||||
text.push(match[1].trim());
|
||||
} else if (line.trim() && !line.startsWith('whisper_') && !line.startsWith('main:')) {
|
||||
// Plain text line (skip whisper.cpp log lines)
|
||||
text.push(line.trim());
|
||||
}
|
||||
}
|
||||
return text.join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up temp files
|
||||
*/
|
||||
function cleanup(tmpFile, wavPath) {
|
||||
try { fs.unlinkSync(tmpFile); } catch(e) {}
|
||||
if (wavPath) { try { fs.unlinkSync(wavPath); } catch(e) {} }
|
||||
}
|
||||
|
||||
module.exports = { transcribeWithLocal, isLocalWhisperConfigured };
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
// ============================================================
|
||||
// TTS GOOGLE — Google Cloud Text-to-Speech (direct, no LiteLLM)
|
||||
// Uses google-auth-library (transitive dep of @google-cloud/vertexai)
|
||||
// Voices: en-US-Journey-F (female), en-US-Journey-D (male),
|
||||
// en-US-Studio-O, en-US-Neural2-C, etc.
|
||||
// Requires: GOOGLE_VERTEX_PROJECT, ADC or GOOGLE_APPLICATION_CREDENTIALS
|
||||
// ============================================================
|
||||
|
||||
async function synthesizeWithGoogleTTS(text, voiceName) {
|
||||
var GoogleAuth = require('google-auth-library').GoogleAuth;
|
||||
var axios = require('axios');
|
||||
|
||||
var auth = new GoogleAuth({
|
||||
scopes: ['https://www.googleapis.com/auth/cloud-platform']
|
||||
});
|
||||
|
||||
var client = await auth.getClient();
|
||||
var tokenData = await client.getAccessToken();
|
||||
var token = tokenData.token;
|
||||
var voice = voiceName || process.env.GOOGLE_TTS_VOICE || 'en-US-Journey-F';
|
||||
var langCode = voice.substring(0, 5); // e.g. en-US
|
||||
|
||||
var resp = await axios.post(
|
||||
'https://texttospeech.googleapis.com/v1/text:synthesize',
|
||||
{
|
||||
input: { text: text },
|
||||
voice: { languageCode: langCode, name: voice },
|
||||
audioConfig: { audioEncoding: 'MP3' }
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
'Authorization': 'Bearer ' + token,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
timeout: 30000
|
||||
}
|
||||
);
|
||||
|
||||
return Buffer.from(resp.data.audioContent, 'base64');
|
||||
}
|
||||
|
||||
function isGoogleTTSConfigured() {
|
||||
if (!process.env.GOOGLE_VERTEX_PROJECT) return false;
|
||||
try { require('google-auth-library'); return true; } catch(e) { return false; }
|
||||
}
|
||||
|
||||
module.exports = { synthesizeWithGoogleTTS, isGoogleTTSConfigured };
|
||||
|
|
@ -1,18 +1,16 @@
|
|||
const { isGoogleTTSConfigured } = require('./ttsGoogle');
|
||||
const { getLiteLLMHeaders } = require('./litellm');
|
||||
|
||||
const GOOGLE_TTS_VOICES = ['en-US-Journey-F','en-US-Journey-D','en-US-Studio-O','en-US-Neural2-A','en-US-Neural2-C','en-US-Neural2-D','en-US-Neural2-E','en-US-Neural2-F','en-US-Neural2-G','en-US-Neural2-H','en-US-Neural2-I','en-US-Neural2-J'];
|
||||
const OPENAI_TTS_VOICES = ['alloy','echo','fable','onyx','nova','shimmer','ash','sage','coral'];
|
||||
const ELEVENLABS_DEFAULT_VOICES = ['pNInz6obpgDQGcFmaJgB'];
|
||||
function parseList(value) {
|
||||
return String(value || '')
|
||||
.split(',')
|
||||
.map(function(item) { return item.trim(); })
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function getTTSProvider() {
|
||||
var env = process.env.TTS_PROVIDER;
|
||||
if (env === 'google') return 'google';
|
||||
if (env === 'litellm') return 'litellm';
|
||||
if (env === 'elevenlabs') return 'elevenlabs';
|
||||
if (process.env.LITELLM_API_BASE) return 'litellm';
|
||||
if (isGoogleTTSConfigured()) return 'google';
|
||||
if (process.env.ELEVENLABS_API_KEY) return 'elevenlabs';
|
||||
return 'none';
|
||||
}
|
||||
|
||||
|
|
@ -22,9 +20,7 @@ function getTTSEnvProvider() {
|
|||
|
||||
function getTTSVoiceLists() {
|
||||
return {
|
||||
google: GOOGLE_TTS_VOICES,
|
||||
litellm: OPENAI_TTS_VOICES,
|
||||
elevenlabs: ELEVENLABS_DEFAULT_VOICES
|
||||
litellm: parseList(process.env.LITELLM_TTS_VOICES)
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -57,16 +53,13 @@ function getLiteLLMTTSDiscoveryItems(models, opts) {
|
|||
if (opts.currentVoice) {
|
||||
pushUniqueTTSItem(items, { id: opts.currentVoice, name: opts.currentVoice, source: 'configured-voice', kind: 'voice' });
|
||||
}
|
||||
OPENAI_TTS_VOICES.forEach(function(voice) {
|
||||
pushUniqueTTSItem(items, { id: voice, name: voice, source: 'openai-compatible', kind: 'voice' });
|
||||
getTTSVoiceLists().litellm.forEach(function(voice) {
|
||||
pushUniqueTTSItem(items, { id: voice, name: voice, source: 'configured-voice-list', kind: 'voice' });
|
||||
});
|
||||
return items;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ELEVENLABS_DEFAULT_VOICES,
|
||||
GOOGLE_TTS_VOICES,
|
||||
OPENAI_TTS_VOICES,
|
||||
getTTSEnvProvider,
|
||||
getLiteLLMTTSDiscoveryItems,
|
||||
getLiteLLMHeaders,
|
||||
|
|
|
|||
|
|
@ -24,14 +24,12 @@ test('STT provider respects explicit provider override', () => {
|
|||
});
|
||||
});
|
||||
|
||||
test('STT provider auto-detects in existing priority order', () => {
|
||||
test('STT provider only auto-detects LiteLLM', () => {
|
||||
const sttProvider = require('../src/utils/sttProvider');
|
||||
withEnv({}, () => {
|
||||
assert.equal(sttProvider.getSTTProvider({ google: true, aws: true, local: true, litellm: true, openai: true }), 'google');
|
||||
assert.equal(sttProvider.getSTTProvider({ aws: true, local: true, litellm: true, openai: true }), 'aws');
|
||||
assert.equal(sttProvider.getSTTProvider({ local: true, litellm: true, openai: true }), 'local');
|
||||
assert.equal(sttProvider.getSTTProvider({ google: true, aws: true, local: true, litellm: true, openai: true }), 'litellm');
|
||||
assert.equal(sttProvider.getSTTProvider({ litellm: true, openai: true }), 'litellm');
|
||||
assert.equal(sttProvider.getSTTProvider({ openai: true }), 'openai');
|
||||
assert.equal(sttProvider.getSTTProvider({ openai: true }), 'none');
|
||||
assert.equal(sttProvider.getSTTProvider({}), 'none');
|
||||
});
|
||||
});
|
||||
|
|
@ -39,11 +37,7 @@ test('STT provider auto-detects in existing priority order', () => {
|
|||
test('STT model lists preserve admin defaults', () => {
|
||||
const sttProvider = require('../src/utils/sttProvider');
|
||||
assert.deepEqual(sttProvider.getSTTModelLists(), {
|
||||
google: ['gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.0-flash-lite'],
|
||||
aws: ['standard', 'medical'],
|
||||
openai: ['whisper-1'],
|
||||
litellm: ['local-whisper-large-v3-turbo', 'local-whisper-large-v3', 'local-parakeet-v3', 'local-whisper-1', 'gemini-2.0-flash', 'gemini-2.5-flash'],
|
||||
local: ['tiny', 'base', 'small', 'medium', 'large']
|
||||
litellm: ['local-whisper-large-v3-turbo', 'local-whisper-large-v3', 'local-parakeet-v3', 'local-whisper-1', 'gemini-2.0-flash', 'gemini-2.5-flash']
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
const { test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const ENV_KEYS = ['TTS_PROVIDER', 'LITELLM_API_BASE', 'LITELLM_API_KEY', 'ELEVENLABS_API_KEY', 'GOOGLE_VERTEX_PROJECT'];
|
||||
const ENV_KEYS = ['TTS_PROVIDER', 'LITELLM_API_BASE', 'LITELLM_API_KEY', 'LITELLM_TTS_VOICES'];
|
||||
|
||||
function withEnv(overrides, fn) {
|
||||
const previous = {};
|
||||
|
|
@ -25,27 +25,22 @@ test('TTS provider defaults to LiteLLM when gateway is configured', () => {
|
|||
});
|
||||
});
|
||||
|
||||
test('TTS provider respects explicit provider override', () => {
|
||||
test('TTS provider ignores non-LiteLLM provider overrides', () => {
|
||||
const ttsProvider = require('../src/utils/ttsProvider');
|
||||
withEnv({ TTS_PROVIDER: 'elevenlabs', LITELLM_API_BASE: 'https://llm.example.com' }, () => {
|
||||
assert.equal(ttsProvider.getTTSEnvProvider(), 'elevenlabs');
|
||||
assert.equal(ttsProvider.getTTSProvider(), 'elevenlabs');
|
||||
assert.equal(ttsProvider.getTTSProvider(), 'litellm');
|
||||
});
|
||||
});
|
||||
|
||||
test('LiteLLM TTS voice list stays OpenAI-compatible', () => {
|
||||
test('LiteLLM TTS voice list comes from configured voice catalog', () => {
|
||||
const ttsProvider = require('../src/utils/ttsProvider');
|
||||
assert.deepEqual(ttsProvider.getTTSVoiceLists().litellm, [
|
||||
'alloy',
|
||||
'echo',
|
||||
'fable',
|
||||
'onyx',
|
||||
'nova',
|
||||
'shimmer',
|
||||
'ash',
|
||||
'sage',
|
||||
'coral'
|
||||
]);
|
||||
withEnv({ LITELLM_TTS_VOICES: 'sherpa/kokoro:am_adam, sherpa/kokoro:af_bella' }, () => {
|
||||
assert.deepEqual(ttsProvider.getTTSVoiceLists().litellm, [
|
||||
'sherpa/kokoro:am_adam',
|
||||
'sherpa/kokoro:af_bella'
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('TTS model discovery uses audio_speech metadata only', () => {
|
||||
|
|
@ -79,20 +74,22 @@ test('LiteLLM TTS model extraction filters gateway model objects', () => {
|
|||
|
||||
test('LiteLLM TTS discovery includes metadata models and configured fallbacks', () => {
|
||||
const ttsProvider = require('../src/utils/ttsProvider');
|
||||
const items = ttsProvider.getLiteLLMTTSDiscoveryItems([
|
||||
{ model_name: 'local-kokoro-tts', model_info: { mode: 'audio_speech' } },
|
||||
{ model_name: 'not-tts-by-name-only' },
|
||||
{ model_name: 'local-parakeet-v3', model_info: { mode: 'audio_transcription' } }
|
||||
], {
|
||||
currentModel: 'local-kokoro-tts',
|
||||
currentVoice: 'sherpa/kokoro:am_adam'
|
||||
withEnv({ LITELLM_TTS_VOICES: 'sherpa/kokoro:am_adam,sherpa/kokoro:af_bella' }, () => {
|
||||
const items = ttsProvider.getLiteLLMTTSDiscoveryItems([
|
||||
{ model_name: 'local-kokoro-tts', model_info: { mode: 'audio_speech' } },
|
||||
{ model_name: 'not-tts-by-name-only' },
|
||||
{ model_name: 'local-parakeet-v3', model_info: { mode: 'audio_transcription' } }
|
||||
], {
|
||||
currentModel: 'local-kokoro-tts',
|
||||
currentVoice: 'sherpa/kokoro:am_adam'
|
||||
});
|
||||
assert.deepEqual(items.slice(0, 3), [
|
||||
{ id: 'local-kokoro-tts', name: 'local-kokoro-tts', source: 'gateway-api', kind: 'model' },
|
||||
{ id: 'sherpa/kokoro:am_adam', name: 'sherpa/kokoro:am_adam', source: 'configured-voice', kind: 'voice' },
|
||||
{ id: 'sherpa/kokoro:af_bella', name: 'sherpa/kokoro:af_bella', source: 'configured-voice-list', kind: 'voice' }
|
||||
]);
|
||||
assert.equal(items.some(function(item) { return item.id === 'not-tts-by-name-only'; }), false);
|
||||
});
|
||||
assert.deepEqual(items.slice(0, 3), [
|
||||
{ id: 'local-kokoro-tts', name: 'local-kokoro-tts', source: 'gateway-api', kind: 'model' },
|
||||
{ id: 'sherpa/kokoro:am_adam', name: 'sherpa/kokoro:am_adam', source: 'configured-voice', kind: 'voice' },
|
||||
{ id: 'alloy', name: 'alloy', source: 'openai-compatible', kind: 'voice' }
|
||||
]);
|
||||
assert.equal(items.some(function(item) { return item.id === 'not-tts-by-name-only'; }), false);
|
||||
});
|
||||
|
||||
test('LiteLLM TTS discovery still shows configured model if metadata lookup fails', () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue