From bb459f57b8120437221012767ddf5056ce689e09 Mon Sep 17 00:00:00 2001
From: Daniel Onyejesi <danielonyejesi@gmail.com>
Date: Mon, 30 Mar 2026 19:52:58 -0400
Subject: [PATCH] Fix LiteLLM STT: use chat/completions with Gemini audio
 instead of broken /audio/transcriptions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

LiteLLM /audio/transcriptions gives 'Unmapped provider' for Vertex AI Chirp.
The correct approach: use /v1/chat/completions with a Gemini model and send
audio as base64 input_audio content block — Gemini natively understands audio.
Set LITELLM_STT_MODEL to your Gemini model name (e.g. gemini-2.5-flash).
---
 src/routes/transcribe.js | 53 ++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/src/routes/transcribe.js b/src/routes/transcribe.js
index 3e26994..a4e45e8 100644
--- a/src/routes/transcribe.js
+++ b/src/routes/transcribe.js
@@ -78,24 +78,41 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
 
     if (provider === 'litellm') {
       if (!process.env.LITELLM_API_BASE) return res.status(400).json({ error: 'LITELLM_API_BASE not set.' });
-      // Default to whisper-1 — vertex_ai/chirp does NOT work via LiteLLM audio proxy.
-      // User should set LITELLM_STT_MODEL to their configured whisper alias, or
-      // use TRANSCRIBE_PROVIDER=google for Vertex AI / Gemini transcription instead.
-      var sttModel = process.env.LITELLM_STT_MODEL || 'whisper-1';
-      var FormData = require('form-data');
-      var axios    = require('axios');
-      var form = new FormData();
-      form.append('file', req.file.buffer, { filename: 'audio.webm', contentType: req.file.mimetype || 'audio/webm' });
-      form.append('model', sttModel);
-      var sttHeaders = Object.assign({}, form.getHeaders());
-      if (process.env.LITELLM_API_KEY) sttHeaders['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
-      var sttResp = await require('axios').post(
-        process.env.LITELLM_API_BASE.replace(/\/+$/, '') + '/v1/audio/transcriptions',
-        form, { headers: sttHeaders, timeout: 120000 }
-      );
-      var text = (sttResp.data && sttResp.data.text) ? sttResp.data.text : String(sttResp.data || '');
-      console.log('[Transcribe] LiteLLM (' + sttModel + ') done in ' + (Date.now() - startTime) + 'ms');
-      return res.json({ success: true, text: text, provider: 'litellm-stt', duration: Date.now() - startTime });
+      // LiteLLM /audio/transcriptions does NOT support Vertex AI Chirp (Unmapped provider error).
+      // Instead, use /v1/chat/completions with a Gemini model — Gemini understands audio natively
+      // via base64 inline data. Set LITELLM_STT_MODEL to your Gemini model name in LiteLLM.
+      var sttModel = process.env.LITELLM_STT_MODEL || 'gemini-2.0-flash';
+      var axios = require('axios');
+      var base64audio = req.file.buffer.toString('base64');
+      var mimeType = req.file.mimetype || 'audio/webm';
+      var base = process.env.LITELLM_API_BASE.replace(/\/+$/, '');
+      var headers = { 'Content-Type': 'application/json' };
+      if (process.env.LITELLM_API_KEY) headers['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
+
+      var sttResp = await axios.post(base + '/v1/chat/completions', {
+        model: sttModel,
+        messages: [{
+          role: 'user',
+          content: [
+            {
+              type: 'input_audio',
+              input_audio: { data: base64audio, format: mimeType.split('/')[1] || 'webm' }
+            },
+            {
+              type: 'text',
+              text: 'Transcribe this audio of a medical encounter. Output the spoken words only, exactly as heard. No commentary, no formatting.'
+            }
+          ]
+        }]
+      }, { headers: headers, timeout: 120000 });
+
+      var text = '';
+      if (sttResp.data && sttResp.data.choices && sttResp.data.choices[0]) {
+        var msg = sttResp.data.choices[0].message;
+        text = (msg && msg.content) ? msg.content : '';
+      }
+      console.log('[Transcribe] LiteLLM/' + sttModel + ' done in ' + (Date.now() - startTime) + 'ms');
+      return res.json({ success: true, text: text.trim(), provider: 'litellm-gemini', duration: Date.now() - startTime });
     }
 
     // OpenAI Whisper (direct)