From 0e9ae8e1af4b721c9ee23ae8855170fcd203a9f2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 May 2026 04:46:06 +0200 Subject: [PATCH] fix litellm metadata model discovery --- src/routes/adminConfig.js | 52 +++++++++++++++++++------------- src/utils/embeddings.js | 30 ++++++++++++++++++ src/utils/ttsProvider.js | 7 +---- test/embeddings-provider.test.js | 25 +++++++++++++++ test/tts-provider.test.js | 14 +++------ 5 files changed, 92 insertions(+), 36 deletions(-) create mode 100644 test/embeddings-provider.test.js diff --git a/src/routes/adminConfig.js b/src/routes/adminConfig.js index ab403e8..2811920 100644 --- a/src/routes/adminConfig.js +++ b/src/routes/adminConfig.js @@ -9,9 +9,10 @@ var { authMiddleware, adminMiddleware } = require('../middleware/auth'); var PROMPTS = require('../utils/prompts'); var logger = require('../utils/logger'); var { gatewayUrl } = require('../utils/errors'); -var { GOOGLE_TTS_VOICES, OPENAI_TTS_VOICES, getTTSEnvProvider, getLiteLLMTTSModels, getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider'); +var { GOOGLE_TTS_VOICES, getTTSEnvProvider, getLiteLLMTTSModels, getTTSProvider, getTTSVoiceLists } = require('../utils/ttsProvider'); var { getLiteLLMHeaders } = require('../utils/litellm'); var { getSTTDependencies, getLiteLLMSTTModels, getSTTModelLists, getSTTProvider } = require('../utils/sttProvider'); +var { getLiteLLMEmbeddingModels } = require('../utils/embeddings'); router.use(authMiddleware); @@ -570,7 +571,16 @@ router.post('/config/tts/test', async function(req, res) { var db = require('../db/database'); var adminModel = await db.getSetting('tts.model') || ''; var adminVoice = await db.getSetting('tts.voice') || ''; - if (!adminModel && adminVoice && (adminVoice.indexOf('/') !== -1 || /^(openai-tts|tts-1|openai-gpt.*tts)/i.test(adminVoice))) { adminModel = adminVoice; adminVoice = ''; } + if (!adminModel && adminVoice) { + try { + var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, ''); + var modelInfoResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 }); + if (getLiteLLMTTSModels(modelInfoResp.data && modelInfoResp.data.data).indexOf(adminVoice) !== -1) { + adminModel = adminVoice; + adminVoice = ''; + } + } catch (e) { logger.warn('LiteLLM TTS mode lookup failed: ' + e.message); } + } var resolvedVoice = voice || adminVoice || ''; var rawModel = adminModel || process.env.LITELLM_TTS_MODEL || 'tts-1'; var ttsModel = rawModel; @@ -659,7 +669,6 @@ router.get('/config/stt/discover', async function(req, res) { } if (provider === 'litellm' && process.env.LITELLM_API_BASE) { - // base URL handled by gatewayUrl() helper try { var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, ''); var sResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 }); @@ -723,9 +732,13 @@ router.post('/config/stt/test', async function(req, res) { var db = require('../db/database'); var adminSttModel = await db.getSetting('stt.model') || ''; var sttModel = adminSttModel || process.env.LITELLM_STT_MODEL || 'gemini-2.0-flash'; - // base URL handled by gatewayUrl() helper - var isTranscriptionModel = /whisper|parakeet|deepgram|nova|groq|scribe|elevenlabs|transcri/i.test(sttModel); + var isTranscriptionModel = false; + try { + var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, ''); + var modelInfoResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 }); + isTranscriptionModel = getLiteLLMSTTModels(modelInfoResp.data && modelInfoResp.data.data).indexOf(sttModel) !== -1; + } catch (e) { logger.warn('LiteLLM STT mode lookup failed: ' + e.message); } if (isTranscriptionModel) { var ext = mime.split('/')[1] || 'webm'; @@ -820,23 +833,20 @@ router.get('/config/embeddings/discover', async function(req, res) { ]; if (provider === 'litellm') { - // base URL handled by gatewayUrl() helper try { - var eResp = await axios.get(gatewayUrl('/models'), { headers: getLiteLLMHeaders(), timeout: 10000 }); - if (eResp.data && eResp.data.data) { - eResp.data.data.forEach(function(m) { - if ((m.id || '').toLowerCase().indexOf('embed') !== -1) { - var dims = m.id.indexOf('3-large') !== -1 ? 3072 : m.id.indexOf('3-small') !== -1 ? 1536 : m.id.indexOf('005') !== -1 ? 768 : '?'; - discovered.push({ id: m.id, name: m.id, dims: dims, source: 'gateway-api' }); - } - }); - } - } catch(e) { logger.warn('LiteLLM embedding discovery failed: ' + e.message); } - knownVertex.forEach(function(m) { - if (!discovered.find(function(d) { return d.id === m.id; })) { - discovered.push(Object.assign({ source: 'builtin' }, m)); - } - }); + var liteLLMBase = (process.env.LITELLM_API_BASE || '').replace(/\/+$/, '').replace(/\/v1\/?$/, ''); + var eResp = await axios.get(liteLLMBase + '/model/info', { headers: getLiteLLMHeaders(), timeout: 10000 }); + getLiteLLMEmbeddingModels(eResp.data && eResp.data.data).forEach(function(m) { + discovered.push({ id: m.id, name: m.name, dims: m.dims, source: 'gateway-api' }); + }); + } catch(e) { logger.warn('LiteLLM embedding metadata discovery failed: ' + e.message); } + if (discovered.length === 0) { + knownVertex.forEach(function(m) { + if (!discovered.find(function(d) { return d.id === m.id; })) { + discovered.push(Object.assign({ source: 'builtin' }, m)); + } + }); + } } if (provider === 'vertex') { diff --git a/src/utils/embeddings.js b/src/utils/embeddings.js index 34eaf36..652f957 100644 --- a/src/utils/embeddings.js +++ b/src/utils/embeddings.js @@ -239,6 +239,34 @@ function stripHtml(html) { return html.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim(); } +function getLiteLLMModelId(model) { + if (!model) return ''; + if (typeof model === 'string') return model; + return model.id || model.model_name || ''; +} + +function isLiteLLMEmbeddingModel(model) { + var mode = model && model.model_info && model.model_info.mode ? String(model.model_info.mode) : ''; + return mode === 'embedding'; +} + +function getLiteLLMEmbeddingDimensions(model) { + var info = model && model.model_info ? model.model_info : {}; + var dims = info.output_vector_size || info.dimensions || info.embedding_dimensions || model.output_vector_size || model.dimensions || model.embedding_dimensions; + var parsed = parseInt(dims, 10); + return Number.isFinite(parsed) ? parsed : '?'; +} + +function getLiteLLMEmbeddingModels(models) { + return (models || []) + .filter(isLiteLLMEmbeddingModel) + .map(function(model) { + var id = getLiteLLMModelId(model); + return { id: id, name: id, dims: getLiteLLMEmbeddingDimensions(model) }; + }) + .filter(function(model) { return !!model.id; }); +} + /** * Check if embeddings are available (provider configured) */ @@ -254,8 +282,10 @@ function isEmbeddingsAvailable() { module.exports = { generateEmbedding, generateContentEmbedding, + getLiteLLMEmbeddingModels, searchSimilar, isEmbeddingsAvailable, + isLiteLLMEmbeddingModel, DEFAULT_MODEL, DEFAULT_DIMS }; diff --git a/src/utils/ttsProvider.js b/src/utils/ttsProvider.js index c3490fb..e278de9 100644 --- a/src/utils/ttsProvider.js +++ b/src/utils/ttsProvider.js @@ -28,10 +28,6 @@ function getTTSVoiceLists() { }; } -function isOpenAITTSModel(modelId) { - return /^openai[-/](tts-1|tts-1-hd|gpt-4o-mini-tts|gpt-audio)/.test(String(modelId || '').toLowerCase()); -} - function isLiteLLMTTSModel(model) { var mode = model && model.model_info && model.model_info.mode ? String(model.model_info.mode) : ''; return mode === 'audio_speech'; @@ -52,6 +48,5 @@ module.exports = { getLiteLLMTTSModels, getTTSProvider, getTTSVoiceLists, - isLiteLLMTTSModel, - isOpenAITTSModel + isLiteLLMTTSModel }; diff --git a/test/embeddings-provider.test.js b/test/embeddings-provider.test.js new file mode 100644 index 0000000..a8c361e --- /dev/null +++ b/test/embeddings-provider.test.js @@ -0,0 +1,25 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); + +test('LiteLLM embedding discovery uses embedding metadata only', () => { + const embeddings = require('../src/utils/embeddings'); + assert.equal(embeddings.isLiteLLMEmbeddingModel({ id: 'openai-text-embedding-3-large' }), false); + assert.equal(embeddings.isLiteLLMEmbeddingModel({ id: 'mistral-codestral-embed' }), false); + assert.equal(embeddings.isLiteLLMEmbeddingModel({ model_name: 'openai-text-embedding-3-large', model_info: { mode: 'embedding' } }), true); + assert.equal(embeddings.isLiteLLMEmbeddingModel({ model_name: 'local-parakeet-v3', model_info: { mode: 'audio_transcription' } }), false); +}); + +test('LiteLLM embedding extraction includes metadata dimensions', () => { + const embeddings = require('../src/utils/embeddings'); + assert.deepEqual(embeddings.getLiteLLMEmbeddingModels([ + { model_name: 'openai-text-embedding-3-large', model_info: { mode: 'embedding', output_vector_size: 3072 } }, + { model_name: 'openai-text-embedding-3-small', model_info: { mode: 'embedding', dimensions: '1536' } }, + { model_name: 'mistral-embed', model_info: { mode: 'embedding' } }, + { model_name: 'looks-like-embed' }, + { model_name: 'local-kokoro-tts', model_info: { mode: 'audio_speech' } } + ]), [ + { id: 'openai-text-embedding-3-large', name: 'openai-text-embedding-3-large', dims: 3072 }, + { id: 'openai-text-embedding-3-small', name: 'openai-text-embedding-3-small', dims: 1536 }, + { id: 'mistral-embed', name: 'mistral-embed', dims: '?' } + ]); +}); diff --git a/test/tts-provider.test.js b/test/tts-provider.test.js index 3f269e9..b6791ab 100644 --- a/test/tts-provider.test.js +++ b/test/tts-provider.test.js @@ -48,19 +48,15 @@ test('LiteLLM TTS voice list stays OpenAI-compatible', () => { ]); }); -test('TTS model discovery keeps OpenAI speech models only', () => { - const ttsProvider = require('../src/utils/ttsProvider'); - assert.equal(ttsProvider.isOpenAITTSModel('openai/tts-1'), true); - assert.equal(ttsProvider.isOpenAITTSModel('openai-tts-1-hd'), true); - assert.equal(ttsProvider.isOpenAITTSModel('openai-gpt-4o-mini-tts'), true); - assert.equal(ttsProvider.isOpenAITTSModel('mistral-voxtral-mini-tts'), false); - assert.equal(ttsProvider.isOpenAITTSModel('vertex-gemini-2.5-flash-tts'), false); -}); - test('TTS model discovery uses audio_speech metadata only', () => { const ttsProvider = require('../src/utils/ttsProvider'); + assert.equal(ttsProvider.isOpenAITTSModel, undefined); assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'local-chatterbox-turbo' }), false); assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'local-kokoro-tts' }), false); + assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'openai-tts-1' }), false); + assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'openai-gpt-4o-mini-tts' }), false); + assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'mistral-voxtral-mini-tts' }), false); + assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'vertex-gemini-2.5-flash-tts' }), false); assert.equal(ttsProvider.isLiteLLMTTSModel({ model_name: 'local-chatterbox-turbo', model_info: { mode: 'audio_speech' } }), true); assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'custom-provider-model', model_info: { mode: 'audio_speech' } }), true); assert.equal(ttsProvider.isLiteLLMTTSModel({ id: 'local-parakeet-v3', model_info: { mode: 'audio_transcription' } }), false);