diff --git a/src/lib/server/tts/audio-format.ts b/src/lib/server/tts/audio-format.ts new file mode 100644 index 0000000..bd537e0 --- /dev/null +++ b/src/lib/server/tts/audio-format.ts @@ -0,0 +1,161 @@ +import { spawn } from 'child_process'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { serverLogger } from '@/lib/server/logger'; + +export type SniffedAudioFormat = 'mp3' | 'wav' | 'ogg' | 'flac' | 'aac' | 'unknown'; + +/** + * Detect an audio container/codec from a buffer's leading bytes (magic numbers). + * + * Biased toward only reporting `mp3` when we are confident, so the common path + * (a real mp3 from the upstream) skips transcoding, while anything ambiguous is + * normalized through ffmpeg rather than mislabeled as mp3. + */ +export function sniffAudioFormat(buffer: Buffer): SniffedAudioFormat { + if (buffer.length < 4) return 'unknown'; + + // RIFF....WAVE + if ( + buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46 && + buffer.length >= 12 && + buffer[8] === 0x57 && buffer[9] === 0x41 && buffer[10] === 0x56 && buffer[11] === 0x45 + ) { + return 'wav'; + } + + // OggS + if (buffer[0] === 0x4f && buffer[1] === 0x67 && buffer[2] === 0x67 && buffer[3] === 0x53) { + return 'ogg'; + } + + // fLaC + if (buffer[0] === 0x66 && buffer[1] === 0x4c && buffer[2] === 0x61 && buffer[3] === 0x43) { + return 'flac'; + } + + // ID3-tagged mp3 + if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) { + return 'mp3'; + } + + // MPEG/AAC frame sync: 11 bits set (0xFFE_). Disambiguate mp3 vs ADTS-AAC via + // the 2 layer bits — mp3 never uses layer `00`, ADTS always does. + if (buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) { + const layerBits = buffer[1] & 0x06; + return layerBits === 0 ? 'aac' : 'mp3'; + } + + return 'unknown'; +} + +function spawnFfmpegToBuffer(args: string[], signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + // Resolve lazily so ffmpeg-static is only loaded in server runtimes that need it. + import('@/lib/server/audiobooks/ffmpeg-bin') + .then(({ getFFmpegPath }) => { + const ffmpeg = spawn(getFFmpegPath(), args); + const stdoutChunks: Buffer[] = []; + let stderr = ''; + let finished = false; + + const onAbort = () => { + if (finished) return; + finished = true; + try { + ffmpeg.kill('SIGKILL'); + } catch {} + reject(new Error('ABORTED')); + }; + + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + + ffmpeg.stdout.on('data', (chunk: Buffer) => { + stdoutChunks.push(chunk); + }); + ffmpeg.stderr.on('data', (chunk) => { + stderr += String(chunk); + }); + + ffmpeg.on('close', (code) => { + if (finished) return; + finished = true; + signal?.removeEventListener('abort', onAbort); + if (code === 0) { + resolve(Buffer.concat(stdoutChunks)); + } else { + reject(new Error(`ffmpeg transcode exited with code ${code}: ${stderr.slice(-500)}`)); + } + }); + + ffmpeg.on('error', (err) => { + if (finished) return; + finished = true; + signal?.removeEventListener('abort', onAbort); + reject(err); + }); + }) + .catch(reject); + }); +} + +/** + * Transcode an arbitrary audio buffer to mp3 using the bundled ffmpeg. The input + * is written to a temp file (ffmpeg needs seekable input for some containers) and + * the mp3 is captured from stdout. The 64k bitrate matches the audiobook pipeline. + */ +export async function transcodeToMp3(buffer: Buffer, signal?: AbortSignal): Promise { + let workDir: string | null = null; + try { + workDir = await mkdtemp(join(tmpdir(), 'openreader-tts-transcode-')); + const inputPath = join(workDir, 'input'); + await writeFile(inputPath, buffer); + + return await spawnFfmpegToBuffer( + [ + '-nostdin', + '-loglevel', 'error', + '-i', inputPath, + '-vn', + '-c:a', 'libmp3lame', + '-b:a', '64k', + '-f', 'mp3', + 'pipe:1', + ], + signal, + ); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +/** + * Ensure a TTS buffer is mp3. OpenAI-compatible servers vary in which audio + * formats they emit (some default to or only support wav), so we sniff the bytes + * and transcode anything that isn't already mp3. Real-mp3 responses pass through + * untouched, so the common path adds zero cost. + */ +export async function normalizeToMp3(buffer: Buffer, signal?: AbortSignal): Promise { + if (buffer.length === 0) return buffer; + + const format = sniffAudioFormat(buffer); + if (format === 'mp3') return buffer; + + const transcoded = await transcodeToMp3(buffer, signal); + serverLogger.info({ + event: 'tts.audio_format.normalized_to_mp3', + sourceFormat: format, + sourceBytes: buffer.length, + outputBytes: transcoded.length, + }, 'Normalized non-mp3 TTS audio to mp3'); + return transcoded; +} diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts index 0b2f719..9599990 100644 --- a/src/lib/server/tts/generate.ts +++ b/src/lib/server/tts/generate.ts @@ -11,6 +11,7 @@ import { resolveReplicateVoiceInputKey, } from '@/lib/server/tts/voice-resolution'; import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response'; +import { normalizeToMp3 } from '@/lib/server/tts/audio-format'; import { LRUCache } from 'lru-cache'; import { createHash } from 'crypto'; import { access, readFile } from 'fs/promises'; @@ -67,6 +68,12 @@ const replicateBlockedUntilByScope = new LRUCache({ max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES, }); const openAiCompatibleLanguageUnsupported = new LRUCache({ max: 256 }); +// Tracks OpenAI-compatible servers that reject an explicit `response_format` (e.g. +// wav-only servers that 400 on `response_format: mp3`). After the first rejection +// we omit the field and let the server emit its default, which `normalizeToMp3` +// then converts. In-memory + per-process: on serverless this may re-probe once per +// cold instance, which is harmless; on long-running self-hosted it persists. +const openAiCompatibleResponseFormatUnsupported = new LRUCache({ max: 256 }); const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024; const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30; @@ -634,10 +641,20 @@ async function runProviderRequest( const mockBuffer = await getTestMockTtsBuffer(request.testNamespace); if (mockBuffer) return mockBuffer; - if (request.provider === 'replicate') { - return runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries); - } + const raw = request.provider === 'replicate' + ? await runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries) + : await runOpenAiCompatibleRequest(request, signal, upstreamSettings); + // OpenAI-compatible servers (and some Replicate models) may emit wav/ogg/etc.; + // normalize to mp3 so the cache, storage, and audiobook pipeline stay mp3-only. + return normalizeToMp3(raw, signal); +} + +async function runOpenAiCompatibleRequest( + request: ResolvedServerTTSRequest, + signal: AbortSignal, + upstreamSettings: ResolvedTtsUpstreamRuntimeSettings, +): Promise { const openai = new OpenAI({ apiKey: request.apiKey, baseURL: request.baseUrl, @@ -646,44 +663,68 @@ async function runProviderRequest( timeout: upstreamSettings.ttsUpstreamTimeoutMs, }); + const formatKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}`; + const skipResponseFormat = openAiCompatibleResponseFormatUnsupported.has(formatKey); + const createParams: ExtendedSpeechParams = { model: request.model, voice: request.voice as SpeechCreateParams['voice'], input: request.text, speed: request.speed, - response_format: request.format as SpeechCreateParams['response_format'], }; - + if (!skipResponseFormat) { + createParams.response_format = request.format as SpeechCreateParams['response_format']; + } if (request.instructions) { createParams.instructions = request.instructions; } - if (request.provider !== 'openai' && request.language) { - const supportKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}|${request.language}`; - if (!openAiCompatibleLanguageUnsupported.has(supportKey)) { - try { - return await fetchTTSBufferWithRetry( - openai, - { ...createParams, language: request.language }, - signal, - upstreamSettings.ttsUpstreamMaxRetries, - ); - } catch (error) { - const status = getUpstreamStatus(error); - if (status !== 400 && status !== 422) throw error; - const fallback = await fetchTTSBufferWithRetry( - openai, - createParams, - signal, - upstreamSettings.ttsUpstreamMaxRetries, - ); - openAiCompatibleLanguageUnsupported.set(supportKey, true); - return fallback; + // Inner attempt with the existing language-unsupported fallback. + const fetchWithLanguageFallback = async (params: ExtendedSpeechParams): Promise => { + if (request.provider !== 'openai' && request.language) { + const supportKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}|${request.language}`; + if (!openAiCompatibleLanguageUnsupported.has(supportKey)) { + try { + return await fetchTTSBufferWithRetry( + openai, + { ...params, language: request.language }, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + } catch (error) { + const status = getUpstreamStatus(error); + if (status !== 400 && status !== 422) throw error; + const fallback = await fetchTTSBufferWithRetry( + openai, + params, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + openAiCompatibleLanguageUnsupported.set(supportKey, true); + return fallback; + } } } - } + return fetchTTSBufferWithRetry(openai, params, signal, upstreamSettings.ttsUpstreamMaxRetries); + }; - return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries); + try { + return await fetchWithLanguageFallback(createParams); + } catch (error) { + const status = getUpstreamStatus(error); + // A wav-only server rejects the explicit mp3 `response_format`. Retry once + // without it, cache the decision, and let `normalizeToMp3` convert the result. + const canRetryWithoutFormat = request.provider !== 'openai' + && (status === 400 || status === 422) + && !skipResponseFormat + && createParams.response_format !== undefined; + if (!canRetryWithoutFormat) throw error; + + openAiCompatibleResponseFormatUnsupported.set(formatKey, true); + const { response_format: _omitted, ...withoutFormat } = createParams; + void _omitted; + return fetchWithLanguageFallback(withoutFormat as ExtendedSpeechParams); + } } export async function generateTTSBuffer( diff --git a/src/lib/server/tts/voice-resolution.ts b/src/lib/server/tts/voice-resolution.ts index 4a517c4..7d02098 100644 --- a/src/lib/server/tts/voice-resolution.ts +++ b/src/lib/server/tts/voice-resolution.ts @@ -2,6 +2,7 @@ import { LRUCache } from 'lru-cache'; import { serverLogger } from '@/lib/server/logger'; import { logServerError } from '@/lib/server/errors/logging'; import { + OPENAI_DEFAULT_VOICES, resolveProviderModels, type ReplicateVoiceInputKey, type ResolveVoicesOptions, @@ -366,15 +367,55 @@ async function fetchDeepinfraVoices(apiKey: string): Promise { } } -async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise { +// Custom OpenAI-compatible servers expose voices under different, non-standardized +// routes. Probe the common ones in order until one returns a usable list. +const CUSTOM_OPENAI_VOICE_ENDPOINTS = ['/audio/voices', '/voices', '/styles'] as const; + +// Extract a list of voice/style names from an arbitrary JSON payload. Handles a +// bare string array, the OpenAI-style `{ voices: [...] }`, the `{ styles: [...] }` +// shape used by some servers, and arrays of objects keyed by name/id/voice/style. +function extractVoiceNames(payload: unknown): string[] | null { + const collect = (value: unknown): string[] | null => { + if (!Array.isArray(value)) { + return null; + } + const names: string[] = []; + for (const item of value) { + if (typeof item === 'string') { + names.push(item); + } else if (isRecord(item)) { + const name = item.name ?? item.id ?? item.voice ?? item.style; + if (typeof name === 'string') { + names.push(name); + } + } + } + const cleaned = Array.from( + new Set(names.map((name) => name.trim()).filter((name) => name.length > 0)), + ); + return cleaned.length > 0 ? cleaned : null; + }; + + if (Array.isArray(payload)) { + return collect(payload); + } + if (isRecord(payload)) { + return collect(payload.voices) ?? collect(payload.styles) ?? collect(payload.data); + } + return null; +} + +async function fetchCustomOpenAiVoicesFromEndpoint( + url: string, + apiKey: string, +): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => { controller.abort(); }, 10_000); try { - const normalizedBaseUrl = baseUrl.replace(/\/+$/, ''); - const response = await fetch(`${normalizedBaseUrl}/audio/voices`, { + const response = await fetch(url, { signal: controller.signal, headers: { ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), @@ -386,22 +427,32 @@ async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise return null; } - const data = await response.json(); - return Array.isArray(data.voices) && data.voices.every((voice: unknown) => typeof voice === 'string') - ? data.voices - : null; + return extractVoiceNames(await response.json()); } catch { - serverLogger.info({ - event: 'tts.voice_resolution.custom_endpoint.voices_unsupported', - degraded: true, - fallbackPath: 'provider_default_voices', - }, 'Custom endpoint does not support voices, using defaults'); return null; } finally { clearTimeout(timeoutId); } } +async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise { + const normalizedBaseUrl = baseUrl.replace(/\/+$/, ''); + + for (const endpoint of CUSTOM_OPENAI_VOICE_ENDPOINTS) { + const voices = await fetchCustomOpenAiVoicesFromEndpoint(`${normalizedBaseUrl}${endpoint}`, apiKey); + if (voices !== null) { + return voices; + } + } + + serverLogger.info({ + event: 'tts.voice_resolution.custom_endpoint.voices_unsupported', + degraded: true, + fallbackPath: 'provider_default_voices', + }, 'Custom endpoint does not support voices, using defaults'); + return null; +} + export async function resolveVoices({ provider, model, apiKey = '', baseUrl = '' }: ResolveVoicesOptions): Promise { const providerModelPolicy = resolveTtsProviderModelPolicy({ providerRef: provider, @@ -420,13 +471,15 @@ export async function resolveVoices({ provider, model, apiKey = '', baseUrl = '' } if (voiceSource === 'custom-openai-api') { - if (!baseUrl) { - return defaultVoices; - } - const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey); - if (apiVoices !== null) { - return apiVoices; + if (baseUrl) { + const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey); + if (apiVoices !== null) { + return apiVoices; + } } + // No voices endpoint resolved on this OpenAI-compatible server. Fall back to the + // canonical OpenAI voices, which every OpenAI-compatible endpoint accepts. + return defaultVoices.length > 0 ? defaultVoices : [...OPENAI_DEFAULT_VOICES]; } if (voiceSource === 'replicate-api') { diff --git a/tests/unit/tts-audio-format.vitest.spec.ts b/tests/unit/tts-audio-format.vitest.spec.ts new file mode 100644 index 0000000..a9acd04 --- /dev/null +++ b/tests/unit/tts-audio-format.vitest.spec.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { sniffAudioFormat } from '@/lib/server/tts/audio-format'; + +function bytes(...values: number[]): Buffer { + return Buffer.from(values); +} + +describe('sniffAudioFormat', () => { + it('detects wav from RIFF/WAVE header', () => { + const wav = Buffer.concat([ + Buffer.from('RIFF'), + bytes(0x24, 0x00, 0x00, 0x00), + Buffer.from('WAVE'), + ]); + expect(sniffAudioFormat(wav)).toBe('wav'); + }); + + it('detects ogg', () => { + expect(sniffAudioFormat(Buffer.from('OggS....'))).toBe('ogg'); + }); + + it('detects flac', () => { + expect(sniffAudioFormat(Buffer.from('fLaC....'))).toBe('flac'); + }); + + it('detects ID3-tagged mp3', () => { + expect(sniffAudioFormat(Buffer.concat([Buffer.from('ID3'), bytes(0x03, 0x00)]))).toBe('mp3'); + }); + + it('detects mp3 frame sync (MPEG-1 Layer III)', () => { + expect(sniffAudioFormat(bytes(0xff, 0xfb, 0x90, 0x00))).toBe('mp3'); + }); + + it('detects mp3 frame sync (MPEG-2 Layer III)', () => { + expect(sniffAudioFormat(bytes(0xff, 0xf3, 0x00, 0x00))).toBe('mp3'); + }); + + it('distinguishes ADTS aac from mp3 via the layer bits', () => { + expect(sniffAudioFormat(bytes(0xff, 0xf1, 0x00, 0x00))).toBe('aac'); + expect(sniffAudioFormat(bytes(0xff, 0xf9, 0x00, 0x00))).toBe('aac'); + }); + + it('returns unknown for unrecognized or short buffers', () => { + expect(sniffAudioFormat(bytes(0x00, 0x01, 0x02, 0x03))).toBe('unknown'); + expect(sniffAudioFormat(bytes(0x00))).toBe('unknown'); + expect(sniffAudioFormat(Buffer.alloc(0))).toBe('unknown'); + }); +});