From ce811f7636c626dd2cd3a737e3a8b2acee23e0bc Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 08:52:41 -0600 Subject: [PATCH 1/6] feat(tts): add mp3 normalization for wav/ogg responses and enhance voice endpoint compatibility Introduce `normalizeToMp3` utility to convert wav/ogg audio buffers from TTS providers to mp3, ensuring consistent mp3-only handling in cache and pipelines. Add logic to detect and adapt to OpenAI-compatible servers that do not support explicit `response_format`, falling back to default output and normalizing as needed. Refactor voice resolution to probe multiple common endpoints and robustly extract voice/style names from various payload shapes, improving compatibility with custom OpenAI-compatible servers. - Add `audio-format.ts` for audio normalization utilities - Add tests for audio format normalization - Update TTS provider request logic to apply normalization and handle response format negotiation - Enhance voice endpoint probing and extraction logic for custom servers --- src/lib/server/tts/audio-format.ts | 161 +++++++++++++++++++++ src/lib/server/tts/generate.ts | 97 +++++++++---- src/lib/server/tts/voice-resolution.ts | 89 +++++++++--- tests/unit/tts-audio-format.vitest.spec.ts | 48 ++++++ 4 files changed, 349 insertions(+), 46 deletions(-) create mode 100644 src/lib/server/tts/audio-format.ts create mode 100644 tests/unit/tts-audio-format.vitest.spec.ts diff --git a/src/lib/server/tts/audio-format.ts b/src/lib/server/tts/audio-format.ts new file mode 100644 index 0000000..bd537e0 --- /dev/null +++ b/src/lib/server/tts/audio-format.ts @@ -0,0 +1,161 @@ +import { spawn } from 'child_process'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { serverLogger } from '@/lib/server/logger'; + +export type SniffedAudioFormat = 'mp3' | 'wav' | 'ogg' | 'flac' | 'aac' | 'unknown'; + +/** + * Detect an audio container/codec from a buffer's leading bytes (magic numbers). + * + * Biased toward only reporting `mp3` when we are confident, so the common path + * (a real mp3 from the upstream) skips transcoding, while anything ambiguous is + * normalized through ffmpeg rather than mislabeled as mp3. + */ +export function sniffAudioFormat(buffer: Buffer): SniffedAudioFormat { + if (buffer.length < 4) return 'unknown'; + + // RIFF....WAVE + if ( + buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46 && + buffer.length >= 12 && + buffer[8] === 0x57 && buffer[9] === 0x41 && buffer[10] === 0x56 && buffer[11] === 0x45 + ) { + return 'wav'; + } + + // OggS + if (buffer[0] === 0x4f && buffer[1] === 0x67 && buffer[2] === 0x67 && buffer[3] === 0x53) { + return 'ogg'; + } + + // fLaC + if (buffer[0] === 0x66 && buffer[1] === 0x4c && buffer[2] === 0x61 && buffer[3] === 0x43) { + return 'flac'; + } + + // ID3-tagged mp3 + if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) { + return 'mp3'; + } + + // MPEG/AAC frame sync: 11 bits set (0xFFE_). Disambiguate mp3 vs ADTS-AAC via + // the 2 layer bits — mp3 never uses layer `00`, ADTS always does. + if (buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) { + const layerBits = buffer[1] & 0x06; + return layerBits === 0 ? 'aac' : 'mp3'; + } + + return 'unknown'; +} + +function spawnFfmpegToBuffer(args: string[], signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + // Resolve lazily so ffmpeg-static is only loaded in server runtimes that need it. + import('@/lib/server/audiobooks/ffmpeg-bin') + .then(({ getFFmpegPath }) => { + const ffmpeg = spawn(getFFmpegPath(), args); + const stdoutChunks: Buffer[] = []; + let stderr = ''; + let finished = false; + + const onAbort = () => { + if (finished) return; + finished = true; + try { + ffmpeg.kill('SIGKILL'); + } catch {} + reject(new Error('ABORTED')); + }; + + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + + ffmpeg.stdout.on('data', (chunk: Buffer) => { + stdoutChunks.push(chunk); + }); + ffmpeg.stderr.on('data', (chunk) => { + stderr += String(chunk); + }); + + ffmpeg.on('close', (code) => { + if (finished) return; + finished = true; + signal?.removeEventListener('abort', onAbort); + if (code === 0) { + resolve(Buffer.concat(stdoutChunks)); + } else { + reject(new Error(`ffmpeg transcode exited with code ${code}: ${stderr.slice(-500)}`)); + } + }); + + ffmpeg.on('error', (err) => { + if (finished) return; + finished = true; + signal?.removeEventListener('abort', onAbort); + reject(err); + }); + }) + .catch(reject); + }); +} + +/** + * Transcode an arbitrary audio buffer to mp3 using the bundled ffmpeg. The input + * is written to a temp file (ffmpeg needs seekable input for some containers) and + * the mp3 is captured from stdout. The 64k bitrate matches the audiobook pipeline. + */ +export async function transcodeToMp3(buffer: Buffer, signal?: AbortSignal): Promise { + let workDir: string | null = null; + try { + workDir = await mkdtemp(join(tmpdir(), 'openreader-tts-transcode-')); + const inputPath = join(workDir, 'input'); + await writeFile(inputPath, buffer); + + return await spawnFfmpegToBuffer( + [ + '-nostdin', + '-loglevel', 'error', + '-i', inputPath, + '-vn', + '-c:a', 'libmp3lame', + '-b:a', '64k', + '-f', 'mp3', + 'pipe:1', + ], + signal, + ); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +/** + * Ensure a TTS buffer is mp3. OpenAI-compatible servers vary in which audio + * formats they emit (some default to or only support wav), so we sniff the bytes + * and transcode anything that isn't already mp3. Real-mp3 responses pass through + * untouched, so the common path adds zero cost. + */ +export async function normalizeToMp3(buffer: Buffer, signal?: AbortSignal): Promise { + if (buffer.length === 0) return buffer; + + const format = sniffAudioFormat(buffer); + if (format === 'mp3') return buffer; + + const transcoded = await transcodeToMp3(buffer, signal); + serverLogger.info({ + event: 'tts.audio_format.normalized_to_mp3', + sourceFormat: format, + sourceBytes: buffer.length, + outputBytes: transcoded.length, + }, 'Normalized non-mp3 TTS audio to mp3'); + return transcoded; +} diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts index 0b2f719..9599990 100644 --- a/src/lib/server/tts/generate.ts +++ b/src/lib/server/tts/generate.ts @@ -11,6 +11,7 @@ import { resolveReplicateVoiceInputKey, } from '@/lib/server/tts/voice-resolution'; import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response'; +import { normalizeToMp3 } from '@/lib/server/tts/audio-format'; import { LRUCache } from 'lru-cache'; import { createHash } from 'crypto'; import { access, readFile } from 'fs/promises'; @@ -67,6 +68,12 @@ const replicateBlockedUntilByScope = new LRUCache({ max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES, }); const openAiCompatibleLanguageUnsupported = new LRUCache({ max: 256 }); +// Tracks OpenAI-compatible servers that reject an explicit `response_format` (e.g. +// wav-only servers that 400 on `response_format: mp3`). After the first rejection +// we omit the field and let the server emit its default, which `normalizeToMp3` +// then converts. In-memory + per-process: on serverless this may re-probe once per +// cold instance, which is harmless; on long-running self-hosted it persists. +const openAiCompatibleResponseFormatUnsupported = new LRUCache({ max: 256 }); const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024; const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30; @@ -634,10 +641,20 @@ async function runProviderRequest( const mockBuffer = await getTestMockTtsBuffer(request.testNamespace); if (mockBuffer) return mockBuffer; - if (request.provider === 'replicate') { - return runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries); - } + const raw = request.provider === 'replicate' + ? await runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries) + : await runOpenAiCompatibleRequest(request, signal, upstreamSettings); + // OpenAI-compatible servers (and some Replicate models) may emit wav/ogg/etc.; + // normalize to mp3 so the cache, storage, and audiobook pipeline stay mp3-only. + return normalizeToMp3(raw, signal); +} + +async function runOpenAiCompatibleRequest( + request: ResolvedServerTTSRequest, + signal: AbortSignal, + upstreamSettings: ResolvedTtsUpstreamRuntimeSettings, +): Promise { const openai = new OpenAI({ apiKey: request.apiKey, baseURL: request.baseUrl, @@ -646,44 +663,68 @@ async function runProviderRequest( timeout: upstreamSettings.ttsUpstreamTimeoutMs, }); + const formatKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}`; + const skipResponseFormat = openAiCompatibleResponseFormatUnsupported.has(formatKey); + const createParams: ExtendedSpeechParams = { model: request.model, voice: request.voice as SpeechCreateParams['voice'], input: request.text, speed: request.speed, - response_format: request.format as SpeechCreateParams['response_format'], }; - + if (!skipResponseFormat) { + createParams.response_format = request.format as SpeechCreateParams['response_format']; + } if (request.instructions) { createParams.instructions = request.instructions; } - if (request.provider !== 'openai' && request.language) { - const supportKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}|${request.language}`; - if (!openAiCompatibleLanguageUnsupported.has(supportKey)) { - try { - return await fetchTTSBufferWithRetry( - openai, - { ...createParams, language: request.language }, - signal, - upstreamSettings.ttsUpstreamMaxRetries, - ); - } catch (error) { - const status = getUpstreamStatus(error); - if (status !== 400 && status !== 422) throw error; - const fallback = await fetchTTSBufferWithRetry( - openai, - createParams, - signal, - upstreamSettings.ttsUpstreamMaxRetries, - ); - openAiCompatibleLanguageUnsupported.set(supportKey, true); - return fallback; + // Inner attempt with the existing language-unsupported fallback. + const fetchWithLanguageFallback = async (params: ExtendedSpeechParams): Promise => { + if (request.provider !== 'openai' && request.language) { + const supportKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}|${request.language}`; + if (!openAiCompatibleLanguageUnsupported.has(supportKey)) { + try { + return await fetchTTSBufferWithRetry( + openai, + { ...params, language: request.language }, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + } catch (error) { + const status = getUpstreamStatus(error); + if (status !== 400 && status !== 422) throw error; + const fallback = await fetchTTSBufferWithRetry( + openai, + params, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + openAiCompatibleLanguageUnsupported.set(supportKey, true); + return fallback; + } } } - } + return fetchTTSBufferWithRetry(openai, params, signal, upstreamSettings.ttsUpstreamMaxRetries); + }; - return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries); + try { + return await fetchWithLanguageFallback(createParams); + } catch (error) { + const status = getUpstreamStatus(error); + // A wav-only server rejects the explicit mp3 `response_format`. Retry once + // without it, cache the decision, and let `normalizeToMp3` convert the result. + const canRetryWithoutFormat = request.provider !== 'openai' + && (status === 400 || status === 422) + && !skipResponseFormat + && createParams.response_format !== undefined; + if (!canRetryWithoutFormat) throw error; + + openAiCompatibleResponseFormatUnsupported.set(formatKey, true); + const { response_format: _omitted, ...withoutFormat } = createParams; + void _omitted; + return fetchWithLanguageFallback(withoutFormat as ExtendedSpeechParams); + } } export async function generateTTSBuffer( diff --git a/src/lib/server/tts/voice-resolution.ts b/src/lib/server/tts/voice-resolution.ts index 4a517c4..7d02098 100644 --- a/src/lib/server/tts/voice-resolution.ts +++ b/src/lib/server/tts/voice-resolution.ts @@ -2,6 +2,7 @@ import { LRUCache } from 'lru-cache'; import { serverLogger } from '@/lib/server/logger'; import { logServerError } from '@/lib/server/errors/logging'; import { + OPENAI_DEFAULT_VOICES, resolveProviderModels, type ReplicateVoiceInputKey, type ResolveVoicesOptions, @@ -366,15 +367,55 @@ async function fetchDeepinfraVoices(apiKey: string): Promise { } } -async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise { +// Custom OpenAI-compatible servers expose voices under different, non-standardized +// routes. Probe the common ones in order until one returns a usable list. +const CUSTOM_OPENAI_VOICE_ENDPOINTS = ['/audio/voices', '/voices', '/styles'] as const; + +// Extract a list of voice/style names from an arbitrary JSON payload. Handles a +// bare string array, the OpenAI-style `{ voices: [...] }`, the `{ styles: [...] }` +// shape used by some servers, and arrays of objects keyed by name/id/voice/style. +function extractVoiceNames(payload: unknown): string[] | null { + const collect = (value: unknown): string[] | null => { + if (!Array.isArray(value)) { + return null; + } + const names: string[] = []; + for (const item of value) { + if (typeof item === 'string') { + names.push(item); + } else if (isRecord(item)) { + const name = item.name ?? item.id ?? item.voice ?? item.style; + if (typeof name === 'string') { + names.push(name); + } + } + } + const cleaned = Array.from( + new Set(names.map((name) => name.trim()).filter((name) => name.length > 0)), + ); + return cleaned.length > 0 ? cleaned : null; + }; + + if (Array.isArray(payload)) { + return collect(payload); + } + if (isRecord(payload)) { + return collect(payload.voices) ?? collect(payload.styles) ?? collect(payload.data); + } + return null; +} + +async function fetchCustomOpenAiVoicesFromEndpoint( + url: string, + apiKey: string, +): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => { controller.abort(); }, 10_000); try { - const normalizedBaseUrl = baseUrl.replace(/\/+$/, ''); - const response = await fetch(`${normalizedBaseUrl}/audio/voices`, { + const response = await fetch(url, { signal: controller.signal, headers: { ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), @@ -386,22 +427,32 @@ async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise return null; } - const data = await response.json(); - return Array.isArray(data.voices) && data.voices.every((voice: unknown) => typeof voice === 'string') - ? data.voices - : null; + return extractVoiceNames(await response.json()); } catch { - serverLogger.info({ - event: 'tts.voice_resolution.custom_endpoint.voices_unsupported', - degraded: true, - fallbackPath: 'provider_default_voices', - }, 'Custom endpoint does not support voices, using defaults'); return null; } finally { clearTimeout(timeoutId); } } +async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise { + const normalizedBaseUrl = baseUrl.replace(/\/+$/, ''); + + for (const endpoint of CUSTOM_OPENAI_VOICE_ENDPOINTS) { + const voices = await fetchCustomOpenAiVoicesFromEndpoint(`${normalizedBaseUrl}${endpoint}`, apiKey); + if (voices !== null) { + return voices; + } + } + + serverLogger.info({ + event: 'tts.voice_resolution.custom_endpoint.voices_unsupported', + degraded: true, + fallbackPath: 'provider_default_voices', + }, 'Custom endpoint does not support voices, using defaults'); + return null; +} + export async function resolveVoices({ provider, model, apiKey = '', baseUrl = '' }: ResolveVoicesOptions): Promise { const providerModelPolicy = resolveTtsProviderModelPolicy({ providerRef: provider, @@ -420,13 +471,15 @@ export async function resolveVoices({ provider, model, apiKey = '', baseUrl = '' } if (voiceSource === 'custom-openai-api') { - if (!baseUrl) { - return defaultVoices; - } - const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey); - if (apiVoices !== null) { - return apiVoices; + if (baseUrl) { + const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey); + if (apiVoices !== null) { + return apiVoices; + } } + // No voices endpoint resolved on this OpenAI-compatible server. Fall back to the + // canonical OpenAI voices, which every OpenAI-compatible endpoint accepts. + return defaultVoices.length > 0 ? defaultVoices : [...OPENAI_DEFAULT_VOICES]; } if (voiceSource === 'replicate-api') { diff --git a/tests/unit/tts-audio-format.vitest.spec.ts b/tests/unit/tts-audio-format.vitest.spec.ts new file mode 100644 index 0000000..a9acd04 --- /dev/null +++ b/tests/unit/tts-audio-format.vitest.spec.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { sniffAudioFormat } from '@/lib/server/tts/audio-format'; + +function bytes(...values: number[]): Buffer { + return Buffer.from(values); +} + +describe('sniffAudioFormat', () => { + it('detects wav from RIFF/WAVE header', () => { + const wav = Buffer.concat([ + Buffer.from('RIFF'), + bytes(0x24, 0x00, 0x00, 0x00), + Buffer.from('WAVE'), + ]); + expect(sniffAudioFormat(wav)).toBe('wav'); + }); + + it('detects ogg', () => { + expect(sniffAudioFormat(Buffer.from('OggS....'))).toBe('ogg'); + }); + + it('detects flac', () => { + expect(sniffAudioFormat(Buffer.from('fLaC....'))).toBe('flac'); + }); + + it('detects ID3-tagged mp3', () => { + expect(sniffAudioFormat(Buffer.concat([Buffer.from('ID3'), bytes(0x03, 0x00)]))).toBe('mp3'); + }); + + it('detects mp3 frame sync (MPEG-1 Layer III)', () => { + expect(sniffAudioFormat(bytes(0xff, 0xfb, 0x90, 0x00))).toBe('mp3'); + }); + + it('detects mp3 frame sync (MPEG-2 Layer III)', () => { + expect(sniffAudioFormat(bytes(0xff, 0xf3, 0x00, 0x00))).toBe('mp3'); + }); + + it('distinguishes ADTS aac from mp3 via the layer bits', () => { + expect(sniffAudioFormat(bytes(0xff, 0xf1, 0x00, 0x00))).toBe('aac'); + expect(sniffAudioFormat(bytes(0xff, 0xf9, 0x00, 0x00))).toBe('aac'); + }); + + it('returns unknown for unrecognized or short buffers', () => { + expect(sniffAudioFormat(bytes(0x00, 0x01, 0x02, 0x03))).toBe('unknown'); + expect(sniffAudioFormat(bytes(0x00))).toBe('unknown'); + expect(sniffAudioFormat(Buffer.alloc(0))).toBe('unknown'); + }); +}); From 5a0b40658ae2219f6337e70217466d90b48d03c5 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 09:17:10 -0600 Subject: [PATCH 2/6] fix(tts): honor explicit empty voices list in custom-openai probing extractVoiceNames collapsed an explicit `{ voices: [] }` response to null, which broke the existing contract of treating an empty list as a valid "no voices" answer. Distinguish a genuinely empty array (honored as []) from an array whose items yielded no usable names (unrecognized shape -> null, so the next endpoint or defaults apply). --- src/lib/server/tts/voice-resolution.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/server/tts/voice-resolution.ts b/src/lib/server/tts/voice-resolution.ts index 7d02098..d447932 100644 --- a/src/lib/server/tts/voice-resolution.ts +++ b/src/lib/server/tts/voice-resolution.ts @@ -390,9 +390,16 @@ function extractVoiceNames(payload: unknown): string[] | null { } } } + // An explicitly empty list is a valid answer ("this server has no voices"); + // honor it rather than probing further or falling back to defaults. + if (value.length === 0) { + return []; + } const cleaned = Array.from( new Set(names.map((name) => name.trim()).filter((name) => name.length > 0)), ); + // The value was an array but yielded no usable names (unrecognized shape); + // treat as not-a-voices-list so the next endpoint / defaults can apply. return cleaned.length > 0 ? cleaned : null; }; From f09d14549af3d296e4d93fadc6e998432f407b94 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 09:52:37 -0600 Subject: [PATCH 3/6] fix(tts): encode normalized audio at VBR q2 instead of 64k to reduce fatigue The wav->mp3 normalization path re-encoded pristine high-fidelity TTS (e.g. Supertonic's 44.1 kHz wav) at 64k CBR, which lowpasses hard and introduces high-frequency artifacts that cause listening fatigue. Switch the live/segment transcode to libmp3lame VBR -q:a 2 (~170-210 kbps for speech), effectively transparent. The audiobook export still re-encodes to 64k for file size. --- src/lib/server/tts/audio-format.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib/server/tts/audio-format.ts b/src/lib/server/tts/audio-format.ts index bd537e0..7b5d8ae 100644 --- a/src/lib/server/tts/audio-format.ts +++ b/src/lib/server/tts/audio-format.ts @@ -109,7 +109,12 @@ function spawnFfmpegToBuffer(args: string[], signal?: AbortSignal): Promise { let workDir: string | null = null; @@ -125,7 +130,7 @@ export async function transcodeToMp3(buffer: Buffer, signal?: AbortSignal): Prom '-i', inputPath, '-vn', '-c:a', 'libmp3lame', - '-b:a', '64k', + '-q:a', '2', '-f', 'mp3', 'pipe:1', ], From e85641bcfe16594d68e26810bef6132519c48dd8 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 09:58:40 -0600 Subject: [PATCH 4/6] refactor(tts): enhance mp3 format detection and improve ffmpeg abort handling Update audio format sniffing to require a full 10-byte ID3 header for mp3 detection and add stricter validation of MPEG frame headers to avoid false positives. Refactor ffmpeg process spawning to register abort listeners before module import, ensuring abort signals are handled promptly and preventing orphaned processes. Expand tests to cover edge cases in mp3 header detection. --- src/lib/server/tts/audio-format.ts | 78 +++++++++++++--------- tests/unit/tts-audio-format.vitest.spec.ts | 15 ++++- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/src/lib/server/tts/audio-format.ts b/src/lib/server/tts/audio-format.ts index 7b5d8ae..38be04f 100644 --- a/src/lib/server/tts/audio-format.ts +++ b/src/lib/server/tts/audio-format.ts @@ -35,16 +35,25 @@ export function sniffAudioFormat(buffer: Buffer): SniffedAudioFormat { return 'flac'; } - // ID3-tagged mp3 - if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) { + // ID3-tagged mp3 (the ID3v2 header is 10 bytes: "ID3" + version + flags + size). + if (buffer.length >= 10 && buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) { return 'mp3'; } - // MPEG/AAC frame sync: 11 bits set (0xFFE_). Disambiguate mp3 vs ADTS-AAC via - // the 2 layer bits — mp3 never uses layer `00`, ADTS always does. + // MPEG/AAC frame sync: 11 bits set (0xFFE_). Validate the rest of the 4-byte + // header so we only claim mp3 on a plausibly-valid frame; ambiguous bytes fall + // through to ffmpeg normalization. Disambiguate mp3 vs ADTS-AAC via the layer + // bits — mp3 never uses layer `00`, ADTS always does. if (buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) { - const layerBits = buffer[1] & 0x06; - return layerBits === 0 ? 'aac' : 'mp3'; + const layerBits = (buffer[1] >> 1) & 0x03; + if (layerBits === 0) return 'aac'; + const versionBits = (buffer[1] >> 3) & 0x03; // 0b01 is reserved/invalid + const bitrateBits = (buffer[2] >> 4) & 0x0f; // 0b1111 is the "bad" index + const sampleRateBits = (buffer[2] >> 2) & 0x03; // 0b11 is reserved + if (versionBits === 1 || bitrateBits === 0x0f || sampleRateBits === 3) { + return 'unknown'; + } + return 'mp3'; } return 'unknown'; @@ -52,39 +61,43 @@ export function sniffAudioFormat(buffer: Buffer): SniffedAudioFormat { function spawnFfmpegToBuffer(args: string[], signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new Error('ABORTED')); + return; + } + + let ffmpeg: ReturnType | null = null; + let finished = false; + + // Register abort handling before the async import so a cancel fired during the + // lazy module load is honored and we never spawn ffmpeg after an abort. + const onAbort = () => { + if (finished) return; + finished = true; + try { + ffmpeg?.kill('SIGKILL'); + } catch {} + reject(new Error('ABORTED')); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + // Resolve lazily so ffmpeg-static is only loaded in server runtimes that need it. import('@/lib/server/audiobooks/ffmpeg-bin') .then(({ getFFmpegPath }) => { - const ffmpeg = spawn(getFFmpegPath(), args); + if (finished) return; // aborted during the import + const child = spawn(getFFmpegPath(), args); + ffmpeg = child; const stdoutChunks: Buffer[] = []; let stderr = ''; - let finished = false; - const onAbort = () => { - if (finished) return; - finished = true; - try { - ffmpeg.kill('SIGKILL'); - } catch {} - reject(new Error('ABORTED')); - }; - - if (signal) { - if (signal.aborted) { - onAbort(); - return; - } - signal.addEventListener('abort', onAbort, { once: true }); - } - - ffmpeg.stdout.on('data', (chunk: Buffer) => { + child.stdout.on('data', (chunk: Buffer) => { stdoutChunks.push(chunk); }); - ffmpeg.stderr.on('data', (chunk) => { + child.stderr.on('data', (chunk) => { stderr += String(chunk); }); - ffmpeg.on('close', (code) => { + child.on('close', (code) => { if (finished) return; finished = true; signal?.removeEventListener('abort', onAbort); @@ -95,14 +108,19 @@ function spawnFfmpegToBuffer(args: string[], signal?: AbortSignal): Promise { + child.on('error', (err) => { if (finished) return; finished = true; signal?.removeEventListener('abort', onAbort); reject(err); }); }) - .catch(reject); + .catch((err) => { + if (finished) return; + finished = true; + signal?.removeEventListener('abort', onAbort); + reject(err); + }); }); } diff --git a/tests/unit/tts-audio-format.vitest.spec.ts b/tests/unit/tts-audio-format.vitest.spec.ts index a9acd04..d3e220c 100644 --- a/tests/unit/tts-audio-format.vitest.spec.ts +++ b/tests/unit/tts-audio-format.vitest.spec.ts @@ -23,8 +23,19 @@ describe('sniffAudioFormat', () => { expect(sniffAudioFormat(Buffer.from('fLaC....'))).toBe('flac'); }); - it('detects ID3-tagged mp3', () => { - expect(sniffAudioFormat(Buffer.concat([Buffer.from('ID3'), bytes(0x03, 0x00)]))).toBe('mp3'); + it('detects ID3-tagged mp3 with a full 10-byte header', () => { + // "ID3" + version(2) + flags(1) + size(4) = 10 bytes + const id3 = Buffer.concat([Buffer.from('ID3'), bytes(0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21)]); + expect(sniffAudioFormat(id3)).toBe('mp3'); + }); + + it('does not claim mp3 on a truncated ID3 header', () => { + expect(sniffAudioFormat(Buffer.concat([Buffer.from('ID3'), bytes(0x03, 0x00)]))).toBe('unknown'); + }); + + it('does not claim mp3 on a frame sync with an invalid header', () => { + // valid 11-bit sync + nonzero layer, but bitrate index 0b1111 ("bad") + expect(sniffAudioFormat(bytes(0xff, 0xfb, 0xf0, 0x00))).toBe('unknown'); }); it('detects mp3 frame sync (MPEG-1 Layer III)', () => { From c1e2b5a5f029d3575f20502953b5a023b1855dac Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 10:27:33 -0600 Subject: [PATCH 5/6] docs(supertonic): add Supertonic TTS provider guide and update docs navigation Include documentation for configuring the Supertonic TTS provider. Update the main TTS providers list and sidebar to reference the new guide, ensuring users can easily find setup instructions for Supertonic. --- .../tts-provider-guides/supertonic.md | 68 +++++++++++++++++++ docs-site/docs/configure/tts-providers.md | 1 + docs-site/sidebars.ts | 1 + 3 files changed, 70 insertions(+) create mode 100644 docs-site/docs/configure/tts-provider-guides/supertonic.md diff --git a/docs-site/docs/configure/tts-provider-guides/supertonic.md b/docs-site/docs/configure/tts-provider-guides/supertonic.md new file mode 100644 index 0000000..498504a --- /dev/null +++ b/docs-site/docs/configure/tts-provider-guides/supertonic.md @@ -0,0 +1,68 @@ +--- +title: Supertonic +--- + +Run [Supertonic](https://github.com/supertone-inc/supertonic-py) locally and connect it to OpenReader using the `Custom OpenAI-Like` provider. Supertonic is a fast, on-device TTS engine that ships its own OpenAI-compatible HTTP server. + +:::note No Docker image +Supertonic does not publish a Docker image — it installs as a Python package and runs as a local HTTP server. These instructions assume OpenReader itself runs in Docker (the common case); see [Running OpenReader directly on the host](#running-openreader-directly-on-the-host) if you don't. +::: + +## Run Supertonic + +Install with `pip` (or `pipx` for an isolated install) and start the server: + +```bash +pipx install 'supertonic[serve]' # or: pip install 'supertonic[serve]' +supertonic serve # defaults; loopback only +``` + +The first run downloads the model (~400MB). Once it's up, the OpenAI-compatible endpoint is at `http://127.0.0.1:7788/v1/audio/speech` and interactive docs are at `http://127.0.0.1:7788/docs`. + +- **Models:** `supertonic-3` (default) or `supertonic-2`. +- **Voices:** built-ins `M1`–`M5` and `F1`–`F5`, plus any custom voices you import. OpenReader discovers them automatically via the `/v1/styles` endpoint. +- **Audio format:** Supertonic emits `wav` by default; OpenReader transcodes it to mp3 transparently, so no extra configuration is needed. + +## Connect to OpenReader + +From a Docker container, your host machine is reachable at `host.docker.internal`, so the base URL is `http://host.docker.internal:7788/v1`. On Docker Desktop (macOS/Windows) this reaches the loopback-bound server above with no extra setup. + +:::note Linux (native Docker Engine) +On native Linux Docker, `host.docker.internal` needs `--add-host=host.docker.internal:host-gateway` on the OpenReader container (or the equivalent `extra_hosts` entry in `docker-compose.yml`), and it routes to the host's bridge interface rather than loopback. Pick one: + +- Run the OpenReader container with `--network host` (or `network_mode: host`), keep Supertonic on `--host 127.0.0.1`, and use `http://127.0.0.1:7788/v1` as the base URL. +- Or start Supertonic with `--host 0.0.0.0` so the bridge can reach it — keep it on a trusted network or behind a firewall. +::: + +**Recommended (auth + admin): Settings → Admin → Shared providers** + +1. Add a shared provider with type `custom-openai`. +2. Set base URL to `http://host.docker.internal:7788/v1`. +3. Leave API key blank — `supertonic serve` does not require one. +4. Set default model to `supertonic-3` (or `supertonic-2`). + +**Legacy bootstrap seed (optional, first boot only):** + +```env +API_BASE=http://host.docker.internal:7788/v1 +``` + +**Or in-app via Settings → TTS Provider:** + +1. Set provider to `Custom OpenAI-Like`. +2. Set `API_BASE` to `http://host.docker.internal:7788/v1`. +3. Leave `API_KEY` blank. +4. Choose model `supertonic-3` (or the model your deployment exposes). + +See [TTS Providers](../tts-providers) for admin-shared vs per-user behavior. + +## Running OpenReader directly on the host + +If OpenReader runs on the same machine (e.g. `pnpm dev`) rather than in Docker, skip `host.docker.internal` and use `http://127.0.0.1:7788/v1` as the base URL everywhere above. + +## References + +- [supertone-inc/supertonic-py](https://github.com/supertone-inc/supertonic-py) +- [Supported Languages](https://github.com/supertone-inc/supertonic-py#supported-languages) +- [TTS Providers](../tts-providers) +- [TTS Environment Variables](../../reference/environment-variables#tts-provider-and-request-behavior) diff --git a/docs-site/docs/configure/tts-providers.md b/docs-site/docs/configure/tts-providers.md index fac1813..376e1dc 100644 --- a/docs-site/docs/configure/tts-providers.md +++ b/docs-site/docs/configure/tts-providers.md @@ -45,6 +45,7 @@ TTS requests originate from the **Next.js server**, not the browser. `API_BASE` - [Kokoro-FastAPI](./tts-provider-guides/kokoro-fastapi) - [KittenTTS-FastAPI](./tts-provider-guides/kitten-tts-fastapi) - [Orpheus-FastAPI](./tts-provider-guides/orpheus-fastapi) +- [Supertonic](./tts-provider-guides/supertonic) - [Replicate](./tts-provider-guides/replicate) - [DeepInfra](./tts-provider-guides/deepinfra) - [OpenAI](./tts-provider-guides/openai) diff --git a/docs-site/sidebars.ts b/docs-site/sidebars.ts index fe5d1a3..30033c2 100644 --- a/docs-site/sidebars.ts +++ b/docs-site/sidebars.ts @@ -23,6 +23,7 @@ const sidebars: SidebarsConfig = { 'configure/tts-provider-guides/kokoro-fastapi', 'configure/tts-provider-guides/kitten-tts-fastapi', 'configure/tts-provider-guides/orpheus-fastapi', + 'configure/tts-provider-guides/supertonic', 'configure/tts-provider-guides/replicate', 'configure/tts-provider-guides/deepinfra', 'configure/tts-provider-guides/openai', From 116b191c9a132663b2641741af553734deec8d44 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 10:32:11 -0600 Subject: [PATCH 6/6] fix(tts): allow OpenAI-compatible TTS with empty apiKey by using placeholder Update OpenAI SDK instantiation to provide a placeholder apiKey when none is supplied, enabling support for local or unauthenticated OpenAI-compatible TTS servers. Ensure Authorization header is cleared when not needed. --- src/lib/server/tts/generate.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts index 9599990..dcd8bae 100644 --- a/src/lib/server/tts/generate.ts +++ b/src/lib/server/tts/generate.ts @@ -656,7 +656,12 @@ async function runOpenAiCompatibleRequest( upstreamSettings: ResolvedTtsUpstreamRuntimeSettings, ): Promise { const openai = new OpenAI({ - apiKey: request.apiKey, + // The SDK constructor rejects an empty apiKey, but many OpenAI-compatible + // servers (e.g. a local Supertonic/Kokoro) need no auth. Pass a placeholder to + // satisfy the constructor; `defaultHeaders` clears the Authorization header + // (merged after the bearer auth, and allowed by validateHeaders), so the + // placeholder is never sent. + apiKey: request.apiKey || 'no-key', baseURL: request.baseUrl, defaultHeaders: request.apiKey ? undefined : { Authorization: null }, maxRetries: 0,