From facbda247749c0d0de1bfc0032024f7dfc549296 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 09:38:14 -0600 Subject: [PATCH] feat(tts): forward language to capable providers --- src/app/api/audiobook/chapter/route.ts | 1 + src/app/api/tts/segments/ensure/route.ts | 1 + src/lib/server/tts/generate.ts | 85 +++++++++++++++++-- src/lib/server/tts/voice-resolution.ts | 64 ++++++++++++++ tests/unit/tts-generate.vitest.spec.ts | 32 ++++++- .../unit/tts-provider-catalog.vitest.spec.ts | 50 ++++++++++- 6 files changed, 226 insertions(+), 7 deletions(-) diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 31f40e6..e63d045 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -597,6 +597,7 @@ export async function POST(request: NextRequest) { format: 'mp3', model, instructions, + language: mergedSettings?.language, provider, apiKey: openApiKey, baseUrl: openApiBaseUrl, diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index e892784..f089231 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -642,6 +642,7 @@ export async function POST(request: NextRequest) { format: 'mp3', model: effectiveSettings.ttsModel, instructions: effectiveSettings.ttsInstructions, + language: effectiveSettings.language, provider: requestCreds.provider, apiKey: requestCreds.apiKey || 'none', baseUrl: requestCreds.baseUrl, diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts index 01a7978..c235e88 100644 --- a/src/lib/server/tts/generate.ts +++ b/src/lib/server/tts/generate.ts @@ -6,12 +6,16 @@ import { REPLICATE_KOKORO_82M_VERSIONED_MODEL, } from '@/lib/shared/tts-provider-catalog'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; -import { resolveReplicateVoiceInputKey } from '@/lib/server/tts/voice-resolution'; +import { + resolveReplicateLanguageInput, + resolveReplicateVoiceInputKey, +} from '@/lib/server/tts/voice-resolution'; import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response'; import { LRUCache } from 'lru-cache'; import { createHash } from 'crypto'; import { access, readFile } from 'fs/promises'; import { resolve } from 'path'; +import { getLanguageDisplayName, toBaseLanguageCode } from '@/lib/shared/language'; export interface ServerTTSRequest { text: string; @@ -20,6 +24,7 @@ export interface ServerTTSRequest { format?: string; model?: string | null; instructions?: string; + language?: string; provider: string; apiKey: string; baseUrl?: string; @@ -30,6 +35,7 @@ type CustomVoice = string; type ExtendedSpeechParams = Omit & { voice: SpeechCreateParams['voice'] | CustomVoice; instructions?: string; + language?: string; }; type ResolvedServerTTSRequest = { @@ -39,6 +45,7 @@ type ResolvedServerTTSRequest = { format: string; model: SpeechCreateParams['model']; instructions?: string; + language?: string; provider: string; apiKey: string; baseUrl?: string; @@ -55,6 +62,7 @@ const REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES = 512; const replicateBlockedUntilByScope = new LRUCache({ max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES, }); +const openAiCompatibleLanguageUnsupported = new LRUCache({ max: 256 }); const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024; const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30; @@ -323,6 +331,7 @@ function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest { format, model, instructions, + language: input.language, provider, apiKey: input.apiKey, baseUrl: input.baseUrl, @@ -338,6 +347,7 @@ function makeCacheKey(input: { format: string; text: string; instructions?: string; + language?: string; testNamespace?: string | null; }) { const canonical = { @@ -348,6 +358,7 @@ function makeCacheKey(input: { format: input.format, text: input.text, instructions: input.instructions || undefined, + language: input.language || undefined, testNamespace: input.testNamespace || undefined, }; return createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); @@ -363,6 +374,7 @@ export function buildTTSCacheKey(request: ServerTTSRequest): string { format: resolved.format, text: resolved.text, instructions: resolved.instructions, + language: resolved.language, testNamespace: resolved.testNamespace, }); } @@ -433,7 +445,7 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise = { text: request.text }; @@ -498,9 +510,46 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise, + request: ResolvedServerTTSRequest, +): Promise> { + if (!request.language) return input; + const languageInput = await resolveReplicateLanguageInput({ + provider: 'replicate', + model: request.model as string, + apiKey: request.apiKey, + }); + if (languageInput) { + const resolvedValue = resolveReplicateLanguageValue(request.language, languageInput.allowedValues); + if (resolvedValue) { + input[languageInput.key] = resolvedValue; + } + } return input; } +export function resolveReplicateLanguageValue(language: string, allowedValues: string[]): string | null { + if (allowedValues.length === 0) return language; + + const baseLanguage = toBaseLanguageCode(language); + const candidates = [ + language, + baseLanguage, + getLanguageDisplayName(language), + getLanguageDisplayName(baseLanguage), + ]; + const allowedByLowercase = new Map( + allowedValues.map((value) => [value.toLocaleLowerCase(), value]), + ); + return candidates + .map((candidate) => allowedByLowercase.get(candidate.toLocaleLowerCase())) + .find((candidate): candidate is string => Boolean(candidate)) ?? null; +} + async function runReplicateRequest( request: ResolvedServerTTSRequest, signal: AbortSignal, @@ -594,6 +643,31 @@ async function runProviderRequest( createParams.instructions = request.instructions; } + if (request.provider !== 'openai' && request.language) { + const supportKey = `${request.baseUrl || ''}|${request.model as string}`; + if (!openAiCompatibleLanguageUnsupported.has(supportKey)) { + try { + return await fetchTTSBufferWithRetry( + openai, + { ...createParams, language: request.language }, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + } catch (error) { + const status = getUpstreamStatus(error); + if (status !== 400 && status !== 422) throw error; + const fallback = await fetchTTSBufferWithRetry( + openai, + createParams, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + openAiCompatibleLanguageUnsupported.set(supportKey, true); + return fallback; + } + } + } + return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries); } @@ -614,6 +688,7 @@ export async function generateTTSBuffer( format: resolved.format, text: resolved.text, instructions: resolved.instructions, + language: resolved.language, testNamespace: resolved.testNamespace, }); diff --git a/src/lib/server/tts/voice-resolution.ts b/src/lib/server/tts/voice-resolution.ts index d42ab3e..1fd9420 100644 --- a/src/lib/server/tts/voice-resolution.ts +++ b/src/lib/server/tts/voice-resolution.ts @@ -102,6 +102,19 @@ function walkRecordGraph(root: unknown, visit: (node: Record) = } const REPLICATE_VOICE_KEYS = ['voice', 'voice_id', 'speaker'] as const satisfies readonly ReplicateVoiceInputKey[]; +const REPLICATE_LANGUAGE_KEYS = [ + 'language', + 'lang', + 'language_code', + 'locale', + 'language_id', + 'language_boost', +] as const; +export type ReplicateLanguageInputKey = typeof REPLICATE_LANGUAGE_KEYS[number]; +export type ReplicateLanguageInput = { + key: ReplicateLanguageInputKey; + allowedValues: string[]; +}; const REPLICATE_BUILT_IN_MODELS = new Set( resolveProviderModels('replicate') .map((model) => model.id) @@ -150,6 +163,26 @@ function extractReplicateVoiceInputKeyFromOpenApiSchema(openApiSchema: unknown): return found; } +function extractReplicateLanguageInputFromOpenApiSchema(openApiSchema: unknown): ReplicateLanguageInput | null { + let found: ReplicateLanguageInput | null = null; + + walkRecordGraph(openApiSchema, (node) => { + const properties = node.properties; + if (!isRecord(properties)) return; + for (const key of REPLICATE_LANGUAGE_KEYS) { + if (key in properties) { + found = { + key, + allowedValues: extractSchemaStringEnums(properties[key]), + }; + return true; + } + } + }); + + return found; +} + async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promise { const parsedModel = parseReplicateModelIdentifier(model); if (!parsedModel) { @@ -207,11 +240,15 @@ async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promi } const REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES = 128; +const REPLICATE_LANGUAGE_INPUT_KEY_CACHE_MAX_ENTRIES = 128; const REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES = 128; const replicateVoiceInputKeyCache = new LRUCache({ max: REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES, }); +const replicateLanguageInputCache = new LRUCache({ + max: REPLICATE_LANGUAGE_INPUT_KEY_CACHE_MAX_ENTRIES, +}); const replicateOpenApiSchemaPromiseCache = new LRUCache>({ max: REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES, }); @@ -260,6 +297,33 @@ export async function resolveReplicateVoiceInputKey({ return inputKey; } +export async function resolveReplicateLanguageInputKey({ + provider, + model, + apiKey = '', +}: ResolveVoicesOptions): Promise { + const input = await resolveReplicateLanguageInput({ provider, model, apiKey }); + return input?.key ?? null; +} + +export async function resolveReplicateLanguageInput({ + provider, + model, + apiKey = '', +}: ResolveVoicesOptions): Promise { + if (provider !== 'replicate' || !apiKey) return null; + + const cached = replicateLanguageInputCache.get(model); + if (cached) return cached; + + const openApiSchema = await getReplicateOpenApiSchemaCached(apiKey, model); + const input = extractReplicateLanguageInputFromOpenApiSchema(openApiSchema); + if (input) { + replicateLanguageInputCache.set(model, input); + } + return input; +} + async function fetchDeepinfraVoices(apiKey: string): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => { diff --git a/tests/unit/tts-generate.vitest.spec.ts b/tests/unit/tts-generate.vitest.spec.ts index a601952..4b35d02 100644 --- a/tests/unit/tts-generate.vitest.spec.ts +++ b/tests/unit/tts-generate.vitest.spec.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from 'vitest'; -import { extractReplicateAudioUrl } from '../../src/lib/server/tts/generate'; +import { + buildTTSCacheKey, + extractReplicateAudioUrl, + resolveReplicateLanguageValue, +} from '../../src/lib/server/tts/generate'; describe('replicate output URL extraction', () => { test('returns direct URL string output', () => { @@ -41,3 +45,29 @@ describe('replicate output URL extraction', () => { expect(extractReplicateAudioUrl(output)).toBeNull(); }); }); + +describe('TTS upstream cache identity', () => { + test('includes language in the upstream audio cache key', () => { + const request = { + text: '同じ文章です。', + voice: 'jf_alpha', + speed: 1, + format: 'mp3', + model: 'kokoro', + provider: 'custom-openai', + apiKey: 'test', + }; + + expect(buildTTSCacheKey({ ...request, language: 'ja' })) + .not.toBe(buildTTSCacheKey({ ...request, language: 'en' })); + }); +}); + +describe('Replicate language schema values', () => { + test('uses language codes or advertised display names without a model table', () => { + expect(resolveReplicateLanguageValue('ja-JP', [])).toBe('ja-JP'); + expect(resolveReplicateLanguageValue('ja-JP', ['en', 'ja', 'zh'])).toBe('ja'); + expect(resolveReplicateLanguageValue('ja-JP', ['English', 'Japanese'])).toBe('Japanese'); + expect(resolveReplicateLanguageValue('ja-JP', ['English', 'French'])).toBeNull(); + }); +}); diff --git a/tests/unit/tts-provider-catalog.vitest.spec.ts b/tests/unit/tts-provider-catalog.vitest.spec.ts index 334ea16..8996617 100644 --- a/tests/unit/tts-provider-catalog.vitest.spec.ts +++ b/tests/unit/tts-provider-catalog.vitest.spec.ts @@ -9,7 +9,11 @@ import { supportsTtsInstructions, } from '../../src/lib/shared/tts-provider-catalog'; import { normalizeLegacyProviderRef, resolveProviderDefaults, resolveTtsModelForProvider } from '../../src/lib/shared/tts-provider-policy'; -import { resolveReplicateVoiceInputKey, resolveVoices } from '../../src/lib/server/tts/voice-resolution'; +import { + resolveReplicateLanguageInputKey, + resolveReplicateVoiceInputKey, + resolveVoices, +} from '../../src/lib/server/tts/voice-resolution'; import { applyConfigUpdate, getVoicePreferenceKey } from '../../src/lib/client/config/updates'; import { buildSyncedPreferencePatch } from '../../src/lib/client/config/preferences'; @@ -387,6 +391,50 @@ describe('tts provider catalog', () => { globalThis.fetch = originalFetch; } }); + + test('discovers Replicate language input key from model schema', async () => { + const originalFetch = globalThis.fetch; + let calls = 0; + globalThis.fetch = async () => { + calls += 1; + return { + ok: true, + json: async () => ({ + latest_version: { + openapi_schema: { + components: { + schemas: { + Input: { + type: 'object', + properties: { + text: { type: 'string' }, + language_code: { type: 'string' }, + }, + }, + }, + }, + }, + }, + }), + } as Response; + }; + + try { + await expect(resolveReplicateLanguageInputKey({ + provider: 'replicate', + model: 'acme/schema-language-model', + apiKey: 'r8_token', + })).resolves.toBe('language_code'); + await expect(resolveReplicateLanguageInputKey({ + provider: 'replicate', + model: 'acme/schema-language-model', + apiKey: 'r8_token', + })).resolves.toBe('language_code'); + expect(calls).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } + }); }); describe('config helpers', () => {