danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tts): forward language to capable providers

Browse source
This commit is contained in:
Richard R 2026-06-06 09:38:14 -06:00
parent 696d3e488f
commit facbda2477
6 changed files with 226 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/app/api/audiobook/chapter/route.ts
View file

@ -597,6 +597,7 @@ export async function POST(request: NextRequest) {
format: 'mp3',
model,
instructions,
language: mergedSettings?.language,
provider,
apiKey: openApiKey,
baseUrl: openApiBaseUrl,

1
src/app/api/tts/segments/ensure/route.ts
View file

@ -642,6 +642,7 @@ export async function POST(request: NextRequest) {
format: 'mp3',
model: effectiveSettings.ttsModel,
instructions: effectiveSettings.ttsInstructions,
language: effectiveSettings.language,
provider: requestCreds.provider,
apiKey: requestCreds.apiKey || 'none',
baseUrl: requestCreds.baseUrl,

85
src/lib/server/tts/generate.ts
View file

@ -6,12 +6,16 @@ import {
REPLICATE_KOKORO_82M_VERSIONED_MODEL,
} from '@/lib/shared/tts-provider-catalog';
import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy';
import { resolveReplicateVoiceInputKey } from '@/lib/server/tts/voice-resolution';
import {
resolveReplicateLanguageInput,
resolveReplicateVoiceInputKey,
} from '@/lib/server/tts/voice-resolution';
import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response';
import { LRUCache } from 'lru-cache';
import { createHash } from 'crypto';
import { access, readFile } from 'fs/promises';
import { resolve } from 'path';
import { getLanguageDisplayName, toBaseLanguageCode } from '@/lib/shared/language';
export interface ServerTTSRequest {
text: string;
@ -20,6 +24,7 @@ export interface ServerTTSRequest {
format?: string;
model?: string | null;
instructions?: string;
language?: string;
provider: string;
apiKey: string;
baseUrl?: string;
@ -30,6 +35,7 @@ type CustomVoice = string;
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
voice: SpeechCreateParams['voice'] | CustomVoice;
instructions?: string;
language?: string;
};
type ResolvedServerTTSRequest = {
@ -39,6 +45,7 @@ type ResolvedServerTTSRequest = {
format: string;
model: SpeechCreateParams['model'];
instructions?: string;
language?: string;
provider: string;
apiKey: string;
baseUrl?: string;
@ -55,6 +62,7 @@ const REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES = 512;
const replicateBlockedUntilByScope = new LRUCache<string, number>({
max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES,
});
const openAiCompatibleLanguageUnsupported = new LRUCache<string, true>({ max: 256 });
const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024;
const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30;
@ -323,6 +331,7 @@ function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
format,
model,
instructions,
language: input.language,
provider,
apiKey: input.apiKey,
baseUrl: input.baseUrl,
@ -338,6 +347,7 @@ function makeCacheKey(input: {
format: string;
text: string;
instructions?: string;
language?: string;
testNamespace?: string | null;
}) {
const canonical = {
@ -348,6 +358,7 @@ function makeCacheKey(input: {
format: input.format,
text: input.text,
instructions: input.instructions || undefined,
language: input.language || undefined,
testNamespace: input.testNamespace || undefined,
};
return createHash('sha256').update(JSON.stringify(canonical)).digest('hex');
@ -363,6 +374,7 @@ export function buildTTSCacheKey(request: ServerTTSRequest): string {
format: resolved.format,
text: resolved.text,
instructions: resolved.instructions,
language: resolved.language,
testNamespace: resolved.testNamespace,
});
}
@ -433,7 +445,7 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<R
if (request.instructions) {
input.prompt = request.instructions;
}
return input;
return addReplicateLanguageInput(input, request);
}
if (model === 'minimax/speech-2.8-turbo') {
@ -445,7 +457,7 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<R
if (request.speed !== 1) {
input.speed = Math.max(0.5, Math.min(2.0, request.speed));
}
return input;
return addReplicateLanguageInput(input, request);
}
if (model === 'qwen/qwen3-tts') {
@ -457,7 +469,7 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<R
if (request.instructions) {
input.style_instruction = request.instructions;
}
return input;
return addReplicateLanguageInput(input, request);
}
if (model === 'inworld/tts-1.5-mini') {
@ -469,7 +481,7 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<R
if (request.speed !== 1) {
input.speaking_rate = request.speed;
}
return input;
return addReplicateLanguageInput(input, request);
}
const input: Record<string, unknown> = { text: request.text };
@ -498,9 +510,46 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<R
input.instructions = request.instructions;
}
return addReplicateLanguageInput(input, request);
}
async function addReplicateLanguageInput(
input: Record<string, unknown>,
request: ResolvedServerTTSRequest,
): Promise<Record<string, unknown>> {
if (!request.language) return input;
const languageInput = await resolveReplicateLanguageInput({
provider: 'replicate',
model: request.model as string,
apiKey: request.apiKey,
});
if (languageInput) {
const resolvedValue = resolveReplicateLanguageValue(request.language, languageInput.allowedValues);
if (resolvedValue) {
input[languageInput.key] = resolvedValue;
}
}
return input;
}
export function resolveReplicateLanguageValue(language: string, allowedValues: string[]): string | null {
if (allowedValues.length === 0) return language;
const baseLanguage = toBaseLanguageCode(language);
const candidates = [
language,
baseLanguage,
getLanguageDisplayName(language),
getLanguageDisplayName(baseLanguage),
];
const allowedByLowercase = new Map(
allowedValues.map((value) => [value.toLocaleLowerCase(), value]),
);
return candidates
.map((candidate) => allowedByLowercase.get(candidate.toLocaleLowerCase()))
.find((candidate): candidate is string => Boolean(candidate)) ?? null;
}
async function runReplicateRequest(
request: ResolvedServerTTSRequest,
signal: AbortSignal,
@ -594,6 +643,31 @@ async function runProviderRequest(
createParams.instructions = request.instructions;
}
if (request.provider !== 'openai' && request.language) {
const supportKey = `${request.baseUrl || ''}|${request.model as string}`;
if (!openAiCompatibleLanguageUnsupported.has(supportKey)) {
try {
return await fetchTTSBufferWithRetry(
openai,
{ ...createParams, language: request.language },
signal,
upstreamSettings.ttsUpstreamMaxRetries,
);
} catch (error) {
const status = getUpstreamStatus(error);
if (status !== 400 && status !== 422) throw error;
const fallback = await fetchTTSBufferWithRetry(
openai,
createParams,
signal,
upstreamSettings.ttsUpstreamMaxRetries,
);
openAiCompatibleLanguageUnsupported.set(supportKey, true);
return fallback;
}
}
}
return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries);
}
@ -614,6 +688,7 @@ export async function generateTTSBuffer(
format: resolved.format,
text: resolved.text,
instructions: resolved.instructions,
language: resolved.language,
testNamespace: resolved.testNamespace,
});

64
src/lib/server/tts/voice-resolution.ts
View file

@ -102,6 +102,19 @@ function walkRecordGraph(root: unknown, visit: (node: Record<string, unknown>) =
}
const REPLICATE_VOICE_KEYS = ['voice', 'voice_id', 'speaker'] as const satisfies readonly ReplicateVoiceInputKey[];
const REPLICATE_LANGUAGE_KEYS = [
'language',
'lang',
'language_code',
'locale',
'language_id',
'language_boost',
] as const;
export type ReplicateLanguageInputKey = typeof REPLICATE_LANGUAGE_KEYS[number];
export type ReplicateLanguageInput = {
key: ReplicateLanguageInputKey;
allowedValues: string[];
};
const REPLICATE_BUILT_IN_MODELS = new Set(
resolveProviderModels('replicate')
.map((model) => model.id)
@ -150,6 +163,26 @@ function extractReplicateVoiceInputKeyFromOpenApiSchema(openApiSchema: unknown):
return found;
}
function extractReplicateLanguageInputFromOpenApiSchema(openApiSchema: unknown): ReplicateLanguageInput | null {
let found: ReplicateLanguageInput | null = null;
walkRecordGraph(openApiSchema, (node) => {
const properties = node.properties;
if (!isRecord(properties)) return;
for (const key of REPLICATE_LANGUAGE_KEYS) {
if (key in properties) {
found = {
key,
allowedValues: extractSchemaStringEnums(properties[key]),
};
return true;
}
}
});
return found;
}
async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promise<unknown | null> {
const parsedModel = parseReplicateModelIdentifier(model);
if (!parsedModel) {
@ -207,11 +240,15 @@ async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promi
}
const REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES = 128;
const REPLICATE_LANGUAGE_INPUT_KEY_CACHE_MAX_ENTRIES = 128;
const REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES = 128;
const replicateVoiceInputKeyCache = new LRUCache<string, ReplicateVoiceInputKey>({
max: REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES,
});
const replicateLanguageInputCache = new LRUCache<string, ReplicateLanguageInput>({
max: REPLICATE_LANGUAGE_INPUT_KEY_CACHE_MAX_ENTRIES,
});
const replicateOpenApiSchemaPromiseCache = new LRUCache<string, Promise<unknown | null>>({
max: REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES,
});
@ -260,6 +297,33 @@ export async function resolveReplicateVoiceInputKey({
return inputKey;
}
export async function resolveReplicateLanguageInputKey({
provider,
model,
apiKey = '',
}: ResolveVoicesOptions): Promise<ReplicateLanguageInputKey | null> {
const input = await resolveReplicateLanguageInput({ provider, model, apiKey });
return input?.key ?? null;
}
export async function resolveReplicateLanguageInput({
provider,
model,
apiKey = '',
}: ResolveVoicesOptions): Promise<ReplicateLanguageInput | null> {
if (provider !== 'replicate' || !apiKey) return null;
const cached = replicateLanguageInputCache.get(model);
if (cached) return cached;
const openApiSchema = await getReplicateOpenApiSchemaCached(apiKey, model);
const input = extractReplicateLanguageInputFromOpenApiSchema(openApiSchema);
if (input) {
replicateLanguageInputCache.set(model, input);
}
return input;
}
async function fetchDeepinfraVoices(apiKey: string): Promise<string[]> {
const controller = new AbortController();
const timeoutId = setTimeout(() => {

32
tests/unit/tts-generate.vitest.spec.ts
View file

@ -1,6 +1,10 @@
import { describe, expect, test } from 'vitest';
import { extractReplicateAudioUrl } from '../../src/lib/server/tts/generate';
import {
buildTTSCacheKey,
extractReplicateAudioUrl,
resolveReplicateLanguageValue,
} from '../../src/lib/server/tts/generate';
describe('replicate output URL extraction', () => {
test('returns direct URL string output', () => {
@ -41,3 +45,29 @@ describe('replicate output URL extraction', () => {
expect(extractReplicateAudioUrl(output)).toBeNull();
});
});
describe('TTS upstream cache identity', () => {
test('includes language in the upstream audio cache key', () => {
const request = {
text: '同じ文章です。',
voice: 'jf_alpha',
speed: 1,
format: 'mp3',
model: 'kokoro',
provider: 'custom-openai',
apiKey: 'test',
};
expect(buildTTSCacheKey({ ...request, language: 'ja' }))
.not.toBe(buildTTSCacheKey({ ...request, language: 'en' }));
});
});
describe('Replicate language schema values', () => {
test('uses language codes or advertised display names without a model table', () => {
expect(resolveReplicateLanguageValue('ja-JP', [])).toBe('ja-JP');
expect(resolveReplicateLanguageValue('ja-JP', ['en', 'ja', 'zh'])).toBe('ja');
expect(resolveReplicateLanguageValue('ja-JP', ['English', 'Japanese'])).toBe('Japanese');
expect(resolveReplicateLanguageValue('ja-JP', ['English', 'French'])).toBeNull();
});
});

50
tests/unit/tts-provider-catalog.vitest.spec.ts
View file

@ -9,7 +9,11 @@ import {
supportsTtsInstructions,
} from '../../src/lib/shared/tts-provider-catalog';
import { normalizeLegacyProviderRef, resolveProviderDefaults, resolveTtsModelForProvider } from '../../src/lib/shared/tts-provider-policy';
import { resolveReplicateVoiceInputKey, resolveVoices } from '../../src/lib/server/tts/voice-resolution';
import {
resolveReplicateLanguageInputKey,
resolveReplicateVoiceInputKey,
resolveVoices,
} from '../../src/lib/server/tts/voice-resolution';
import { applyConfigUpdate, getVoicePreferenceKey } from '../../src/lib/client/config/updates';
import { buildSyncedPreferencePatch } from '../../src/lib/client/config/preferences';
@ -387,6 +391,50 @@ describe('tts provider catalog', () => {
globalThis.fetch = originalFetch;
}
});
test('discovers Replicate language input key from model schema', async () => {
const originalFetch = globalThis.fetch;
let calls = 0;
globalThis.fetch = async () => {
calls += 1;
return {
ok: true,
json: async () => ({
latest_version: {
openapi_schema: {
components: {
schemas: {
Input: {
type: 'object',
properties: {
text: { type: 'string' },
language_code: { type: 'string' },
},
},
},
},
},
},
}),
} as Response;
};
try {
await expect(resolveReplicateLanguageInputKey({
provider: 'replicate',
model: 'acme/schema-language-model',
apiKey: 'r8_token',
})).resolves.toBe('language_code');
await expect(resolveReplicateLanguageInputKey({
provider: 'replicate',
model: 'acme/schema-language-model',
apiKey: 'r8_token',
})).resolves.toBe('language_code');
expect(calls).toBe(1);
} finally {
globalThis.fetch = originalFetch;
}
});
});
describe('config helpers', () => {