feat(tts): add Replicate Kokoro language code resolution and input mapping
Implement logic to resolve and inject Replicate Kokoro model-specific language codes based on language tag and voice, using new mapping utilities. Update `buildReplicateInput` to set `language_code` for the built-in Kokoro model. Expand language utilities with code mapping and expose `resolveReplicateKokoroLanguageCode`. Enhance tests to cover language code resolution and input mapping for Replicate Kokoro, and verify language input key discovery for built-in models.
This commit is contained in:
parent
925c995274
commit
f8182c97df
6 changed files with 147 additions and 3 deletions
|
|
@ -15,7 +15,11 @@ import { LRUCache } from 'lru-cache';
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import { access, readFile } from 'fs/promises';
|
import { access, readFile } from 'fs/promises';
|
||||||
import { resolve } from 'path';
|
import { resolve } from 'path';
|
||||||
import { getLanguageDisplayName, toBaseLanguageCode } from '@/lib/shared/language';
|
import {
|
||||||
|
getLanguageDisplayName,
|
||||||
|
resolveReplicateKokoroLanguageCode,
|
||||||
|
toBaseLanguageCode,
|
||||||
|
} from '@/lib/shared/language';
|
||||||
|
|
||||||
export interface ServerTTSRequest {
|
export interface ServerTTSRequest {
|
||||||
text: string;
|
text: string;
|
||||||
|
|
@ -434,7 +438,7 @@ async function fetchTTSBufferWithRetry(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<Record<string, unknown>> {
|
export async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise<Record<string, unknown>> {
|
||||||
const model = request.model as string;
|
const model = request.model as string;
|
||||||
|
|
||||||
if (model === 'google/gemini-3.1-flash-tts') {
|
if (model === 'google/gemini-3.1-flash-tts') {
|
||||||
|
|
@ -518,6 +522,16 @@ async function addReplicateLanguageInput(
|
||||||
request: ResolvedServerTTSRequest,
|
request: ResolvedServerTTSRequest,
|
||||||
): Promise<Record<string, unknown>> {
|
): Promise<Record<string, unknown>> {
|
||||||
if (!request.language) return input;
|
if (!request.language) return input;
|
||||||
|
if (request.model === REPLICATE_KOKORO_82M_VERSIONED_MODEL) {
|
||||||
|
const languageCode = resolveReplicateKokoroLanguageCode({
|
||||||
|
language: request.language,
|
||||||
|
voice: request.voice,
|
||||||
|
});
|
||||||
|
if (languageCode) {
|
||||||
|
input.language_code = languageCode;
|
||||||
|
}
|
||||||
|
return input;
|
||||||
|
}
|
||||||
const languageInput = await resolveReplicateLanguageInput({
|
const languageInput = await resolveReplicateLanguageInput({
|
||||||
provider: 'replicate',
|
provider: 'replicate',
|
||||||
model: request.model as string,
|
model: request.model as string,
|
||||||
|
|
|
||||||
|
|
@ -311,7 +311,7 @@ export async function resolveReplicateLanguageInput({
|
||||||
model,
|
model,
|
||||||
apiKey = '',
|
apiKey = '',
|
||||||
}: ResolveVoicesOptions): Promise<ReplicateLanguageInput | null> {
|
}: ResolveVoicesOptions): Promise<ReplicateLanguageInput | null> {
|
||||||
if (provider !== 'replicate' || REPLICATE_BUILT_IN_MODELS.has(model) || !apiKey) return null;
|
if (provider !== 'replicate' || !apiKey) return null;
|
||||||
|
|
||||||
const cached = replicateLanguageInputCache.get(model);
|
const cached = replicateLanguageInputCache.get(model);
|
||||||
if (cached) return cached;
|
if (cached) return cached;
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,28 @@ const KOKORO_LANGUAGE_BY_PREFIX: Readonly<Record<string, string>> = {
|
||||||
zm: 'zh-CN',
|
zm: 'zh-CN',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const REPLICATE_KOKORO_LANGUAGE_CODE_BY_TAG: Readonly<Record<string, string>> = {
|
||||||
|
'en-US': 'a',
|
||||||
|
'en-GB': 'b',
|
||||||
|
es: 'e',
|
||||||
|
fr: 'f',
|
||||||
|
hi: 'h',
|
||||||
|
it: 'i',
|
||||||
|
ja: 'j',
|
||||||
|
'pt-BR': 'p',
|
||||||
|
'zh-CN': 'z',
|
||||||
|
};
|
||||||
|
|
||||||
|
const REPLICATE_KOKORO_LANGUAGE_CODE_BY_BASE_TAG: Readonly<Record<string, string>> = {
|
||||||
|
es: 'e',
|
||||||
|
fr: 'f',
|
||||||
|
hi: 'h',
|
||||||
|
it: 'i',
|
||||||
|
ja: 'j',
|
||||||
|
pt: 'p',
|
||||||
|
zh: 'z',
|
||||||
|
};
|
||||||
|
|
||||||
export const KOKORO_SUPPORTED_LANGUAGES = [
|
export const KOKORO_SUPPORTED_LANGUAGES = [
|
||||||
'en',
|
'en',
|
||||||
'es',
|
'es',
|
||||||
|
|
@ -79,6 +101,26 @@ export function inferKokoroLanguageFromVoice(voice: string | null | undefined):
|
||||||
return languages.size === 1 ? [...languages][0] : null;
|
return languages.size === 1 ? [...languages][0] : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveReplicateKokoroLanguageCode(input: {
|
||||||
|
language?: string | null;
|
||||||
|
voice?: string | null;
|
||||||
|
}): string | null {
|
||||||
|
const normalizedLanguage = input.language ? normalizeLanguageTag(input.language) : null;
|
||||||
|
const voiceLanguage = inferKokoroLanguageFromVoice(input.voice);
|
||||||
|
|
||||||
|
for (const candidate of [normalizedLanguage, voiceLanguage]) {
|
||||||
|
if (!candidate) continue;
|
||||||
|
|
||||||
|
const exactCode = REPLICATE_KOKORO_LANGUAGE_CODE_BY_TAG[candidate];
|
||||||
|
if (exactCode) return exactCode;
|
||||||
|
|
||||||
|
const baseCode = REPLICATE_KOKORO_LANGUAGE_CODE_BY_BASE_TAG[toBaseLanguageCode(candidate)];
|
||||||
|
if (baseCode) return baseCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
export function getKokoroVoiceLanguages(voice: string | null | undefined): string[] {
|
export function getKokoroVoiceLanguages(voice: string | null | undefined): string[] {
|
||||||
if (!voice?.trim()) return [];
|
if (!voice?.trim()) return [];
|
||||||
return Array.from(new Set(
|
return Array.from(new Set(
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import {
|
||||||
keepKokoroVoicesInOneLanguage,
|
keepKokoroVoicesInOneLanguage,
|
||||||
normalizeOptionalLanguageTag,
|
normalizeOptionalLanguageTag,
|
||||||
normalizeUnicodeToken,
|
normalizeUnicodeToken,
|
||||||
|
resolveReplicateKokoroLanguageCode,
|
||||||
resolveTtsLanguage,
|
resolveTtsLanguage,
|
||||||
segmentSentences,
|
segmentSentences,
|
||||||
segmentWords,
|
segmentWords,
|
||||||
|
|
@ -95,6 +96,13 @@ describe('multilingual language utilities', () => {
|
||||||
expect(toBaseLanguageCode('zh-CN')).toBe('zh');
|
expect(toBaseLanguageCode('zh-CN')).toBe('zh');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('maps Kokoro voices and normalized language tags to Replicate language codes', () => {
|
||||||
|
expect(resolveReplicateKokoroLanguageCode({ language: 'en', voice: 'af_sarah' })).toBe('a');
|
||||||
|
expect(resolveReplicateKokoroLanguageCode({ language: 'en', voice: 'bf_emma' })).toBe('b');
|
||||||
|
expect(resolveReplicateKokoroLanguageCode({ language: 'ja-JP', voice: 'jf_alpha' })).toBe('j');
|
||||||
|
expect(resolveReplicateKokoroLanguageCode({ language: 'zh-TW', voice: 'zf_xiaobei' })).toBe('z');
|
||||||
|
});
|
||||||
|
|
||||||
test('normalizes Unicode tokens without dropping non-Latin scripts', () => {
|
test('normalizes Unicode tokens without dropping non-Latin scripts', () => {
|
||||||
expect(normalizeUnicodeToken('École!')).toBe('école');
|
expect(normalizeUnicodeToken('École!')).toBe('école');
|
||||||
expect(normalizeUnicodeToken('日本語。')).toBe('日本語');
|
expect(normalizeUnicodeToken('日本語。')).toBe('日本語');
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,12 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
buildReplicateInput,
|
||||||
buildTTSCacheKey,
|
buildTTSCacheKey,
|
||||||
extractReplicateAudioUrl,
|
extractReplicateAudioUrl,
|
||||||
resolveReplicateLanguageValue,
|
resolveReplicateLanguageValue,
|
||||||
} from '../../src/lib/server/tts/generate';
|
} from '../../src/lib/server/tts/generate';
|
||||||
|
import { REPLICATE_KOKORO_82M_VERSIONED_MODEL } from '../../src/lib/shared/tts-provider-catalog';
|
||||||
|
|
||||||
describe('replicate output URL extraction', () => {
|
describe('replicate output URL extraction', () => {
|
||||||
test('returns direct URL string output', () => {
|
test('returns direct URL string output', () => {
|
||||||
|
|
@ -70,4 +72,38 @@ describe('Replicate language schema values', () => {
|
||||||
expect(resolveReplicateLanguageValue('ja-JP', ['English', 'Japanese'])).toBe('Japanese');
|
expect(resolveReplicateLanguageValue('ja-JP', ['English', 'Japanese'])).toBe('Japanese');
|
||||||
expect(resolveReplicateLanguageValue('ja-JP', ['English', 'French'])).toBeNull();
|
expect(resolveReplicateLanguageValue('ja-JP', ['English', 'French'])).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('includes language_code for the built-in Replicate Kokoro model', async () => {
|
||||||
|
await expect(buildReplicateInput({
|
||||||
|
text: 'Hello world',
|
||||||
|
voice: 'af_sarah',
|
||||||
|
speed: 1,
|
||||||
|
format: 'mp3',
|
||||||
|
model: REPLICATE_KOKORO_82M_VERSIONED_MODEL,
|
||||||
|
language: 'en',
|
||||||
|
provider: 'replicate',
|
||||||
|
apiKey: 'r8_token',
|
||||||
|
testNamespace: null,
|
||||||
|
})).resolves.toEqual({
|
||||||
|
text: 'Hello world',
|
||||||
|
voice: 'af_sarah',
|
||||||
|
language_code: 'a',
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(buildReplicateInput({
|
||||||
|
text: 'Hello world',
|
||||||
|
voice: 'bf_emma',
|
||||||
|
speed: 1,
|
||||||
|
format: 'mp3',
|
||||||
|
model: REPLICATE_KOKORO_82M_VERSIONED_MODEL,
|
||||||
|
language: 'en',
|
||||||
|
provider: 'replicate',
|
||||||
|
apiKey: 'r8_token',
|
||||||
|
testNamespace: null,
|
||||||
|
})).resolves.toEqual({
|
||||||
|
text: 'Hello world',
|
||||||
|
voice: 'bf_emma',
|
||||||
|
language_code: 'b',
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -435,6 +435,50 @@ describe('tts provider catalog', () => {
|
||||||
globalThis.fetch = originalFetch;
|
globalThis.fetch = originalFetch;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('discovers Replicate language input key for built-in models too', async () => {
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
let calls = 0;
|
||||||
|
globalThis.fetch = async () => {
|
||||||
|
calls += 1;
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({
|
||||||
|
latest_version: {
|
||||||
|
openapi_schema: {
|
||||||
|
components: {
|
||||||
|
schemas: {
|
||||||
|
Input: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
text: { type: 'string' },
|
||||||
|
language: { type: 'string', enum: ['auto', 'en', 'ja'] },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
} as Response;
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await expect(resolveReplicateLanguageInputKey({
|
||||||
|
provider: 'replicate',
|
||||||
|
model: 'minimax/speech-2.8-turbo',
|
||||||
|
apiKey: 'r8_token',
|
||||||
|
})).resolves.toBe('language');
|
||||||
|
await expect(resolveReplicateLanguageInputKey({
|
||||||
|
provider: 'replicate',
|
||||||
|
model: 'minimax/speech-2.8-turbo',
|
||||||
|
apiKey: 'r8_token',
|
||||||
|
})).resolves.toBe('language');
|
||||||
|
expect(calls).toBe(1);
|
||||||
|
} finally {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('config helpers', () => {
|
describe('config helpers', () => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue