379 lines
10 KiB
TypeScript
379 lines
10 KiB
TypeScript
import { LRUCache } from 'lru-cache';
|
|
import { serverLogger } from '@/lib/server/logger';
|
|
import { logServerError } from '@/lib/server/errors/logging';
|
|
import {
|
|
resolveProviderModels,
|
|
type ReplicateVoiceInputKey,
|
|
type ResolveVoicesOptions,
|
|
} from '@/lib/shared/tts-provider-catalog';
|
|
import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy';
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === 'object' && value !== null;
|
|
}
|
|
|
|
function parseReplicateModelIdentifier(model: string): {
|
|
owner: string;
|
|
name: string;
|
|
version?: string;
|
|
} | null {
|
|
const [ref, version] = model.split(':', 2);
|
|
const segments = ref.split('/');
|
|
if (segments.length !== 2 || !segments[0] || !segments[1]) {
|
|
return null;
|
|
}
|
|
|
|
const parsed = {
|
|
owner: segments[0],
|
|
name: segments[1],
|
|
};
|
|
|
|
return version
|
|
? { ...parsed, version }
|
|
: parsed;
|
|
}
|
|
|
|
function extractSchemaStringEnums(schemaNode: unknown, seen = new Set<object>()): string[] {
|
|
if (!isRecord(schemaNode)) {
|
|
return [];
|
|
}
|
|
if (seen.has(schemaNode)) {
|
|
return [];
|
|
}
|
|
seen.add(schemaNode);
|
|
|
|
const values: string[] = [];
|
|
if (Array.isArray(schemaNode.enum)) {
|
|
values.push(...schemaNode.enum.filter((value): value is string => typeof value === 'string'));
|
|
}
|
|
if (typeof schemaNode.const === 'string') {
|
|
values.push(schemaNode.const);
|
|
}
|
|
|
|
for (const key of ['anyOf', 'allOf', 'oneOf'] as const) {
|
|
const branch = schemaNode[key];
|
|
if (!Array.isArray(branch)) continue;
|
|
for (const item of branch) {
|
|
values.push(...extractSchemaStringEnums(item, seen));
|
|
}
|
|
}
|
|
|
|
if (schemaNode.items) {
|
|
values.push(...extractSchemaStringEnums(schemaNode.items, seen));
|
|
}
|
|
|
|
return values;
|
|
}
|
|
|
|
function walkRecordGraph(root: unknown, visit: (node: Record<string, unknown>) => boolean | void): void {
|
|
if (!isRecord(root)) {
|
|
return;
|
|
}
|
|
|
|
const stack: Record<string, unknown>[] = [root];
|
|
const seen = new Set<object>();
|
|
|
|
while (stack.length > 0) {
|
|
const current = stack.pop();
|
|
if (!current) {
|
|
continue;
|
|
}
|
|
if (seen.has(current)) {
|
|
continue;
|
|
}
|
|
seen.add(current);
|
|
|
|
if (visit(current)) {
|
|
return;
|
|
}
|
|
|
|
for (const value of Object.values(current)) {
|
|
if (Array.isArray(value)) {
|
|
for (const item of value) {
|
|
if (isRecord(item)) {
|
|
stack.push(item);
|
|
}
|
|
}
|
|
} else if (isRecord(value)) {
|
|
stack.push(value);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const REPLICATE_VOICE_KEYS = ['voice', 'voice_id', 'speaker'] as const satisfies readonly ReplicateVoiceInputKey[];
|
|
const REPLICATE_BUILT_IN_MODELS = new Set(
|
|
resolveProviderModels('replicate')
|
|
.map((model) => model.id)
|
|
.filter((id) => id !== 'custom')
|
|
);
|
|
|
|
function extractReplicateVoicesFromOpenApiSchema(openApiSchema: unknown): string[] {
|
|
const voices: string[] = [];
|
|
|
|
walkRecordGraph(openApiSchema, (node) => {
|
|
const properties = node.properties;
|
|
if (!isRecord(properties)) {
|
|
return;
|
|
}
|
|
for (const key of REPLICATE_VOICE_KEYS) {
|
|
if (!(key in properties)) continue;
|
|
voices.push(...extractSchemaStringEnums(properties[key]));
|
|
}
|
|
});
|
|
|
|
return Array.from(
|
|
new Set(
|
|
voices
|
|
.map((voice) => voice.trim())
|
|
.filter((voice) => voice.length > 0)
|
|
)
|
|
);
|
|
}
|
|
|
|
function extractReplicateVoiceInputKeyFromOpenApiSchema(openApiSchema: unknown): ReplicateVoiceInputKey | null {
|
|
let found: ReplicateVoiceInputKey | null = null;
|
|
|
|
walkRecordGraph(openApiSchema, (node) => {
|
|
const properties = node.properties;
|
|
if (!isRecord(properties)) {
|
|
return;
|
|
}
|
|
for (const key of REPLICATE_VOICE_KEYS) {
|
|
if (key in properties) {
|
|
found = key;
|
|
return true;
|
|
}
|
|
}
|
|
});
|
|
|
|
return found;
|
|
}
|
|
|
|
async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promise<unknown | null> {
|
|
const parsedModel = parseReplicateModelIdentifier(model);
|
|
if (!parsedModel) {
|
|
return null;
|
|
}
|
|
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => {
|
|
controller.abort();
|
|
}, 10_000);
|
|
|
|
try {
|
|
const endpoint = parsedModel.version
|
|
? `https://api.replicate.com/v1/models/${parsedModel.owner}/${parsedModel.name}/versions/${parsedModel.version}`
|
|
: `https://api.replicate.com/v1/models/${parsedModel.owner}/${parsedModel.name}`;
|
|
|
|
const response = await fetch(endpoint, {
|
|
signal: controller.signal,
|
|
headers: {
|
|
Authorization: `Bearer ${apiKey}`,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
});
|
|
|
|
if (!response.ok) {
|
|
return null;
|
|
}
|
|
|
|
const data = await response.json();
|
|
let openApiSchema: unknown = null;
|
|
|
|
if (parsedModel.version) {
|
|
if (isRecord(data)) {
|
|
openApiSchema = data.openapi_schema;
|
|
}
|
|
} else if (isRecord(data) && isRecord(data.latest_version)) {
|
|
openApiSchema = data.latest_version.openapi_schema;
|
|
}
|
|
|
|
return openApiSchema;
|
|
} catch (error) {
|
|
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
return null;
|
|
}
|
|
logServerError(serverLogger, {
|
|
event: 'tts.voice_resolution.replicate_schema_fetch.failed',
|
|
msg: 'Failed fetching Replicate model schema',
|
|
error,
|
|
normalize: { code: 'TTS_VOICE_RESOLUTION_REPLICATE_SCHEMA_FETCH_FAILED', errorClass: 'upstream' },
|
|
});
|
|
return null;
|
|
} finally {
|
|
clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
const REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES = 128;
|
|
const REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES = 128;
|
|
|
|
const replicateVoiceInputKeyCache = new LRUCache<string, ReplicateVoiceInputKey>({
|
|
max: REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES,
|
|
});
|
|
const replicateOpenApiSchemaPromiseCache = new LRUCache<string, Promise<unknown | null>>({
|
|
max: REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES,
|
|
});
|
|
|
|
async function getReplicateOpenApiSchemaCached(apiKey: string, model: string): Promise<unknown | null> {
|
|
const cachedPromise = replicateOpenApiSchemaPromiseCache.get(model);
|
|
if (cachedPromise) {
|
|
return cachedPromise;
|
|
}
|
|
|
|
const fetchPromise = fetchReplicateOpenApiSchema(apiKey, model);
|
|
replicateOpenApiSchemaPromiseCache.set(model, fetchPromise);
|
|
|
|
const schema = await fetchPromise;
|
|
if (schema === null) {
|
|
replicateOpenApiSchemaPromiseCache.delete(model);
|
|
}
|
|
return schema;
|
|
}
|
|
|
|
async function fetchReplicateVoices(apiKey: string, model: string): Promise<string[] | null> {
|
|
const openApiSchema = await getReplicateOpenApiSchemaCached(apiKey, model);
|
|
const apiVoices = extractReplicateVoicesFromOpenApiSchema(openApiSchema);
|
|
return apiVoices.length > 0 ? apiVoices : null;
|
|
}
|
|
|
|
export async function resolveReplicateVoiceInputKey({
|
|
provider,
|
|
model,
|
|
apiKey = '',
|
|
}: ResolveVoicesOptions): Promise<ReplicateVoiceInputKey | null> {
|
|
if (provider !== 'replicate' || REPLICATE_BUILT_IN_MODELS.has(model) || !apiKey) {
|
|
return null;
|
|
}
|
|
|
|
const cached = replicateVoiceInputKeyCache.get(model);
|
|
if (cached) {
|
|
return cached;
|
|
}
|
|
|
|
const openApiSchema = await getReplicateOpenApiSchemaCached(apiKey, model);
|
|
const inputKey = extractReplicateVoiceInputKeyFromOpenApiSchema(openApiSchema);
|
|
if (inputKey) {
|
|
replicateVoiceInputKeyCache.set(model, inputKey);
|
|
}
|
|
return inputKey;
|
|
}
|
|
|
|
async function fetchDeepinfraVoices(apiKey: string): Promise<string[]> {
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => {
|
|
controller.abort();
|
|
}, 10_000);
|
|
|
|
try {
|
|
const response = await fetch('https://api.deepinfra.com/v1/voices', {
|
|
signal: controller.signal,
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
Authorization: `Bearer ${apiKey}`,
|
|
},
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error('Failed to fetch Deepinfra voices');
|
|
}
|
|
|
|
const data = await response.json();
|
|
if (data.voices && Array.isArray(data.voices)) {
|
|
return data.voices
|
|
.filter((voice: { user_id?: string }) => voice.user_id !== 'preset')
|
|
.map((voice: { name: string }) => voice.name);
|
|
}
|
|
return [];
|
|
} catch (error) {
|
|
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
return [];
|
|
}
|
|
logServerError(serverLogger, {
|
|
event: 'tts.voice_resolution.deepinfra_voices_fetch.failed',
|
|
msg: 'Failed fetching Deepinfra voices',
|
|
error,
|
|
normalize: { code: 'TTS_VOICE_RESOLUTION_DEEPINFRA_FETCH_FAILED', errorClass: 'upstream' },
|
|
});
|
|
return [];
|
|
} finally {
|
|
clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise<string[] | null> {
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => {
|
|
controller.abort();
|
|
}, 10_000);
|
|
|
|
try {
|
|
const normalizedBaseUrl = baseUrl.replace(/\/+$/, '');
|
|
const response = await fetch(`${normalizedBaseUrl}/audio/voices`, {
|
|
signal: controller.signal,
|
|
headers: {
|
|
Authorization: `Bearer ${apiKey}`,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
});
|
|
|
|
if (!response.ok) {
|
|
return null;
|
|
}
|
|
|
|
const data = await response.json();
|
|
return Array.isArray(data.voices) && data.voices.every((voice: unknown) => typeof voice === 'string')
|
|
? data.voices
|
|
: null;
|
|
} catch {
|
|
serverLogger.info({
|
|
event: 'tts.voice_resolution.custom_endpoint.voices_unsupported',
|
|
degraded: true,
|
|
fallbackPath: 'provider_default_voices',
|
|
}, 'Custom endpoint does not support voices, using defaults');
|
|
return null;
|
|
} finally {
|
|
clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
export async function resolveVoices({ provider, model, apiKey = '', baseUrl = '' }: ResolveVoicesOptions): Promise<string[]> {
|
|
const providerModelPolicy = resolveTtsProviderModelPolicy({
|
|
providerRef: provider,
|
|
providerType: provider,
|
|
model,
|
|
});
|
|
const defaultVoices = providerModelPolicy.defaultVoices;
|
|
const voiceSource = providerModelPolicy.voiceSource;
|
|
|
|
if (voiceSource === 'deepinfra-api') {
|
|
const apiVoices = await fetchDeepinfraVoices(apiKey);
|
|
if (apiVoices.length > 0) {
|
|
return [...defaultVoices, ...apiVoices];
|
|
}
|
|
return defaultVoices;
|
|
}
|
|
|
|
if (voiceSource === 'custom-openai-api') {
|
|
if (!baseUrl) {
|
|
return defaultVoices;
|
|
}
|
|
const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey);
|
|
if (apiVoices !== null) {
|
|
return apiVoices;
|
|
}
|
|
}
|
|
|
|
if (voiceSource === 'replicate-api') {
|
|
if (!apiKey) {
|
|
return defaultVoices;
|
|
}
|
|
const apiVoices = await fetchReplicateVoices(apiKey, model);
|
|
if (apiVoices !== null) {
|
|
return apiVoices;
|
|
}
|
|
}
|
|
|
|
return defaultVoices;
|
|
}
|