feat(tts): add mp3 normalization for wav/ogg responses and enhance voice endpoint compatibility
Introduce `normalizeToMp3` utility to convert wav/ogg audio buffers from TTS providers to mp3, ensuring consistent mp3-only handling in cache and pipelines. Add logic to detect and adapt to OpenAI-compatible servers that do not support explicit `response_format`, falling back to default output and normalizing as needed. Refactor voice resolution to probe multiple common endpoints and robustly extract voice/style names from various payload shapes, improving compatibility with custom OpenAI-compatible servers. - Add `audio-format.ts` for audio normalization utilities - Add tests for audio format normalization - Update TTS provider request logic to apply normalization and handle response format negotiation - Enhance voice endpoint probing and extraction logic for custom servers
This commit is contained in:
Richard R
parent
7d99011cf4
commit
ce811f7636
4 changed files with 349 additions and 46 deletions
161
src/lib/server/tts/audio-format.ts
Normal file
161
src/lib/server/tts/audio-format.ts
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import { spawn } from 'child_process';
|
||||
import { mkdtemp, rm, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { serverLogger } from '@/lib/server/logger';
|
||||
|
||||
export type SniffedAudioFormat = 'mp3' | 'wav' | 'ogg' | 'flac' | 'aac' | 'unknown';
|
||||
|
||||
/**
|
||||
* Detect an audio container/codec from a buffer's leading bytes (magic numbers).
|
||||
*
|
||||
* Biased toward only reporting `mp3` when we are confident, so the common path
|
||||
* (a real mp3 from the upstream) skips transcoding, while anything ambiguous is
|
||||
* normalized through ffmpeg rather than mislabeled as mp3.
|
||||
*/
|
||||
export function sniffAudioFormat(buffer: Buffer): SniffedAudioFormat {
|
||||
if (buffer.length < 4) return 'unknown';
|
||||
|
||||
// RIFF....WAVE
|
||||
if (
|
||||
buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46 &&
|
||||
buffer.length >= 12 &&
|
||||
buffer[8] === 0x57 && buffer[9] === 0x41 && buffer[10] === 0x56 && buffer[11] === 0x45
|
||||
) {
|
||||
return 'wav';
|
||||
}
|
||||
|
||||
// OggS
|
||||
if (buffer[0] === 0x4f && buffer[1] === 0x67 && buffer[2] === 0x67 && buffer[3] === 0x53) {
|
||||
return 'ogg';
|
||||
}
|
||||
|
||||
// fLaC
|
||||
if (buffer[0] === 0x66 && buffer[1] === 0x4c && buffer[2] === 0x61 && buffer[3] === 0x43) {
|
||||
return 'flac';
|
||||
}
|
||||
|
||||
// ID3-tagged mp3
|
||||
if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) {
|
||||
return 'mp3';
|
||||
}
|
||||
|
||||
// MPEG/AAC frame sync: 11 bits set (0xFFE_). Disambiguate mp3 vs ADTS-AAC via
|
||||
// the 2 layer bits — mp3 never uses layer `00`, ADTS always does.
|
||||
if (buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) {
|
||||
const layerBits = buffer[1] & 0x06;
|
||||
return layerBits === 0 ? 'aac' : 'mp3';
|
||||
}
|
||||
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function spawnFfmpegToBuffer(args: string[], signal?: AbortSignal): Promise<Buffer> {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
// Resolve lazily so ffmpeg-static is only loaded in server runtimes that need it.
|
||||
import('@/lib/server/audiobooks/ffmpeg-bin')
|
||||
.then(({ getFFmpegPath }) => {
|
||||
const ffmpeg = spawn(getFFmpegPath(), args);
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
let stderr = '';
|
||||
let finished = false;
|
||||
|
||||
const onAbort = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
try {
|
||||
ffmpeg.kill('SIGKILL');
|
||||
} catch {}
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
ffmpeg.stdout.on('data', (chunk: Buffer) => {
|
||||
stdoutChunks.push(chunk);
|
||||
});
|
||||
ffmpeg.stderr.on('data', (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
if (code === 0) {
|
||||
resolve(Buffer.concat(stdoutChunks));
|
||||
} else {
|
||||
reject(new Error(`ffmpeg transcode exited with code ${code}: ${stderr.slice(-500)}`));
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
reject(err);
|
||||
});
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcode an arbitrary audio buffer to mp3 using the bundled ffmpeg. The input
|
||||
* is written to a temp file (ffmpeg needs seekable input for some containers) and
|
||||
* the mp3 is captured from stdout. The 64k bitrate matches the audiobook pipeline.
|
||||
*/
|
||||
export async function transcodeToMp3(buffer: Buffer, signal?: AbortSignal): Promise<Buffer> {
|
||||
let workDir: string | null = null;
|
||||
try {
|
||||
workDir = await mkdtemp(join(tmpdir(), 'openreader-tts-transcode-'));
|
||||
const inputPath = join(workDir, 'input');
|
||||
await writeFile(inputPath, buffer);
|
||||
|
||||
return await spawnFfmpegToBuffer(
|
||||
[
|
||||
'-nostdin',
|
||||
'-loglevel', 'error',
|
||||
'-i', inputPath,
|
||||
'-vn',
|
||||
'-c:a', 'libmp3lame',
|
||||
'-b:a', '64k',
|
||||
'-f', 'mp3',
|
||||
'pipe:1',
|
||||
],
|
||||
signal,
|
||||
);
|
||||
} finally {
|
||||
if (workDir) {
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a TTS buffer is mp3. OpenAI-compatible servers vary in which audio
|
||||
* formats they emit (some default to or only support wav), so we sniff the bytes
|
||||
* and transcode anything that isn't already mp3. Real-mp3 responses pass through
|
||||
* untouched, so the common path adds zero cost.
|
||||
*/
|
||||
export async function normalizeToMp3(buffer: Buffer, signal?: AbortSignal): Promise<Buffer> {
|
||||
if (buffer.length === 0) return buffer;
|
||||
|
||||
const format = sniffAudioFormat(buffer);
|
||||
if (format === 'mp3') return buffer;
|
||||
|
||||
const transcoded = await transcodeToMp3(buffer, signal);
|
||||
serverLogger.info({
|
||||
event: 'tts.audio_format.normalized_to_mp3',
|
||||
sourceFormat: format,
|
||||
sourceBytes: buffer.length,
|
||||
outputBytes: transcoded.length,
|
||||
}, 'Normalized non-mp3 TTS audio to mp3');
|
||||
return transcoded;
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ import {
|
|||
resolveReplicateVoiceInputKey,
|
||||
} from '@/lib/server/tts/voice-resolution';
|
||||
import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response';
|
||||
import { normalizeToMp3 } from '@/lib/server/tts/audio-format';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { createHash } from 'crypto';
|
||||
import { access, readFile } from 'fs/promises';
|
||||
|
|
@ -67,6 +68,12 @@ const replicateBlockedUntilByScope = new LRUCache<string, number>({
|
|||
max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES,
|
||||
});
|
||||
const openAiCompatibleLanguageUnsupported = new LRUCache<string, true>({ max: 256 });
|
||||
// Tracks OpenAI-compatible servers that reject an explicit `response_format` (e.g.
|
||||
// wav-only servers that 400 on `response_format: mp3`). After the first rejection
|
||||
// we omit the field and let the server emit its default, which `normalizeToMp3`
|
||||
// then converts. In-memory + per-process: on serverless this may re-probe once per
|
||||
// cold instance, which is harmless; on long-running self-hosted it persists.
|
||||
const openAiCompatibleResponseFormatUnsupported = new LRUCache<string, true>({ max: 256 });
|
||||
|
||||
const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024;
|
||||
const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30;
|
||||
|
|
@ -634,10 +641,20 @@ async function runProviderRequest(
|
|||
const mockBuffer = await getTestMockTtsBuffer(request.testNamespace);
|
||||
if (mockBuffer) return mockBuffer;
|
||||
|
||||
if (request.provider === 'replicate') {
|
||||
return runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries);
|
||||
}
|
||||
const raw = request.provider === 'replicate'
|
||||
? await runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries)
|
||||
: await runOpenAiCompatibleRequest(request, signal, upstreamSettings);
|
||||
|
||||
// OpenAI-compatible servers (and some Replicate models) may emit wav/ogg/etc.;
|
||||
// normalize to mp3 so the cache, storage, and audiobook pipeline stay mp3-only.
|
||||
return normalizeToMp3(raw, signal);
|
||||
}
|
||||
|
||||
async function runOpenAiCompatibleRequest(
|
||||
request: ResolvedServerTTSRequest,
|
||||
signal: AbortSignal,
|
||||
upstreamSettings: ResolvedTtsUpstreamRuntimeSettings,
|
||||
): Promise<Buffer> {
|
||||
const openai = new OpenAI({
|
||||
apiKey: request.apiKey,
|
||||
baseURL: request.baseUrl,
|
||||
|
|
@ -646,44 +663,68 @@ async function runProviderRequest(
|
|||
timeout: upstreamSettings.ttsUpstreamTimeoutMs,
|
||||
});
|
||||
|
||||
const formatKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}`;
|
||||
const skipResponseFormat = openAiCompatibleResponseFormatUnsupported.has(formatKey);
|
||||
|
||||
const createParams: ExtendedSpeechParams = {
|
||||
model: request.model,
|
||||
voice: request.voice as SpeechCreateParams['voice'],
|
||||
input: request.text,
|
||||
speed: request.speed,
|
||||
response_format: request.format as SpeechCreateParams['response_format'],
|
||||
};
|
||||
|
||||
if (!skipResponseFormat) {
|
||||
createParams.response_format = request.format as SpeechCreateParams['response_format'];
|
||||
}
|
||||
if (request.instructions) {
|
||||
createParams.instructions = request.instructions;
|
||||
}
|
||||
|
||||
if (request.provider !== 'openai' && request.language) {
|
||||
const supportKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}|${request.language}`;
|
||||
if (!openAiCompatibleLanguageUnsupported.has(supportKey)) {
|
||||
try {
|
||||
return await fetchTTSBufferWithRetry(
|
||||
openai,
|
||||
{ ...createParams, language: request.language },
|
||||
signal,
|
||||
upstreamSettings.ttsUpstreamMaxRetries,
|
||||
);
|
||||
} catch (error) {
|
||||
const status = getUpstreamStatus(error);
|
||||
if (status !== 400 && status !== 422) throw error;
|
||||
const fallback = await fetchTTSBufferWithRetry(
|
||||
openai,
|
||||
createParams,
|
||||
signal,
|
||||
upstreamSettings.ttsUpstreamMaxRetries,
|
||||
);
|
||||
openAiCompatibleLanguageUnsupported.set(supportKey, true);
|
||||
return fallback;
|
||||
// Inner attempt with the existing language-unsupported fallback.
|
||||
const fetchWithLanguageFallback = async (params: ExtendedSpeechParams): Promise<Buffer> => {
|
||||
if (request.provider !== 'openai' && request.language) {
|
||||
const supportKey = `${request.provider}|${request.baseUrl || ''}|${request.model as string}|${request.language}`;
|
||||
if (!openAiCompatibleLanguageUnsupported.has(supportKey)) {
|
||||
try {
|
||||
return await fetchTTSBufferWithRetry(
|
||||
openai,
|
||||
{ ...params, language: request.language },
|
||||
signal,
|
||||
upstreamSettings.ttsUpstreamMaxRetries,
|
||||
);
|
||||
} catch (error) {
|
||||
const status = getUpstreamStatus(error);
|
||||
if (status !== 400 && status !== 422) throw error;
|
||||
const fallback = await fetchTTSBufferWithRetry(
|
||||
openai,
|
||||
params,
|
||||
signal,
|
||||
upstreamSettings.ttsUpstreamMaxRetries,
|
||||
);
|
||||
openAiCompatibleLanguageUnsupported.set(supportKey, true);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return fetchTTSBufferWithRetry(openai, params, signal, upstreamSettings.ttsUpstreamMaxRetries);
|
||||
};
|
||||
|
||||
return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries);
|
||||
try {
|
||||
return await fetchWithLanguageFallback(createParams);
|
||||
} catch (error) {
|
||||
const status = getUpstreamStatus(error);
|
||||
// A wav-only server rejects the explicit mp3 `response_format`. Retry once
|
||||
// without it, cache the decision, and let `normalizeToMp3` convert the result.
|
||||
const canRetryWithoutFormat = request.provider !== 'openai'
|
||||
&& (status === 400 || status === 422)
|
||||
&& !skipResponseFormat
|
||||
&& createParams.response_format !== undefined;
|
||||
if (!canRetryWithoutFormat) throw error;
|
||||
|
||||
openAiCompatibleResponseFormatUnsupported.set(formatKey, true);
|
||||
const { response_format: _omitted, ...withoutFormat } = createParams;
|
||||
void _omitted;
|
||||
return fetchWithLanguageFallback(withoutFormat as ExtendedSpeechParams);
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateTTSBuffer(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { LRUCache } from 'lru-cache';
|
|||
import { serverLogger } from '@/lib/server/logger';
|
||||
import { logServerError } from '@/lib/server/errors/logging';
|
||||
import {
|
||||
OPENAI_DEFAULT_VOICES,
|
||||
resolveProviderModels,
|
||||
type ReplicateVoiceInputKey,
|
||||
type ResolveVoicesOptions,
|
||||
|
|
@ -366,15 +367,55 @@ async function fetchDeepinfraVoices(apiKey: string): Promise<string[]> {
|
|||
}
|
||||
}
|
||||
|
||||
async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise<string[] | null> {
|
||||
// Custom OpenAI-compatible servers expose voices under different, non-standardized
|
||||
// routes. Probe the common ones in order until one returns a usable list.
|
||||
const CUSTOM_OPENAI_VOICE_ENDPOINTS = ['/audio/voices', '/voices', '/styles'] as const;
|
||||
|
||||
// Extract a list of voice/style names from an arbitrary JSON payload. Handles a
|
||||
// bare string array, the OpenAI-style `{ voices: [...] }`, the `{ styles: [...] }`
|
||||
// shape used by some servers, and arrays of objects keyed by name/id/voice/style.
|
||||
function extractVoiceNames(payload: unknown): string[] | null {
|
||||
const collect = (value: unknown): string[] | null => {
|
||||
if (!Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
const names: string[] = [];
|
||||
for (const item of value) {
|
||||
if (typeof item === 'string') {
|
||||
names.push(item);
|
||||
} else if (isRecord(item)) {
|
||||
const name = item.name ?? item.id ?? item.voice ?? item.style;
|
||||
if (typeof name === 'string') {
|
||||
names.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
const cleaned = Array.from(
|
||||
new Set(names.map((name) => name.trim()).filter((name) => name.length > 0)),
|
||||
);
|
||||
return cleaned.length > 0 ? cleaned : null;
|
||||
};
|
||||
|
||||
if (Array.isArray(payload)) {
|
||||
return collect(payload);
|
||||
}
|
||||
if (isRecord(payload)) {
|
||||
return collect(payload.voices) ?? collect(payload.styles) ?? collect(payload.data);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchCustomOpenAiVoicesFromEndpoint(
|
||||
url: string,
|
||||
apiKey: string,
|
||||
): Promise<string[] | null> {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => {
|
||||
controller.abort();
|
||||
}, 10_000);
|
||||
|
||||
try {
|
||||
const normalizedBaseUrl = baseUrl.replace(/\/+$/, '');
|
||||
const response = await fetch(`${normalizedBaseUrl}/audio/voices`, {
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
||||
|
|
@ -386,22 +427,32 @@ async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise
|
|||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return Array.isArray(data.voices) && data.voices.every((voice: unknown) => typeof voice === 'string')
|
||||
? data.voices
|
||||
: null;
|
||||
return extractVoiceNames(await response.json());
|
||||
} catch {
|
||||
serverLogger.info({
|
||||
event: 'tts.voice_resolution.custom_endpoint.voices_unsupported',
|
||||
degraded: true,
|
||||
fallbackPath: 'provider_default_voices',
|
||||
}, 'Custom endpoint does not support voices, using defaults');
|
||||
return null;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchCustomOpenAiVoices(baseUrl: string, apiKey: string): Promise<string[] | null> {
|
||||
const normalizedBaseUrl = baseUrl.replace(/\/+$/, '');
|
||||
|
||||
for (const endpoint of CUSTOM_OPENAI_VOICE_ENDPOINTS) {
|
||||
const voices = await fetchCustomOpenAiVoicesFromEndpoint(`${normalizedBaseUrl}${endpoint}`, apiKey);
|
||||
if (voices !== null) {
|
||||
return voices;
|
||||
}
|
||||
}
|
||||
|
||||
serverLogger.info({
|
||||
event: 'tts.voice_resolution.custom_endpoint.voices_unsupported',
|
||||
degraded: true,
|
||||
fallbackPath: 'provider_default_voices',
|
||||
}, 'Custom endpoint does not support voices, using defaults');
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function resolveVoices({ provider, model, apiKey = '', baseUrl = '' }: ResolveVoicesOptions): Promise<string[]> {
|
||||
const providerModelPolicy = resolveTtsProviderModelPolicy({
|
||||
providerRef: provider,
|
||||
|
|
@ -420,13 +471,15 @@ export async function resolveVoices({ provider, model, apiKey = '', baseUrl = ''
|
|||
}
|
||||
|
||||
if (voiceSource === 'custom-openai-api') {
|
||||
if (!baseUrl) {
|
||||
return defaultVoices;
|
||||
}
|
||||
const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey);
|
||||
if (apiVoices !== null) {
|
||||
return apiVoices;
|
||||
if (baseUrl) {
|
||||
const apiVoices = await fetchCustomOpenAiVoices(baseUrl, apiKey);
|
||||
if (apiVoices !== null) {
|
||||
return apiVoices;
|
||||
}
|
||||
}
|
||||
// No voices endpoint resolved on this OpenAI-compatible server. Fall back to the
|
||||
// canonical OpenAI voices, which every OpenAI-compatible endpoint accepts.
|
||||
return defaultVoices.length > 0 ? defaultVoices : [...OPENAI_DEFAULT_VOICES];
|
||||
}
|
||||
|
||||
if (voiceSource === 'replicate-api') {
|
||||
|
|
|
|||
48
tests/unit/tts-audio-format.vitest.spec.ts
Normal file
48
tests/unit/tts-audio-format.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { sniffAudioFormat } from '@/lib/server/tts/audio-format';
|
||||
|
||||
function bytes(...values: number[]): Buffer {
|
||||
return Buffer.from(values);
|
||||
}
|
||||
|
||||
describe('sniffAudioFormat', () => {
|
||||
it('detects wav from RIFF/WAVE header', () => {
|
||||
const wav = Buffer.concat([
|
||||
Buffer.from('RIFF'),
|
||||
bytes(0x24, 0x00, 0x00, 0x00),
|
||||
Buffer.from('WAVE'),
|
||||
]);
|
||||
expect(sniffAudioFormat(wav)).toBe('wav');
|
||||
});
|
||||
|
||||
it('detects ogg', () => {
|
||||
expect(sniffAudioFormat(Buffer.from('OggS....'))).toBe('ogg');
|
||||
});
|
||||
|
||||
it('detects flac', () => {
|
||||
expect(sniffAudioFormat(Buffer.from('fLaC....'))).toBe('flac');
|
||||
});
|
||||
|
||||
it('detects ID3-tagged mp3', () => {
|
||||
expect(sniffAudioFormat(Buffer.concat([Buffer.from('ID3'), bytes(0x03, 0x00)]))).toBe('mp3');
|
||||
});
|
||||
|
||||
it('detects mp3 frame sync (MPEG-1 Layer III)', () => {
|
||||
expect(sniffAudioFormat(bytes(0xff, 0xfb, 0x90, 0x00))).toBe('mp3');
|
||||
});
|
||||
|
||||
it('detects mp3 frame sync (MPEG-2 Layer III)', () => {
|
||||
expect(sniffAudioFormat(bytes(0xff, 0xf3, 0x00, 0x00))).toBe('mp3');
|
||||
});
|
||||
|
||||
it('distinguishes ADTS aac from mp3 via the layer bits', () => {
|
||||
expect(sniffAudioFormat(bytes(0xff, 0xf1, 0x00, 0x00))).toBe('aac');
|
||||
expect(sniffAudioFormat(bytes(0xff, 0xf9, 0x00, 0x00))).toBe('aac');
|
||||
});
|
||||
|
||||
it('returns unknown for unrecognized or short buffers', () => {
|
||||
expect(sniffAudioFormat(bytes(0x00, 0x01, 0x02, 0x03))).toBe('unknown');
|
||||
expect(sniffAudioFormat(bytes(0x00))).toBe('unknown');
|
||||
expect(sniffAudioFormat(Buffer.alloc(0))).toBe('unknown');
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue