refactor(audiobooks): Refactor TTS generation and audiobook chapter creation
- Removed unused imports and types from TTS route. - Consolidated TTS buffer generation logic into a new module. - Implemented caching for TTS audio buffers using LRU cache. - Updated TTS request handling to support instructions for specific models. - Refactored audiobook chapter creation to use new TTS generation logic. - Simplified error handling and response management in TTS API. - Enhanced client-side logic to manage TTS requests and retries. - Updated types to reflect changes in TTS request payload structure.
This commit is contained in:
Richard R
parent
df0e4341ff
commit
e41a5ba481
8 changed files with 559 additions and 424 deletions
|
|
@ -8,6 +8,9 @@ import { and, eq, inArray } from 'drizzle-orm';
|
|||
import { db } from '@/db';
|
||||
import { audiobooks, audiobookChapters } from '@/db/schema';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter';
|
||||
import { getClientIp } from '@/lib/server/rate-limit/request-ip';
|
||||
import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id';
|
||||
import {
|
||||
deleteAudiobookObject,
|
||||
getAudiobookObjectBuffer,
|
||||
|
|
@ -25,14 +28,15 @@ import { isS3Configured } from '@/lib/server/storage/s3';
|
|||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobooks/user-scope';
|
||||
import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin';
|
||||
import { generateTTSBuffer } from '@/lib/server/tts/generate';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
|
||||
import type { TTSAudiobookFormat } from '@/types/tts';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
interface ConversionRequest {
|
||||
chapterTitle: string;
|
||||
buffer: TTSAudioBytes;
|
||||
text: string;
|
||||
bookId?: string;
|
||||
format?: TTSAudiobookFormat;
|
||||
chapterIndex?: number;
|
||||
|
|
@ -47,6 +51,35 @@ type ChapterObject = {
|
|||
};
|
||||
|
||||
const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const PROBLEM_TYPES = {
|
||||
dailyQuotaExceeded: 'https://openreader.app/problems/daily-quota-exceeded',
|
||||
} as const;
|
||||
|
||||
type ProblemDetails = {
|
||||
type: string;
|
||||
title: string;
|
||||
status: number;
|
||||
detail?: string;
|
||||
instance?: string;
|
||||
code?: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
function attachDeviceIdCookie(response: NextResponse, deviceId: string | null, didCreate: boolean) {
|
||||
if (didCreate && deviceId) {
|
||||
setDeviceIdCookie(response, deviceId);
|
||||
}
|
||||
}
|
||||
|
||||
function formatLimitForHint(limit: number): string {
|
||||
if (!Number.isFinite(limit) || limit <= 0) return String(limit);
|
||||
if (limit >= 1_000_000) {
|
||||
const m = limit / 1_000_000;
|
||||
return `${m % 1 === 0 ? m.toFixed(0) : m.toFixed(1)}M`;
|
||||
}
|
||||
if (limit >= 1_000) return `${Math.round(limit / 1_000)}K`;
|
||||
return String(limit);
|
||||
}
|
||||
|
||||
function isSafeId(value: string): boolean {
|
||||
return SAFE_ID_REGEX.test(value);
|
||||
|
|
@ -208,16 +241,21 @@ function findChapterFileNameByIndex(fileNames: string[], index: number): { fileN
|
|||
|
||||
export async function POST(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
let didCreateDeviceIdCookie = false;
|
||||
let deviceIdToSet: string | null = null;
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const data: ConversionRequest = await request.json();
|
||||
const requestedFormat = data.format || 'm4b';
|
||||
if (!data.text || typeof data.text !== 'string') {
|
||||
return NextResponse.json({ error: 'Missing text for TTS generation' }, { status: 400 });
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const { userId, authEnabled } = ctxOrRes;
|
||||
const { userId, authEnabled, user } = ctxOrRes;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const { preferredUserId, allowedUserIds } = buildAllowedAudiobookUserIds(authEnabled, userId, unclaimedUserId);
|
||||
|
|
@ -270,7 +308,8 @@ export async function POST(request: NextRequest) {
|
|||
existingSettings.voice !== incomingSettings.voice ||
|
||||
existingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
||||
existingSettings.postSpeed !== incomingSettings.postSpeed ||
|
||||
existingSettings.format !== incomingSettings.format;
|
||||
existingSettings.format !== incomingSettings.format ||
|
||||
(existingSettings.ttsInstructions || '') !== (incomingSettings.ttsInstructions || '');
|
||||
if (mismatch) {
|
||||
return NextResponse.json({ error: 'Audiobook settings mismatch', settings: existingSettings }, { status: 409 });
|
||||
}
|
||||
|
|
@ -309,12 +348,102 @@ export async function POST(request: NextRequest) {
|
|||
chapterIndex = next;
|
||||
}
|
||||
|
||||
const provider = request.headers.get('x-tts-provider')
|
||||
|| incomingSettings?.ttsProvider
|
||||
|| existingSettings?.ttsProvider
|
||||
|| 'openai';
|
||||
const openApiKey = request.headers.get('x-openai-key') || process.env.API_KEY || 'none';
|
||||
const openApiBaseUrl = request.headers.get('x-openai-base-url') || process.env.API_BASE;
|
||||
const model = incomingSettings?.ttsModel ?? existingSettings?.ttsModel;
|
||||
const voice = incomingSettings?.voice
|
||||
|| existingSettings?.voice
|
||||
|| (provider === 'openai'
|
||||
? 'alloy'
|
||||
: provider === 'deepinfra'
|
||||
? 'af_bella'
|
||||
: 'af_sarah');
|
||||
const rawNativeSpeed = incomingSettings?.nativeSpeed ?? existingSettings?.nativeSpeed ?? 1;
|
||||
const nativeSpeed = Number.isFinite(Number(rawNativeSpeed)) ? Number(rawNativeSpeed) : 1;
|
||||
const instructions = incomingSettings?.ttsInstructions ?? existingSettings?.ttsInstructions;
|
||||
|
||||
if (authEnabled && userId && isTtsRateLimitEnabled()) {
|
||||
const isAnonymous = Boolean(user?.isAnonymous);
|
||||
const charCount = data.text.length;
|
||||
const ip = getClientIp(request);
|
||||
const device = isAnonymous ? getOrCreateDeviceId(request) : null;
|
||||
if (device?.didCreate) {
|
||||
didCreateDeviceIdCookie = true;
|
||||
deviceIdToSet = device.deviceId;
|
||||
}
|
||||
|
||||
const rateLimitResult = await rateLimiter.checkAndIncrementLimit(
|
||||
{ id: userId, isAnonymous },
|
||||
charCount,
|
||||
{
|
||||
deviceId: device?.deviceId ?? null,
|
||||
ip,
|
||||
},
|
||||
);
|
||||
|
||||
if (!rateLimitResult.allowed) {
|
||||
const resetTimeMs = rateLimitResult.resetTimeMs;
|
||||
const retryAfterSeconds = Math.max(
|
||||
0,
|
||||
Math.ceil((resetTimeMs - Date.now()) / 1000),
|
||||
);
|
||||
|
||||
const problem: ProblemDetails = {
|
||||
type: PROBLEM_TYPES.dailyQuotaExceeded,
|
||||
title: 'Daily quota exceeded',
|
||||
status: 429,
|
||||
detail: 'Daily character limit exceeded',
|
||||
code: 'USER_DAILY_QUOTA_EXCEEDED',
|
||||
currentCount: rateLimitResult.currentCount,
|
||||
limit: rateLimitResult.limit,
|
||||
remainingChars: rateLimitResult.remainingChars,
|
||||
resetTimeMs,
|
||||
userType: isAnonymous ? 'anonymous' : 'authenticated',
|
||||
upgradeHint: isAnonymous
|
||||
? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day`
|
||||
: undefined,
|
||||
instance: request.nextUrl.pathname,
|
||||
};
|
||||
|
||||
const response = new NextResponse(JSON.stringify(problem), {
|
||||
status: 429,
|
||||
headers: {
|
||||
'Content-Type': 'application/problem+json',
|
||||
'Retry-After': String(retryAfterSeconds),
|
||||
},
|
||||
});
|
||||
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
const ttsBuffer = await generateTTSBuffer(
|
||||
{
|
||||
text: data.text,
|
||||
voice,
|
||||
speed: nativeSpeed,
|
||||
format: 'mp3',
|
||||
model,
|
||||
instructions,
|
||||
provider,
|
||||
apiKey: openApiKey,
|
||||
baseUrl: openApiBaseUrl,
|
||||
testNamespace,
|
||||
},
|
||||
request.signal,
|
||||
);
|
||||
|
||||
workDir = await mkdtemp(join(tmpdir(), 'openreader-audiobook-'));
|
||||
const inputPath = join(workDir, `${chapterIndex}-input.mp3`);
|
||||
const chapterOutputTempPath = join(workDir, `${chapterIndex}-chapter.tmp.${format}`);
|
||||
const titleTag = encodeChapterTitleTag(chapterIndex, data.chapterTitle);
|
||||
|
||||
await writeFile(inputPath, Buffer.from(new Uint8Array(data.buffer)));
|
||||
await writeFile(inputPath, ttsBuffer);
|
||||
|
||||
const canCopyMp3WithoutReencode = format === 'mp3' && postSpeed === 1;
|
||||
if (canCopyMp3WithoutReencode) {
|
||||
|
|
@ -398,7 +527,7 @@ export async function POST(request: NextRequest) {
|
|||
set: { title: data.chapterTitle, duration, format, filePath: finalChapterName },
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
const response = NextResponse.json({
|
||||
index: chapterIndex,
|
||||
title: data.chapterTitle,
|
||||
duration,
|
||||
|
|
@ -406,12 +535,18 @@ export async function POST(request: NextRequest) {
|
|||
bookId,
|
||||
format,
|
||||
});
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
return response;
|
||||
} catch (error) {
|
||||
if ((error as Error)?.message === 'ABORTED' || request.signal.aborted) {
|
||||
return NextResponse.json({ error: 'cancelled' }, { status: 499 });
|
||||
if ((error as Error)?.message === 'ABORTED' || (error as Error)?.name === 'AbortError' || request.signal.aborted) {
|
||||
const response = NextResponse.json({ error: 'cancelled' }, { status: 499 });
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
return response;
|
||||
}
|
||||
console.error('Error processing audio chapter:', error);
|
||||
return NextResponse.json({ error: 'Failed to process audio chapter' }, { status: 500 });
|
||||
const response = NextResponse.json({ error: 'Failed to process audio chapter' }, { status: 500 });
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
return response;
|
||||
} finally {
|
||||
if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,19 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import OpenAI from 'openai';
|
||||
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { createHash } from 'crypto';
|
||||
import type { TTSRequestPayload } from '@/types/client';
|
||||
import type { TTSError, TTSAudioBuffer } from '@/types/tts';
|
||||
import type { TTSError } from '@/types/tts';
|
||||
import { headers } from 'next/headers';
|
||||
import { auth } from '@/lib/server/auth/auth';
|
||||
import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter';
|
||||
import { isAuthEnabled } from '@/lib/server/auth/config';
|
||||
import { getClientIp } from '@/lib/server/rate-limit/request-ip';
|
||||
import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import {
|
||||
buildTTSCacheKey,
|
||||
generateTTSBuffer,
|
||||
getCachedTTSBuffer,
|
||||
getTTSContentType,
|
||||
} from '@/lib/server/tts/generate';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const maxDuration = 60;
|
||||
|
|
@ -22,30 +24,6 @@ function attachDeviceIdCookie(response: NextResponse, deviceId: string | null, d
|
|||
}
|
||||
}
|
||||
|
||||
type CustomVoice = string;
|
||||
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
|
||||
voice: SpeechCreateParams['voice'] | CustomVoice;
|
||||
instructions?: string;
|
||||
};
|
||||
type AudioBufferValue = TTSAudioBuffer;
|
||||
|
||||
const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB
|
||||
const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes
|
||||
|
||||
const ttsAudioCache = new LRUCache<string, AudioBufferValue>({
|
||||
maxSize: TTS_CACHE_MAX_SIZE_BYTES,
|
||||
sizeCalculation: (value) => value.byteLength,
|
||||
ttl: TTS_CACHE_TTL_MS,
|
||||
});
|
||||
|
||||
type InflightEntry = {
|
||||
promise: Promise<TTSAudioBuffer>;
|
||||
controller: AbortController;
|
||||
consumers: number;
|
||||
};
|
||||
|
||||
const inflightRequests = new Map<string, InflightEntry>();
|
||||
|
||||
const PROBLEM_TYPES = {
|
||||
dailyQuotaExceeded: 'https://openreader.app/problems/daily-quota-exceeded',
|
||||
upstreamRateLimited: 'https://openreader.app/problems/upstream-rate-limited',
|
||||
|
|
@ -61,71 +39,6 @@ type ProblemDetails = {
|
|||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((res) => setTimeout(res, ms));
|
||||
}
|
||||
|
||||
async function fetchTTSBufferWithRetry(
|
||||
openai: OpenAI,
|
||||
createParams: ExtendedSpeechParams,
|
||||
signal: AbortSignal
|
||||
): Promise<TTSAudioBuffer> {
|
||||
let attempt = 0;
|
||||
const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2);
|
||||
let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250);
|
||||
const maxDelay = Number(process.env.TTS_RETRY_MAX_MS ?? 2000);
|
||||
const backoff = Number(process.env.TTS_RETRY_BACKOFF ?? 2);
|
||||
|
||||
// Retry on 429 and 5xx only; never retry aborts
|
||||
for (; ;) {
|
||||
try {
|
||||
const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal });
|
||||
return await response.arrayBuffer();
|
||||
} catch (err: unknown) {
|
||||
if (signal?.aborted || (err instanceof Error && err.name === 'AbortError')) {
|
||||
throw err;
|
||||
}
|
||||
const status = (() => {
|
||||
if (typeof err === 'object' && err !== null) {
|
||||
const rec = err as Record<string, unknown>;
|
||||
if (typeof rec.status === 'number') return rec.status as number;
|
||||
if (typeof rec.statusCode === 'number') return rec.statusCode as number;
|
||||
}
|
||||
return 0;
|
||||
})();
|
||||
const retryable = status === 429 || status >= 500;
|
||||
if (!retryable || attempt >= maxRetries) {
|
||||
throw err;
|
||||
}
|
||||
await sleep(Math.min(delay, maxDelay));
|
||||
delay = Math.min(maxDelay, delay * backoff);
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function makeCacheKey(input: {
|
||||
provider: string;
|
||||
model: string | null | undefined;
|
||||
voice: string | undefined;
|
||||
speed: number;
|
||||
format: string;
|
||||
text: string;
|
||||
instructions?: string;
|
||||
}) {
|
||||
const canonical = {
|
||||
provider: input.provider,
|
||||
model: input.model || '',
|
||||
voice: input.voice || '',
|
||||
speed: input.speed,
|
||||
format: input.format,
|
||||
text: input.text,
|
||||
// Only include instructions when present (for models like gpt-4o-mini-tts)
|
||||
instructions: input.instructions || undefined,
|
||||
};
|
||||
return createHash('sha256').update(JSON.stringify(canonical)).digest('hex');
|
||||
}
|
||||
|
||||
function formatLimitForHint(limit: number): string {
|
||||
if (!Number.isFinite(limit) || limit <= 0) return String(limit);
|
||||
if (limit >= 1_000_000) {
|
||||
|
|
@ -136,12 +49,22 @@ function formatLimitForHint(limit: number): string {
|
|||
return String(limit);
|
||||
}
|
||||
|
||||
function getUpstreamStatus(error: unknown): number | undefined {
|
||||
if (typeof error !== 'object' || error === null) return undefined;
|
||||
const rec = error as Record<string, unknown>;
|
||||
if (typeof rec.status === 'number') return rec.status;
|
||||
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
let providerForError: string | null = null;
|
||||
let didCreateDeviceIdCookie = false;
|
||||
let deviceIdToSet: string | null = null;
|
||||
|
||||
try {
|
||||
// Parse body first to get text for rate limiting
|
||||
const body = (await req.json()) as TTSRequestPayload;
|
||||
const { text, voice, speed, format, model: req_model, instructions } = body;
|
||||
const { text, voice, speed, format, model: reqModel, instructions } = body;
|
||||
|
||||
if (!text || !voice || !speed) {
|
||||
const errorBody: TTSError = {
|
||||
|
|
@ -151,13 +74,9 @@ export async function POST(req: NextRequest) {
|
|||
return NextResponse.json(errorBody, { status: 400 });
|
||||
}
|
||||
|
||||
// Auth and TTS char rate limiting check (only when auth is enabled)
|
||||
let didCreateDeviceIdCookie = false;
|
||||
let deviceIdToSet: string | null = null;
|
||||
|
||||
if (isAuthEnabled() && auth) {
|
||||
const session = await auth.api.getSession({
|
||||
headers: await headers()
|
||||
headers: await headers(),
|
||||
});
|
||||
|
||||
if (!session?.user) {
|
||||
|
|
@ -177,7 +96,6 @@ export async function POST(req: NextRequest) {
|
|||
deviceIdToSet = device.deviceId;
|
||||
}
|
||||
|
||||
// Check rate limit
|
||||
const rateLimitResult = await rateLimiter.checkAndIncrementLimit(
|
||||
{ id: session.user.id, isAnonymous },
|
||||
charCount,
|
||||
|
|
@ -220,204 +138,88 @@ export async function POST(req: NextRequest) {
|
|||
});
|
||||
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get API credentials from headers or fall back to environment variables
|
||||
const openApiKey = req.headers.get('x-openai-key') || process.env.API_KEY || 'none';
|
||||
const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
|
||||
const provider = req.headers.get('x-tts-provider') || 'openai';
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
providerForError = provider;
|
||||
console.log('Received TTS request:', { provider, req_model, voice, speed, format, hasInstructions: Boolean(instructions) });
|
||||
// Use default Kokoro model for Deepinfra if none specified, then fall back to a safe default
|
||||
const rawModel = provider === 'deepinfra' && !req_model ? 'hexgrad/Kokoro-82M' : req_model;
|
||||
const model: SpeechCreateParams['model'] = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model'];
|
||||
const varyHeader = 'x-tts-provider, x-openai-key, x-openai-base-url, x-openreader-test-namespace';
|
||||
|
||||
// Initialize OpenAI client with abort signal (OpenAI/deepinfra)
|
||||
const openai = new OpenAI({
|
||||
apiKey: openApiKey,
|
||||
baseURL: openApiBaseUrl,
|
||||
// Keep retry policy centralized in this route's fetchTTSBufferWithRetry.
|
||||
maxRetries: 0,
|
||||
// Keep upstream timeout below route max duration so we can return a controlled error.
|
||||
timeout: Number(process.env.TTS_UPSTREAM_TIMEOUT_MS ?? 45_000),
|
||||
});
|
||||
|
||||
const normalizedVoice = (
|
||||
!isKokoroModel(model as string) && voice.includes('+')
|
||||
? (voice.split('+')[0].trim())
|
||||
: voice
|
||||
) as SpeechCreateParams['voice'];
|
||||
|
||||
const resolvedFormat = format || 'mp3';
|
||||
|
||||
const createParams: ExtendedSpeechParams = {
|
||||
model: model,
|
||||
voice: normalizedVoice,
|
||||
input: text,
|
||||
speed: speed,
|
||||
response_format: resolvedFormat,
|
||||
};
|
||||
// Only add instructions if model is gpt-4o-mini-tts and instructions are provided
|
||||
if ((model as string) === 'gpt-4o-mini-tts' && instructions) {
|
||||
createParams.instructions = instructions;
|
||||
}
|
||||
|
||||
// Compute cache key and check LRU before making provider call
|
||||
const contentType = 'audio/mpeg';
|
||||
|
||||
// Preserve voice string as-is for cache key (no weight stripping)
|
||||
const voiceForKey = typeof createParams.voice === 'string'
|
||||
? createParams.voice
|
||||
: String(createParams.voice);
|
||||
|
||||
const cacheKey = makeCacheKey({
|
||||
provider,
|
||||
model: createParams.model,
|
||||
voice: voiceForKey,
|
||||
speed: Number(createParams.speed),
|
||||
format: String(createParams.response_format),
|
||||
const ttsRequest = {
|
||||
text,
|
||||
instructions: createParams.instructions,
|
||||
});
|
||||
voice,
|
||||
speed,
|
||||
format,
|
||||
model: reqModel,
|
||||
instructions,
|
||||
provider,
|
||||
apiKey: openApiKey,
|
||||
baseUrl: openApiBaseUrl,
|
||||
testNamespace,
|
||||
};
|
||||
|
||||
const contentType = getTTSContentType(format);
|
||||
const cacheKey = buildTTSCacheKey(ttsRequest);
|
||||
const etag = `W/"${cacheKey}"`;
|
||||
const ifNoneMatch = req.headers.get('if-none-match');
|
||||
|
||||
const cachedBuffer = ttsAudioCache.get(cacheKey);
|
||||
const cachedBuffer = getCachedTTSBuffer(cacheKey);
|
||||
if (cachedBuffer) {
|
||||
if (ifNoneMatch && (ifNoneMatch.includes(cacheKey) || ifNoneMatch.includes(etag))) {
|
||||
const response = new NextResponse(null, {
|
||||
status: 304,
|
||||
headers: {
|
||||
'ETag': etag,
|
||||
ETag: etag,
|
||||
'Cache-Control': 'private, max-age=1800',
|
||||
'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url'
|
||||
}
|
||||
Vary: varyHeader,
|
||||
},
|
||||
});
|
||||
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
|
||||
return response;
|
||||
}
|
||||
console.log('TTS cache HIT for key:', cacheKey.slice(0, 8));
|
||||
const response = new NextResponse(cachedBuffer, {
|
||||
|
||||
const response = new NextResponse(new Uint8Array(cachedBuffer), {
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'X-Cache': 'HIT',
|
||||
'ETag': etag,
|
||||
ETag: etag,
|
||||
'Content-Length': String(cachedBuffer.byteLength),
|
||||
'Cache-Control': 'private, max-age=1800',
|
||||
'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url'
|
||||
}
|
||||
Vary: varyHeader,
|
||||
},
|
||||
});
|
||||
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
// De-duplicate identical in-flight requests
|
||||
const existing = inflightRequests.get(cacheKey);
|
||||
if (existing) {
|
||||
console.log('TTS in-flight JOIN for key:', cacheKey.slice(0, 8));
|
||||
existing.consumers += 1;
|
||||
const buffer = await generateTTSBuffer(ttsRequest, req.signal);
|
||||
|
||||
const onAbort = () => {
|
||||
existing.consumers = Math.max(0, existing.consumers - 1);
|
||||
if (existing.consumers === 0) {
|
||||
existing.controller.abort();
|
||||
}
|
||||
};
|
||||
req.signal.addEventListener('abort', onAbort, { once: true });
|
||||
|
||||
try {
|
||||
const buffer = await existing.promise;
|
||||
const response = new NextResponse(buffer, {
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'X-Cache': 'INFLIGHT',
|
||||
'ETag': etag,
|
||||
'Content-Length': String(buffer.byteLength),
|
||||
'Cache-Control': 'private, max-age=1800',
|
||||
'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url'
|
||||
}
|
||||
});
|
||||
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
|
||||
return response;
|
||||
} finally {
|
||||
try { req.signal.removeEventListener('abort', onAbort); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const entry: InflightEntry = {
|
||||
controller,
|
||||
consumers: 1,
|
||||
promise: (async () => {
|
||||
try {
|
||||
const buffer = await fetchTTSBufferWithRetry(openai, createParams, controller.signal);
|
||||
// Save to cache
|
||||
ttsAudioCache.set(cacheKey, buffer);
|
||||
return buffer;
|
||||
} finally {
|
||||
inflightRequests.delete(cacheKey);
|
||||
}
|
||||
})()
|
||||
};
|
||||
|
||||
inflightRequests.set(cacheKey, entry);
|
||||
|
||||
const onAbort = () => {
|
||||
entry.consumers = Math.max(0, entry.consumers - 1);
|
||||
if (entry.consumers === 0) {
|
||||
entry.controller.abort();
|
||||
}
|
||||
};
|
||||
req.signal.addEventListener('abort', onAbort, { once: true });
|
||||
|
||||
let buffer: TTSAudioBuffer;
|
||||
try {
|
||||
buffer = await entry.promise;
|
||||
} finally {
|
||||
try { req.signal.removeEventListener('abort', onAbort); } catch { }
|
||||
}
|
||||
|
||||
const response = new NextResponse(buffer, {
|
||||
const response = new NextResponse(new Uint8Array(buffer), {
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'X-Cache': 'MISS',
|
||||
'ETag': etag,
|
||||
ETag: etag,
|
||||
'Content-Length': String(buffer.byteLength),
|
||||
'Cache-Control': 'private, max-age=1800',
|
||||
'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url'
|
||||
}
|
||||
Vary: varyHeader,
|
||||
},
|
||||
});
|
||||
|
||||
attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie);
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
// Check if this was an abort error
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
console.log('TTS request aborted by client');
|
||||
return new NextResponse(null, { status: 499 }); // Use 499 status for client closed request
|
||||
return new NextResponse(null, { status: 499 });
|
||||
}
|
||||
|
||||
const upstreamStatus = (() => {
|
||||
if (typeof error === 'object' && error !== null) {
|
||||
const rec = error as Record<string, unknown>;
|
||||
if (typeof rec.status === 'number') return rec.status as number;
|
||||
if (typeof rec.statusCode === 'number') return rec.statusCode as number;
|
||||
}
|
||||
return undefined;
|
||||
})();
|
||||
|
||||
const upstreamStatus = getUpstreamStatus(error);
|
||||
if (upstreamStatus === 429) {
|
||||
const problem: ProblemDetails = {
|
||||
type: PROBLEM_TYPES.upstreamRateLimited,
|
||||
|
|
@ -444,9 +246,6 @@ export async function POST(req: NextRequest) {
|
|||
message: 'Failed to generate audio',
|
||||
details: process.env.NODE_ENV !== 'production' ? String(error) : undefined,
|
||||
};
|
||||
return NextResponse.json(
|
||||
errorBody,
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json(errorBody, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ export function AudiobookExportModal({
|
|||
onGenerateAudiobook,
|
||||
onRegenerateChapter
|
||||
}: AudiobookExportModalProps) {
|
||||
const { isLoading, isDBReady, ttsProvider, ttsModel, voice: configVoice, voiceSpeed, audioPlayerSpeed } = useConfig();
|
||||
const { isLoading, isDBReady, ttsProvider, ttsModel, ttsInstructions, voice: configVoice, voiceSpeed, audioPlayerSpeed } = useConfig();
|
||||
const { availableVoices } = useTTS();
|
||||
const { progress, setProgress, estimatedTimeRemaining } = useTimeEstimation();
|
||||
const [isGenerating, setIsGenerating] = useState(false);
|
||||
|
|
@ -115,8 +115,9 @@ export function AudiobookExportModal({
|
|||
nativeSpeed,
|
||||
postSpeed,
|
||||
format,
|
||||
ttsInstructions: ttsModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined,
|
||||
};
|
||||
}, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, nativeSpeed, postSpeed, format]);
|
||||
}, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, ttsInstructions, nativeSpeed, postSpeed, format]);
|
||||
|
||||
const fetchExistingChapters = useCallback(async (soft: boolean = false) => {
|
||||
if (soft) {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import { useAuthConfig } from '@/contexts/AuthRateLimitContext';
|
|||
import { createRangeCfi } from '@/lib/client/epub';
|
||||
import { useParams } from 'next/navigation';
|
||||
import { useConfig } from './ConfigContext';
|
||||
import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client/api/audiobooks';
|
||||
import { withRetry, getAudiobookStatus, createAudiobookChapter } from '@/lib/client/api/audiobooks';
|
||||
import { CmpStr } from 'cmpstr';
|
||||
import type {
|
||||
TTSSentenceAlignment,
|
||||
|
|
@ -34,7 +34,6 @@ import type {
|
|||
} from '@/types/tts';
|
||||
import type {
|
||||
TTSRequestHeaders,
|
||||
TTSRequestPayload,
|
||||
TTSRetryOptions,
|
||||
AudiobookGenerationSettings,
|
||||
} from '@/types/client';
|
||||
|
|
@ -179,11 +178,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
const {
|
||||
apiKey,
|
||||
baseUrl,
|
||||
voiceSpeed,
|
||||
voice,
|
||||
ttsProvider,
|
||||
ttsModel,
|
||||
ttsInstructions,
|
||||
smartSentenceSplitting,
|
||||
epubHighlightEnabled,
|
||||
} = useConfig();
|
||||
|
|
@ -358,16 +353,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
if (!sections.length) throw new Error('No text content found in book');
|
||||
|
||||
const effectiveProvider = settings?.ttsProvider ?? ttsProvider;
|
||||
const effectiveModel = settings?.ttsModel ?? ttsModel;
|
||||
const effectiveVoice =
|
||||
settings?.voice ||
|
||||
voice ||
|
||||
(effectiveProvider === 'openai'
|
||||
? 'alloy'
|
||||
: effectiveProvider === 'deepinfra'
|
||||
? 'af_bella'
|
||||
: 'af_sarah');
|
||||
const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed;
|
||||
const effectiveFormat = settings?.format ?? format;
|
||||
|
||||
// Calculate total length for accurate progress tracking
|
||||
|
|
@ -448,16 +433,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
'x-tts-provider': effectiveProvider,
|
||||
};
|
||||
|
||||
const reqBody: TTSRequestPayload = {
|
||||
text: trimmedText,
|
||||
voice: effectiveVoice,
|
||||
speed: effectiveNativeSpeed,
|
||||
format: 'mp3',
|
||||
model: effectiveModel,
|
||||
instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined
|
||||
};
|
||||
|
||||
// Allow one narrow client retry for transient browser->/api/tts transport failures.
|
||||
// Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures.
|
||||
// HTTP failures are not retried client-side.
|
||||
const retryOptions: TTSRetryOptions = {
|
||||
maxRetries: 2,
|
||||
|
|
@ -465,18 +441,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
maxDelay: 300,
|
||||
};
|
||||
|
||||
const audioBuffer = await withRetry(
|
||||
async () => {
|
||||
// Check for abort before starting TTS request
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
|
||||
return await generateTTS(reqBody, reqHeaders, signal);
|
||||
},
|
||||
retryOptions
|
||||
);
|
||||
|
||||
// Get the chapter title from our pre-computed map
|
||||
let chapterTitle = sectionTitleMap.get(section.href);
|
||||
|
||||
|
|
@ -485,6 +449,25 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
chapterTitle = `Chapter ${i + 1}`;
|
||||
}
|
||||
|
||||
const chapter = await withRetry(
|
||||
async () => {
|
||||
// Check for abort before starting chapter request
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
|
||||
return await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
text: trimmedText,
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex: i,
|
||||
settings
|
||||
}, reqHeaders, signal);
|
||||
},
|
||||
retryOptions
|
||||
);
|
||||
|
||||
// Check for abort before sending to server
|
||||
if (signal?.aborted) {
|
||||
console.log('Generation cancelled before saving chapter');
|
||||
|
|
@ -494,16 +477,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
throw new Error('Audiobook generation cancelled');
|
||||
}
|
||||
|
||||
// Send to server for conversion and storage
|
||||
const chapter = await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
buffer: Array.from(new Uint8Array(audioBuffer)),
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex: i,
|
||||
settings
|
||||
}, signal);
|
||||
|
||||
if (!bookId) {
|
||||
bookId = chapter.bookId!;
|
||||
}
|
||||
|
|
@ -548,7 +521,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
console.error('Error creating audiobook:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [extractBookText, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions]);
|
||||
}, [extractBookText, apiKey, baseUrl, ttsProvider]);
|
||||
|
||||
/**
|
||||
* Regenerates a specific chapter of the audiobook
|
||||
|
|
@ -567,16 +540,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
}
|
||||
|
||||
const effectiveProvider = settings?.ttsProvider ?? ttsProvider;
|
||||
const effectiveModel = settings?.ttsModel ?? ttsModel;
|
||||
const effectiveVoice =
|
||||
settings?.voice ||
|
||||
voice ||
|
||||
(effectiveProvider === 'openai'
|
||||
? 'alloy'
|
||||
: effectiveProvider === 'deepinfra'
|
||||
? 'af_bella'
|
||||
: 'af_sarah');
|
||||
const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed;
|
||||
const effectiveFormat = settings?.format ?? format;
|
||||
|
||||
const section = sections[chapterIndex];
|
||||
|
|
@ -615,16 +578,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
'x-tts-provider': effectiveProvider,
|
||||
};
|
||||
|
||||
const reqBody: TTSRequestPayload = {
|
||||
text: trimmedText,
|
||||
voice: effectiveVoice,
|
||||
speed: effectiveNativeSpeed,
|
||||
format: 'mp3',
|
||||
model: effectiveModel,
|
||||
instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined
|
||||
};
|
||||
|
||||
// Allow one narrow client retry for transient browser->/api/tts transport failures.
|
||||
// Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures.
|
||||
// HTTP failures are not retried client-side.
|
||||
const retryOptions: TTSRetryOptions = {
|
||||
maxRetries: 2,
|
||||
|
|
@ -632,13 +586,20 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
maxDelay: 300,
|
||||
};
|
||||
|
||||
const audioBuffer = await withRetry(
|
||||
const chapter = await withRetry(
|
||||
async () => {
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
|
||||
return await generateTTS(reqBody, reqHeaders, signal);
|
||||
return await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
text: trimmedText,
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex,
|
||||
settings
|
||||
}, reqHeaders, signal);
|
||||
},
|
||||
retryOptions
|
||||
);
|
||||
|
|
@ -647,16 +608,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
throw new Error('Chapter regeneration cancelled');
|
||||
}
|
||||
|
||||
// Send to server for conversion and storage
|
||||
const chapter = await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
buffer: Array.from(new Uint8Array(audioBuffer)),
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex,
|
||||
settings
|
||||
}, signal);
|
||||
|
||||
return chapter;
|
||||
|
||||
} catch (error) {
|
||||
|
|
@ -666,7 +617,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
console.error('Error regenerating chapter:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [extractBookText, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions]);
|
||||
}, [extractBookText, apiKey, baseUrl, ttsProvider]);
|
||||
|
||||
const setRendition = useCallback((rendition: Rendition) => {
|
||||
bookRef.current = rendition.book;
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import { ensureCachedDocument } from '@/lib/client/cache/documents';
|
|||
import { useTTS } from '@/contexts/TTSContext';
|
||||
import { useConfig } from '@/contexts/ConfigContext';
|
||||
import { normalizeTextForTts } from '@/lib/shared/nlp';
|
||||
import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client/api/audiobooks';
|
||||
import { withRetry, getAudiobookStatus, createAudiobookChapter } from '@/lib/client/api/audiobooks';
|
||||
import {
|
||||
extractTextFromPDF,
|
||||
highlightPattern,
|
||||
|
|
@ -43,13 +43,11 @@ import {
|
|||
|
||||
import type {
|
||||
TTSSentenceAlignment,
|
||||
TTSAudioBuffer,
|
||||
TTSAudiobookFormat,
|
||||
TTSAudiobookChapter,
|
||||
} from '@/types/tts';
|
||||
import type {
|
||||
TTSRequestHeaders,
|
||||
TTSRequestPayload,
|
||||
TTSRetryOptions,
|
||||
AudiobookGenerationSettings,
|
||||
} from '@/types/client';
|
||||
|
|
@ -130,11 +128,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
rightMargin,
|
||||
apiKey,
|
||||
baseUrl,
|
||||
voiceSpeed,
|
||||
voice,
|
||||
ttsProvider,
|
||||
ttsModel,
|
||||
ttsInstructions,
|
||||
smartSentenceSplitting,
|
||||
} = useConfig();
|
||||
|
||||
|
|
@ -420,16 +414,6 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
}
|
||||
|
||||
const effectiveProvider = settings?.ttsProvider ?? ttsProvider;
|
||||
const effectiveModel = settings?.ttsModel ?? ttsModel;
|
||||
const effectiveVoice =
|
||||
settings?.voice ||
|
||||
voice ||
|
||||
(effectiveProvider === 'openai'
|
||||
? 'alloy'
|
||||
: effectiveProvider === 'deepinfra'
|
||||
? 'af_bella'
|
||||
: 'af_sarah');
|
||||
const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed;
|
||||
const effectiveFormat = settings?.format ?? format;
|
||||
|
||||
// First pass: extract and measure all text
|
||||
|
|
@ -504,16 +488,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
'x-tts-provider': effectiveProvider,
|
||||
};
|
||||
|
||||
const reqBody: TTSRequestPayload = {
|
||||
text,
|
||||
voice: effectiveVoice,
|
||||
speed: effectiveNativeSpeed,
|
||||
format: 'mp3',
|
||||
model: effectiveModel,
|
||||
instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined
|
||||
};
|
||||
|
||||
// Allow one narrow client retry for transient browser->/api/tts transport failures.
|
||||
// Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures.
|
||||
// HTTP failures are not retried client-side.
|
||||
const retryOptions: TTSRetryOptions = {
|
||||
maxRetries: 2,
|
||||
|
|
@ -522,20 +497,27 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
};
|
||||
|
||||
try {
|
||||
const audioBuffer = await withRetry(
|
||||
const chapterTitle = `Page ${i + 1}`;
|
||||
|
||||
const chapter = await withRetry(
|
||||
async () => {
|
||||
// Check for abort before starting TTS request
|
||||
// Check for abort before starting chapter request
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
|
||||
return await generateTTS(reqBody, reqHeaders, signal);
|
||||
return await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
text,
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex: i,
|
||||
settings
|
||||
}, reqHeaders, signal);
|
||||
},
|
||||
retryOptions
|
||||
);
|
||||
|
||||
const chapterTitle = `Page ${i + 1}`;
|
||||
|
||||
// Check for abort before sending to server
|
||||
if (signal?.aborted) {
|
||||
console.log('Generation cancelled before saving page');
|
||||
|
|
@ -545,16 +527,6 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
throw new Error('Audiobook generation cancelled');
|
||||
}
|
||||
|
||||
// Send to server for conversion and storage
|
||||
const chapter = await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
buffer: Array.from(new Uint8Array(audioBuffer)),
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex: i,
|
||||
settings
|
||||
}, signal);
|
||||
|
||||
if (!bookId) {
|
||||
bookId = chapter.bookId!;
|
||||
}
|
||||
|
|
@ -599,7 +571,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
console.error('Error creating audiobook:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions, smartSentenceSplitting]);
|
||||
}, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, ttsProvider, smartSentenceSplitting]);
|
||||
|
||||
/**
|
||||
* Regenerates a specific chapter (page) of the PDF audiobook
|
||||
|
|
@ -617,16 +589,6 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
}
|
||||
|
||||
const effectiveProvider = settings?.ttsProvider ?? ttsProvider;
|
||||
const effectiveModel = settings?.ttsModel ?? ttsModel;
|
||||
const effectiveVoice =
|
||||
settings?.voice ||
|
||||
voice ||
|
||||
(effectiveProvider === 'openai'
|
||||
? 'alloy'
|
||||
: effectiveProvider === 'deepinfra'
|
||||
? 'af_bella'
|
||||
: 'af_sarah');
|
||||
const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed;
|
||||
const effectiveFormat = settings?.format ?? format;
|
||||
|
||||
// IMPORTANT: Chapter indices are based on non-empty pages used during generation.
|
||||
|
|
@ -678,16 +640,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
'x-tts-provider': effectiveProvider,
|
||||
};
|
||||
|
||||
const reqBody: TTSRequestPayload = {
|
||||
text: textForTTS,
|
||||
voice: effectiveVoice,
|
||||
speed: effectiveNativeSpeed,
|
||||
format: 'mp3',
|
||||
model: effectiveModel,
|
||||
instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined
|
||||
};
|
||||
|
||||
// Allow one narrow client retry for transient browser->/api/tts transport failures.
|
||||
// Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures.
|
||||
// HTTP failures are not retried client-side.
|
||||
const retryOptions: TTSRetryOptions = {
|
||||
maxRetries: 2,
|
||||
|
|
@ -695,13 +648,20 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
maxDelay: 300,
|
||||
};
|
||||
|
||||
const audioBuffer: TTSAudioBuffer = await withRetry(
|
||||
const chapter = await withRetry(
|
||||
async () => {
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
|
||||
return await generateTTS(reqBody, reqHeaders, signal);
|
||||
return await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
text: textForTTS,
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex,
|
||||
settings
|
||||
}, reqHeaders, signal);
|
||||
},
|
||||
retryOptions
|
||||
);
|
||||
|
|
@ -710,16 +670,6 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
throw new Error('Page regeneration cancelled');
|
||||
}
|
||||
|
||||
// Send to server for conversion and storage
|
||||
const chapter = await createAudiobookChapter({
|
||||
chapterTitle,
|
||||
buffer: Array.from(new Uint8Array(audioBuffer)),
|
||||
bookId,
|
||||
format: effectiveFormat,
|
||||
chapterIndex,
|
||||
settings
|
||||
}, signal);
|
||||
|
||||
return chapter;
|
||||
|
||||
} catch (error) {
|
||||
|
|
@ -729,7 +679,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
console.error('Error regenerating page:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions, smartSentenceSplitting]);
|
||||
}, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, ttsProvider, smartSentenceSplitting]);
|
||||
|
||||
/**
|
||||
* Effect hook to initialize TTS as non-EPUB mode
|
||||
|
|
|
|||
|
|
@ -115,12 +115,14 @@ export const getAudiobookStatus = async (bookId: string): Promise<AudiobookStatu
|
|||
|
||||
export const createAudiobookChapter = async (
|
||||
payload: CreateChapterPayload,
|
||||
headers: TTSRequestHeaders,
|
||||
signal?: AbortSignal
|
||||
): Promise<TTSAudiobookChapter> => {
|
||||
const response = await fetch(`/api/audiobook/chapter`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal
|
||||
|
|
|
|||
296
src/lib/server/tts/generate.ts
Normal file
296
src/lib/server/tts/generate.ts
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
import OpenAI from 'openai';
|
||||
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { createHash } from 'crypto';
|
||||
import { access, readFile } from 'fs/promises';
|
||||
import { resolve } from 'path';
|
||||
|
||||
export interface ServerTTSRequest {
|
||||
text: string;
|
||||
voice: string;
|
||||
speed: number;
|
||||
format?: string;
|
||||
model?: string | null;
|
||||
instructions?: string;
|
||||
provider: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
testNamespace?: string | null;
|
||||
}
|
||||
|
||||
type CustomVoice = string;
|
||||
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
|
||||
voice: SpeechCreateParams['voice'] | CustomVoice;
|
||||
instructions?: string;
|
||||
};
|
||||
|
||||
type ResolvedServerTTSRequest = {
|
||||
text: string;
|
||||
voice: string;
|
||||
speed: number;
|
||||
format: string;
|
||||
model: SpeechCreateParams['model'];
|
||||
instructions?: string;
|
||||
provider: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
testNamespace?: string | null;
|
||||
};
|
||||
|
||||
type InflightEntry = {
|
||||
promise: Promise<Buffer>;
|
||||
controller: AbortController;
|
||||
consumers: number;
|
||||
};
|
||||
|
||||
const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB
|
||||
const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes
|
||||
|
||||
const ttsAudioCache = new LRUCache<string, Buffer>({
|
||||
maxSize: TTS_CACHE_MAX_SIZE_BYTES,
|
||||
sizeCalculation: (value) => value.byteLength,
|
||||
ttl: TTS_CACHE_TTL_MS,
|
||||
});
|
||||
|
||||
const inflightRequests = new Map<string, InflightEntry>();
|
||||
|
||||
const TEST_TTS_MOCK_PATH = resolve(process.cwd(), 'tests/files/sample.mp3');
|
||||
let testMockTtsBufferPromise: Promise<Buffer | null> | null = null;
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((res) => setTimeout(res, ms));
|
||||
}
|
||||
|
||||
function getUpstreamStatus(error: unknown): number | undefined {
|
||||
if (typeof error !== 'object' || error === null) return undefined;
|
||||
const rec = error as Record<string, unknown>;
|
||||
if (typeof rec.status === 'number') return rec.status;
|
||||
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
|
||||
const provider = input.provider || 'openai';
|
||||
const rawModel = provider === 'deepinfra' && !input.model ? 'hexgrad/Kokoro-82M' : input.model;
|
||||
const model = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model'];
|
||||
|
||||
const normalizedVoice = (
|
||||
!isKokoroModel(model as string) && input.voice.includes('+')
|
||||
? input.voice.split('+')[0].trim()
|
||||
: input.voice
|
||||
) as string;
|
||||
|
||||
const format = input.format || 'mp3';
|
||||
const speed = Number.isFinite(Number(input.speed)) ? Number(input.speed) : 1;
|
||||
const instructions = (model as string) === 'gpt-4o-mini-tts' && input.instructions
|
||||
? input.instructions
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
text: input.text,
|
||||
voice: normalizedVoice,
|
||||
speed,
|
||||
format,
|
||||
model,
|
||||
instructions,
|
||||
provider,
|
||||
apiKey: input.apiKey,
|
||||
baseUrl: input.baseUrl,
|
||||
testNamespace: input.testNamespace || null,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCacheKey(input: {
|
||||
provider: string;
|
||||
model: string | null | undefined;
|
||||
voice: string | undefined;
|
||||
speed: number;
|
||||
format: string;
|
||||
text: string;
|
||||
instructions?: string;
|
||||
testNamespace?: string | null;
|
||||
}) {
|
||||
const canonical = {
|
||||
provider: input.provider,
|
||||
model: input.model || '',
|
||||
voice: input.voice || '',
|
||||
speed: input.speed,
|
||||
format: input.format,
|
||||
text: input.text,
|
||||
instructions: input.instructions || undefined,
|
||||
testNamespace: input.testNamespace || undefined,
|
||||
};
|
||||
return createHash('sha256').update(JSON.stringify(canonical)).digest('hex');
|
||||
}
|
||||
|
||||
export function buildTTSCacheKey(request: ServerTTSRequest): string {
|
||||
const resolved = resolveTTSRequest(request);
|
||||
return makeCacheKey({
|
||||
provider: resolved.provider,
|
||||
model: resolved.model,
|
||||
voice: resolved.voice,
|
||||
speed: resolved.speed,
|
||||
format: resolved.format,
|
||||
text: resolved.text,
|
||||
instructions: resolved.instructions,
|
||||
testNamespace: resolved.testNamespace,
|
||||
});
|
||||
}
|
||||
|
||||
export function getCachedTTSBuffer(cacheKey: string): Buffer | undefined {
|
||||
return ttsAudioCache.get(cacheKey);
|
||||
}
|
||||
|
||||
export function getTTSContentType(format: string | undefined): string {
|
||||
return (format || 'mp3') === 'mp3' ? 'audio/mpeg' : 'application/octet-stream';
|
||||
}
|
||||
|
||||
async function getTestMockTtsBuffer(testNamespace?: string | null): Promise<Buffer | null> {
|
||||
if (!testNamespace) return null;
|
||||
if (!testMockTtsBufferPromise) {
|
||||
testMockTtsBufferPromise = (async () => {
|
||||
try {
|
||||
await access(TEST_TTS_MOCK_PATH);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return readFile(TEST_TTS_MOCK_PATH);
|
||||
})();
|
||||
}
|
||||
return testMockTtsBufferPromise;
|
||||
}
|
||||
|
||||
async function fetchTTSBufferWithRetry(
|
||||
openai: OpenAI,
|
||||
createParams: ExtendedSpeechParams,
|
||||
signal: AbortSignal
|
||||
): Promise<Buffer> {
|
||||
let attempt = 0;
|
||||
const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2);
|
||||
let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250);
|
||||
const maxDelay = Number(process.env.TTS_RETRY_MAX_MS ?? 2000);
|
||||
const backoff = Number(process.env.TTS_RETRY_BACKOFF ?? 2);
|
||||
|
||||
for (; ;) {
|
||||
try {
|
||||
const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal });
|
||||
const buffer = await response.arrayBuffer();
|
||||
return Buffer.from(buffer);
|
||||
} catch (error) {
|
||||
if (signal.aborted || (error instanceof Error && error.name === 'AbortError')) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const status = getUpstreamStatus(error) ?? 0;
|
||||
const retryable = status === 429 || status >= 500;
|
||||
if (!retryable || attempt >= maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await sleep(Math.min(delay, maxDelay));
|
||||
delay = Math.min(maxDelay, delay * backoff);
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runProviderRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise<Buffer> {
|
||||
const mockBuffer = await getTestMockTtsBuffer(request.testNamespace);
|
||||
if (mockBuffer) return mockBuffer;
|
||||
|
||||
const openai = new OpenAI({
|
||||
apiKey: request.apiKey,
|
||||
baseURL: request.baseUrl,
|
||||
maxRetries: 0,
|
||||
timeout: Number(process.env.TTS_UPSTREAM_TIMEOUT_MS ?? 45_000),
|
||||
});
|
||||
|
||||
const createParams: ExtendedSpeechParams = {
|
||||
model: request.model,
|
||||
voice: request.voice as SpeechCreateParams['voice'],
|
||||
input: request.text,
|
||||
speed: request.speed,
|
||||
response_format: request.format as SpeechCreateParams['response_format'],
|
||||
};
|
||||
|
||||
if (request.instructions) {
|
||||
createParams.instructions = request.instructions;
|
||||
}
|
||||
|
||||
return fetchTTSBufferWithRetry(openai, createParams, signal);
|
||||
}
|
||||
|
||||
export async function generateTTSBuffer(
|
||||
request: ServerTTSRequest,
|
||||
signal?: AbortSignal
|
||||
): Promise<Buffer> {
|
||||
const resolved = resolveTTSRequest(request);
|
||||
const cacheKey = makeCacheKey({
|
||||
provider: resolved.provider,
|
||||
model: resolved.model,
|
||||
voice: resolved.voice,
|
||||
speed: resolved.speed,
|
||||
format: resolved.format,
|
||||
text: resolved.text,
|
||||
instructions: resolved.instructions,
|
||||
testNamespace: resolved.testNamespace,
|
||||
});
|
||||
|
||||
const cachedBuffer = ttsAudioCache.get(cacheKey);
|
||||
if (cachedBuffer) return cachedBuffer;
|
||||
|
||||
const existing = inflightRequests.get(cacheKey);
|
||||
if (existing) {
|
||||
existing.consumers += 1;
|
||||
|
||||
const onAbort = () => {
|
||||
existing.consumers = Math.max(0, existing.consumers - 1);
|
||||
if (existing.consumers === 0) {
|
||||
existing.controller.abort();
|
||||
}
|
||||
};
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
try {
|
||||
return await existing.promise;
|
||||
} finally {
|
||||
try {
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
} catch { }
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const entry: InflightEntry = {
|
||||
controller,
|
||||
consumers: 1,
|
||||
promise: (async () => {
|
||||
try {
|
||||
const buffer = await runProviderRequest(resolved, controller.signal);
|
||||
ttsAudioCache.set(cacheKey, buffer);
|
||||
return buffer;
|
||||
} finally {
|
||||
inflightRequests.delete(cacheKey);
|
||||
}
|
||||
})(),
|
||||
};
|
||||
|
||||
inflightRequests.set(cacheKey, entry);
|
||||
|
||||
const onAbort = () => {
|
||||
entry.consumers = Math.max(0, entry.consumers - 1);
|
||||
if (entry.consumers === 0) {
|
||||
entry.controller.abort();
|
||||
}
|
||||
};
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
try {
|
||||
return await entry.promise;
|
||||
} finally {
|
||||
try {
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
} catch { }
|
||||
}
|
||||
}
|
||||
|
|
@ -56,11 +56,12 @@ export interface AudiobookGenerationSettings {
|
|||
nativeSpeed: number;
|
||||
postSpeed: number;
|
||||
format: TTSAudiobookFormat;
|
||||
ttsInstructions?: string;
|
||||
}
|
||||
|
||||
export interface CreateChapterPayload {
|
||||
chapterTitle: string;
|
||||
buffer: TTSAudioBytes; // Array.from(new Uint8Array(audioBuffer))
|
||||
text: string;
|
||||
bookId: string;
|
||||
format: TTSAudiobookFormat;
|
||||
chapterIndex: number;
|
||||
|
|
|
|||
Loading…
Reference in a new issue