diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 15d7450..dbb08b3 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -8,6 +8,9 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { audiobooks, audiobookChapters } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; +import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { getClientIp } from '@/lib/server/rate-limit/request-ip'; +import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; import { deleteAudiobookObject, getAudiobookObjectBuffer, @@ -25,14 +28,15 @@ import { isS3Configured } from '@/lib/server/storage/s3'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobooks/user-scope'; import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin'; +import { generateTTSBuffer } from '@/lib/server/tts/generate'; import type { AudiobookGenerationSettings } from '@/types/client'; -import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts'; +import type { TTSAudiobookFormat } from '@/types/tts'; export const dynamic = 'force-dynamic'; interface ConversionRequest { chapterTitle: string; - buffer: TTSAudioBytes; + text: string; bookId?: string; format?: TTSAudiobookFormat; chapterIndex?: number; @@ -47,6 +51,35 @@ type ChapterObject = { }; const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const PROBLEM_TYPES = { + dailyQuotaExceeded: 'https://openreader.app/problems/daily-quota-exceeded', +} as const; + +type ProblemDetails = { + type: string; + title: string; + status: number; + detail?: string; + instance?: string; + code?: string; + [key: string]: unknown; +}; + +function attachDeviceIdCookie(response: NextResponse, deviceId: string | null, didCreate: boolean) { + if (didCreate && deviceId) { + setDeviceIdCookie(response, deviceId); + } +} + +function formatLimitForHint(limit: number): string { + if (!Number.isFinite(limit) || limit <= 0) return String(limit); + if (limit >= 1_000_000) { + const m = limit / 1_000_000; + return `${m % 1 === 0 ? m.toFixed(0) : m.toFixed(1)}M`; + } + if (limit >= 1_000) return `${Math.round(limit / 1_000)}K`; + return String(limit); +} function isSafeId(value: string): boolean { return SAFE_ID_REGEX.test(value); @@ -208,16 +241,21 @@ function findChapterFileNameByIndex(fileNames: string[], index: number): { fileN export async function POST(request: NextRequest) { let workDir: string | null = null; + let didCreateDeviceIdCookie = false; + let deviceIdToSet: string | null = null; try { if (!isS3Configured()) return s3NotConfiguredResponse(); const data: ConversionRequest = await request.json(); const requestedFormat = data.format || 'm4b'; + if (!data.text || typeof data.text !== 'string') { + return NextResponse.json({ error: 'Missing text for TTS generation' }, { status: 400 }); + } const ctxOrRes = await requireAuthContext(request); if (ctxOrRes instanceof Response) return ctxOrRes; - const { userId, authEnabled } = ctxOrRes; + const { userId, authEnabled, user } = ctxOrRes; const testNamespace = getOpenReaderTestNamespace(request.headers); const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); const { preferredUserId, allowedUserIds } = buildAllowedAudiobookUserIds(authEnabled, userId, unclaimedUserId); @@ -270,7 +308,8 @@ export async function POST(request: NextRequest) { existingSettings.voice !== incomingSettings.voice || existingSettings.nativeSpeed !== incomingSettings.nativeSpeed || existingSettings.postSpeed !== incomingSettings.postSpeed || - existingSettings.format !== incomingSettings.format; + existingSettings.format !== incomingSettings.format || + (existingSettings.ttsInstructions || '') !== (incomingSettings.ttsInstructions || ''); if (mismatch) { return NextResponse.json({ error: 'Audiobook settings mismatch', settings: existingSettings }, { status: 409 }); } @@ -309,12 +348,102 @@ export async function POST(request: NextRequest) { chapterIndex = next; } + const provider = request.headers.get('x-tts-provider') + || incomingSettings?.ttsProvider + || existingSettings?.ttsProvider + || 'openai'; + const openApiKey = request.headers.get('x-openai-key') || process.env.API_KEY || 'none'; + const openApiBaseUrl = request.headers.get('x-openai-base-url') || process.env.API_BASE; + const model = incomingSettings?.ttsModel ?? existingSettings?.ttsModel; + const voice = incomingSettings?.voice + || existingSettings?.voice + || (provider === 'openai' + ? 'alloy' + : provider === 'deepinfra' + ? 'af_bella' + : 'af_sarah'); + const rawNativeSpeed = incomingSettings?.nativeSpeed ?? existingSettings?.nativeSpeed ?? 1; + const nativeSpeed = Number.isFinite(Number(rawNativeSpeed)) ? Number(rawNativeSpeed) : 1; + const instructions = incomingSettings?.ttsInstructions ?? existingSettings?.ttsInstructions; + + if (authEnabled && userId && isTtsRateLimitEnabled()) { + const isAnonymous = Boolean(user?.isAnonymous); + const charCount = data.text.length; + const ip = getClientIp(request); + const device = isAnonymous ? getOrCreateDeviceId(request) : null; + if (device?.didCreate) { + didCreateDeviceIdCookie = true; + deviceIdToSet = device.deviceId; + } + + const rateLimitResult = await rateLimiter.checkAndIncrementLimit( + { id: userId, isAnonymous }, + charCount, + { + deviceId: device?.deviceId ?? null, + ip, + }, + ); + + if (!rateLimitResult.allowed) { + const resetTimeMs = rateLimitResult.resetTimeMs; + const retryAfterSeconds = Math.max( + 0, + Math.ceil((resetTimeMs - Date.now()) / 1000), + ); + + const problem: ProblemDetails = { + type: PROBLEM_TYPES.dailyQuotaExceeded, + title: 'Daily quota exceeded', + status: 429, + detail: 'Daily character limit exceeded', + code: 'USER_DAILY_QUOTA_EXCEEDED', + currentCount: rateLimitResult.currentCount, + limit: rateLimitResult.limit, + remainingChars: rateLimitResult.remainingChars, + resetTimeMs, + userType: isAnonymous ? 'anonymous' : 'authenticated', + upgradeHint: isAnonymous + ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + : undefined, + instance: request.nextUrl.pathname, + }; + + const response = new NextResponse(JSON.stringify(problem), { + status: 429, + headers: { + 'Content-Type': 'application/problem+json', + 'Retry-After': String(retryAfterSeconds), + }, + }); + + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } + } + + const ttsBuffer = await generateTTSBuffer( + { + text: data.text, + voice, + speed: nativeSpeed, + format: 'mp3', + model, + instructions, + provider, + apiKey: openApiKey, + baseUrl: openApiBaseUrl, + testNamespace, + }, + request.signal, + ); + workDir = await mkdtemp(join(tmpdir(), 'openreader-audiobook-')); const inputPath = join(workDir, `${chapterIndex}-input.mp3`); const chapterOutputTempPath = join(workDir, `${chapterIndex}-chapter.tmp.${format}`); const titleTag = encodeChapterTitleTag(chapterIndex, data.chapterTitle); - await writeFile(inputPath, Buffer.from(new Uint8Array(data.buffer))); + await writeFile(inputPath, ttsBuffer); const canCopyMp3WithoutReencode = format === 'mp3' && postSpeed === 1; if (canCopyMp3WithoutReencode) { @@ -398,7 +527,7 @@ export async function POST(request: NextRequest) { set: { title: data.chapterTitle, duration, format, filePath: finalChapterName }, }); - return NextResponse.json({ + const response = NextResponse.json({ index: chapterIndex, title: data.chapterTitle, duration, @@ -406,12 +535,18 @@ export async function POST(request: NextRequest) { bookId, format, }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; } catch (error) { - if ((error as Error)?.message === 'ABORTED' || request.signal.aborted) { - return NextResponse.json({ error: 'cancelled' }, { status: 499 }); + if ((error as Error)?.message === 'ABORTED' || (error as Error)?.name === 'AbortError' || request.signal.aborted) { + const response = NextResponse.json({ error: 'cancelled' }, { status: 499 }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; } console.error('Error processing audio chapter:', error); - return NextResponse.json({ error: 'Failed to process audio chapter' }, { status: 500 }); + const response = NextResponse.json({ error: 'Failed to process audio chapter' }, { status: 500 }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; } finally { if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {}); } diff --git a/src/app/api/tts/route.ts b/src/app/api/tts/route.ts index 3c8cc2e..43de629 100644 --- a/src/app/api/tts/route.ts +++ b/src/app/api/tts/route.ts @@ -1,17 +1,19 @@ import { NextRequest, NextResponse } from 'next/server'; -import OpenAI from 'openai'; -import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs'; -import { isKokoroModel } from '@/lib/shared/kokoro'; -import { LRUCache } from 'lru-cache'; -import { createHash } from 'crypto'; import type { TTSRequestPayload } from '@/types/client'; -import type { TTSError, TTSAudioBuffer } from '@/types/tts'; +import type { TTSError } from '@/types/tts'; import { headers } from 'next/headers'; import { auth } from '@/lib/server/auth/auth'; import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; import { isAuthEnabled } from '@/lib/server/auth/config'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; +import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; +import { + buildTTSCacheKey, + generateTTSBuffer, + getCachedTTSBuffer, + getTTSContentType, +} from '@/lib/server/tts/generate'; export const runtime = 'nodejs'; export const maxDuration = 60; @@ -22,30 +24,6 @@ function attachDeviceIdCookie(response: NextResponse, deviceId: string | null, d } } -type CustomVoice = string; -type ExtendedSpeechParams = Omit & { - voice: SpeechCreateParams['voice'] | CustomVoice; - instructions?: string; -}; -type AudioBufferValue = TTSAudioBuffer; - -const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB -const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes - -const ttsAudioCache = new LRUCache({ - maxSize: TTS_CACHE_MAX_SIZE_BYTES, - sizeCalculation: (value) => value.byteLength, - ttl: TTS_CACHE_TTL_MS, -}); - -type InflightEntry = { - promise: Promise; - controller: AbortController; - consumers: number; -}; - -const inflightRequests = new Map(); - const PROBLEM_TYPES = { dailyQuotaExceeded: 'https://openreader.app/problems/daily-quota-exceeded', upstreamRateLimited: 'https://openreader.app/problems/upstream-rate-limited', @@ -61,71 +39,6 @@ type ProblemDetails = { [key: string]: unknown; }; -function sleep(ms: number) { - return new Promise((res) => setTimeout(res, ms)); -} - -async function fetchTTSBufferWithRetry( - openai: OpenAI, - createParams: ExtendedSpeechParams, - signal: AbortSignal -): Promise { - let attempt = 0; - const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2); - let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250); - const maxDelay = Number(process.env.TTS_RETRY_MAX_MS ?? 2000); - const backoff = Number(process.env.TTS_RETRY_BACKOFF ?? 2); - - // Retry on 429 and 5xx only; never retry aborts - for (; ;) { - try { - const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal }); - return await response.arrayBuffer(); - } catch (err: unknown) { - if (signal?.aborted || (err instanceof Error && err.name === 'AbortError')) { - throw err; - } - const status = (() => { - if (typeof err === 'object' && err !== null) { - const rec = err as Record; - if (typeof rec.status === 'number') return rec.status as number; - if (typeof rec.statusCode === 'number') return rec.statusCode as number; - } - return 0; - })(); - const retryable = status === 429 || status >= 500; - if (!retryable || attempt >= maxRetries) { - throw err; - } - await sleep(Math.min(delay, maxDelay)); - delay = Math.min(maxDelay, delay * backoff); - attempt += 1; - } - } -} - -function makeCacheKey(input: { - provider: string; - model: string | null | undefined; - voice: string | undefined; - speed: number; - format: string; - text: string; - instructions?: string; -}) { - const canonical = { - provider: input.provider, - model: input.model || '', - voice: input.voice || '', - speed: input.speed, - format: input.format, - text: input.text, - // Only include instructions when present (for models like gpt-4o-mini-tts) - instructions: input.instructions || undefined, - }; - return createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); -} - function formatLimitForHint(limit: number): string { if (!Number.isFinite(limit) || limit <= 0) return String(limit); if (limit >= 1_000_000) { @@ -136,12 +49,22 @@ function formatLimitForHint(limit: number): string { return String(limit); } +function getUpstreamStatus(error: unknown): number | undefined { + if (typeof error !== 'object' || error === null) return undefined; + const rec = error as Record; + if (typeof rec.status === 'number') return rec.status; + if (typeof rec.statusCode === 'number') return rec.statusCode; + return undefined; +} + export async function POST(req: NextRequest) { let providerForError: string | null = null; + let didCreateDeviceIdCookie = false; + let deviceIdToSet: string | null = null; + try { - // Parse body first to get text for rate limiting const body = (await req.json()) as TTSRequestPayload; - const { text, voice, speed, format, model: req_model, instructions } = body; + const { text, voice, speed, format, model: reqModel, instructions } = body; if (!text || !voice || !speed) { const errorBody: TTSError = { @@ -151,13 +74,9 @@ export async function POST(req: NextRequest) { return NextResponse.json(errorBody, { status: 400 }); } - // Auth and TTS char rate limiting check (only when auth is enabled) - let didCreateDeviceIdCookie = false; - let deviceIdToSet: string | null = null; - if (isAuthEnabled() && auth) { const session = await auth.api.getSession({ - headers: await headers() + headers: await headers(), }); if (!session?.user) { @@ -177,7 +96,6 @@ export async function POST(req: NextRequest) { deviceIdToSet = device.deviceId; } - // Check rate limit const rateLimitResult = await rateLimiter.checkAndIncrementLimit( { id: session.user.id, isAnonymous }, charCount, @@ -220,204 +138,88 @@ export async function POST(req: NextRequest) { }); attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); - return response; } } } - // Get API credentials from headers or fall back to environment variables const openApiKey = req.headers.get('x-openai-key') || process.env.API_KEY || 'none'; const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE; const provider = req.headers.get('x-tts-provider') || 'openai'; + const testNamespace = getOpenReaderTestNamespace(req.headers); providerForError = provider; - console.log('Received TTS request:', { provider, req_model, voice, speed, format, hasInstructions: Boolean(instructions) }); - // Use default Kokoro model for Deepinfra if none specified, then fall back to a safe default - const rawModel = provider === 'deepinfra' && !req_model ? 'hexgrad/Kokoro-82M' : req_model; - const model: SpeechCreateParams['model'] = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model']; + const varyHeader = 'x-tts-provider, x-openai-key, x-openai-base-url, x-openreader-test-namespace'; - // Initialize OpenAI client with abort signal (OpenAI/deepinfra) - const openai = new OpenAI({ - apiKey: openApiKey, - baseURL: openApiBaseUrl, - // Keep retry policy centralized in this route's fetchTTSBufferWithRetry. - maxRetries: 0, - // Keep upstream timeout below route max duration so we can return a controlled error. - timeout: Number(process.env.TTS_UPSTREAM_TIMEOUT_MS ?? 45_000), - }); - - const normalizedVoice = ( - !isKokoroModel(model as string) && voice.includes('+') - ? (voice.split('+')[0].trim()) - : voice - ) as SpeechCreateParams['voice']; - - const resolvedFormat = format || 'mp3'; - - const createParams: ExtendedSpeechParams = { - model: model, - voice: normalizedVoice, - input: text, - speed: speed, - response_format: resolvedFormat, - }; - // Only add instructions if model is gpt-4o-mini-tts and instructions are provided - if ((model as string) === 'gpt-4o-mini-tts' && instructions) { - createParams.instructions = instructions; - } - - // Compute cache key and check LRU before making provider call - const contentType = 'audio/mpeg'; - - // Preserve voice string as-is for cache key (no weight stripping) - const voiceForKey = typeof createParams.voice === 'string' - ? createParams.voice - : String(createParams.voice); - - const cacheKey = makeCacheKey({ - provider, - model: createParams.model, - voice: voiceForKey, - speed: Number(createParams.speed), - format: String(createParams.response_format), + const ttsRequest = { text, - instructions: createParams.instructions, - }); + voice, + speed, + format, + model: reqModel, + instructions, + provider, + apiKey: openApiKey, + baseUrl: openApiBaseUrl, + testNamespace, + }; + const contentType = getTTSContentType(format); + const cacheKey = buildTTSCacheKey(ttsRequest); const etag = `W/"${cacheKey}"`; const ifNoneMatch = req.headers.get('if-none-match'); - const cachedBuffer = ttsAudioCache.get(cacheKey); + const cachedBuffer = getCachedTTSBuffer(cacheKey); if (cachedBuffer) { if (ifNoneMatch && (ifNoneMatch.includes(cacheKey) || ifNoneMatch.includes(etag))) { const response = new NextResponse(null, { status: 304, headers: { - 'ETag': etag, + ETag: etag, 'Cache-Control': 'private, max-age=1800', - 'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url' - } + Vary: varyHeader, + }, }); attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); - return response; } - console.log('TTS cache HIT for key:', cacheKey.slice(0, 8)); - const response = new NextResponse(cachedBuffer, { + + const response = new NextResponse(new Uint8Array(cachedBuffer), { headers: { 'Content-Type': contentType, 'X-Cache': 'HIT', - 'ETag': etag, + ETag: etag, 'Content-Length': String(cachedBuffer.byteLength), 'Cache-Control': 'private, max-age=1800', - 'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url' - } + Vary: varyHeader, + }, }); attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); - return response; } - // De-duplicate identical in-flight requests - const existing = inflightRequests.get(cacheKey); - if (existing) { - console.log('TTS in-flight JOIN for key:', cacheKey.slice(0, 8)); - existing.consumers += 1; + const buffer = await generateTTSBuffer(ttsRequest, req.signal); - const onAbort = () => { - existing.consumers = Math.max(0, existing.consumers - 1); - if (existing.consumers === 0) { - existing.controller.abort(); - } - }; - req.signal.addEventListener('abort', onAbort, { once: true }); - - try { - const buffer = await existing.promise; - const response = new NextResponse(buffer, { - headers: { - 'Content-Type': contentType, - 'X-Cache': 'INFLIGHT', - 'ETag': etag, - 'Content-Length': String(buffer.byteLength), - 'Cache-Control': 'private, max-age=1800', - 'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url' - } - }); - - attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); - - return response; - } finally { - try { req.signal.removeEventListener('abort', onAbort); } catch { } - } - } - - const controller = new AbortController(); - const entry: InflightEntry = { - controller, - consumers: 1, - promise: (async () => { - try { - const buffer = await fetchTTSBufferWithRetry(openai, createParams, controller.signal); - // Save to cache - ttsAudioCache.set(cacheKey, buffer); - return buffer; - } finally { - inflightRequests.delete(cacheKey); - } - })() - }; - - inflightRequests.set(cacheKey, entry); - - const onAbort = () => { - entry.consumers = Math.max(0, entry.consumers - 1); - if (entry.consumers === 0) { - entry.controller.abort(); - } - }; - req.signal.addEventListener('abort', onAbort, { once: true }); - - let buffer: TTSAudioBuffer; - try { - buffer = await entry.promise; - } finally { - try { req.signal.removeEventListener('abort', onAbort); } catch { } - } - - const response = new NextResponse(buffer, { + const response = new NextResponse(new Uint8Array(buffer), { headers: { 'Content-Type': contentType, 'X-Cache': 'MISS', - 'ETag': etag, + ETag: etag, 'Content-Length': String(buffer.byteLength), 'Cache-Control': 'private, max-age=1800', - 'Vary': 'x-tts-provider, x-openai-key, x-openai-base-url' - } + Vary: varyHeader, + }, }); attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); - return response; } catch (error) { - // Check if this was an abort error if (error instanceof Error && error.name === 'AbortError') { - console.log('TTS request aborted by client'); - return new NextResponse(null, { status: 499 }); // Use 499 status for client closed request + return new NextResponse(null, { status: 499 }); } - const upstreamStatus = (() => { - if (typeof error === 'object' && error !== null) { - const rec = error as Record; - if (typeof rec.status === 'number') return rec.status as number; - if (typeof rec.statusCode === 'number') return rec.statusCode as number; - } - return undefined; - })(); - + const upstreamStatus = getUpstreamStatus(error); if (upstreamStatus === 429) { const problem: ProblemDetails = { type: PROBLEM_TYPES.upstreamRateLimited, @@ -444,9 +246,6 @@ export async function POST(req: NextRequest) { message: 'Failed to generate audio', details: process.env.NODE_ENV !== 'production' ? String(error) : undefined, }; - return NextResponse.json( - errorBody, - { status: 500 } - ); + return NextResponse.json(errorBody, { status: 500 }); } } diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index 87f5ae1..b7341f4 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -47,7 +47,7 @@ export function AudiobookExportModal({ onGenerateAudiobook, onRegenerateChapter }: AudiobookExportModalProps) { - const { isLoading, isDBReady, ttsProvider, ttsModel, voice: configVoice, voiceSpeed, audioPlayerSpeed } = useConfig(); + const { isLoading, isDBReady, ttsProvider, ttsModel, ttsInstructions, voice: configVoice, voiceSpeed, audioPlayerSpeed } = useConfig(); const { availableVoices } = useTTS(); const { progress, setProgress, estimatedTimeRemaining } = useTimeEstimation(); const [isGenerating, setIsGenerating] = useState(false); @@ -115,8 +115,9 @@ export function AudiobookExportModal({ nativeSpeed, postSpeed, format, + ttsInstructions: ttsModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined, }; - }, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, nativeSpeed, postSpeed, format]); + }, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, ttsInstructions, nativeSpeed, postSpeed, format]); const fetchExistingChapters = useCallback(async (soft: boolean = false) => { if (soft) { diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index f2bb968..875c44b 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -25,7 +25,7 @@ import { useAuthConfig } from '@/contexts/AuthRateLimitContext'; import { createRangeCfi } from '@/lib/client/epub'; import { useParams } from 'next/navigation'; import { useConfig } from './ConfigContext'; -import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client/api/audiobooks'; +import { withRetry, getAudiobookStatus, createAudiobookChapter } from '@/lib/client/api/audiobooks'; import { CmpStr } from 'cmpstr'; import type { TTSSentenceAlignment, @@ -34,7 +34,6 @@ import type { } from '@/types/tts'; import type { TTSRequestHeaders, - TTSRequestPayload, TTSRetryOptions, AudiobookGenerationSettings, } from '@/types/client'; @@ -179,11 +178,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const { apiKey, baseUrl, - voiceSpeed, - voice, ttsProvider, - ttsModel, - ttsInstructions, smartSentenceSplitting, epubHighlightEnabled, } = useConfig(); @@ -358,16 +353,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) { if (!sections.length) throw new Error('No text content found in book'); const effectiveProvider = settings?.ttsProvider ?? ttsProvider; - const effectiveModel = settings?.ttsModel ?? ttsModel; - const effectiveVoice = - settings?.voice || - voice || - (effectiveProvider === 'openai' - ? 'alloy' - : effectiveProvider === 'deepinfra' - ? 'af_bella' - : 'af_sarah'); - const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed; const effectiveFormat = settings?.format ?? format; // Calculate total length for accurate progress tracking @@ -448,16 +433,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { 'x-tts-provider': effectiveProvider, }; - const reqBody: TTSRequestPayload = { - text: trimmedText, - voice: effectiveVoice, - speed: effectiveNativeSpeed, - format: 'mp3', - model: effectiveModel, - instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined - }; - - // Allow one narrow client retry for transient browser->/api/tts transport failures. + // Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures. // HTTP failures are not retried client-side. const retryOptions: TTSRetryOptions = { maxRetries: 2, @@ -465,18 +441,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) { maxDelay: 300, }; - const audioBuffer = await withRetry( - async () => { - // Check for abort before starting TTS request - if (signal?.aborted) { - throw new DOMException('Aborted', 'AbortError'); - } - - return await generateTTS(reqBody, reqHeaders, signal); - }, - retryOptions - ); - // Get the chapter title from our pre-computed map let chapterTitle = sectionTitleMap.get(section.href); @@ -485,6 +449,25 @@ export function EPUBProvider({ children }: { children: ReactNode }) { chapterTitle = `Chapter ${i + 1}`; } + const chapter = await withRetry( + async () => { + // Check for abort before starting chapter request + if (signal?.aborted) { + throw new DOMException('Aborted', 'AbortError'); + } + + return await createAudiobookChapter({ + chapterTitle, + text: trimmedText, + bookId, + format: effectiveFormat, + chapterIndex: i, + settings + }, reqHeaders, signal); + }, + retryOptions + ); + // Check for abort before sending to server if (signal?.aborted) { console.log('Generation cancelled before saving chapter'); @@ -494,16 +477,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) { throw new Error('Audiobook generation cancelled'); } - // Send to server for conversion and storage - const chapter = await createAudiobookChapter({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format: effectiveFormat, - chapterIndex: i, - settings - }, signal); - if (!bookId) { bookId = chapter.bookId!; } @@ -548,7 +521,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { console.error('Error creating audiobook:', error); throw error; } - }, [extractBookText, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions]); + }, [extractBookText, apiKey, baseUrl, ttsProvider]); /** * Regenerates a specific chapter of the audiobook @@ -567,16 +540,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } const effectiveProvider = settings?.ttsProvider ?? ttsProvider; - const effectiveModel = settings?.ttsModel ?? ttsModel; - const effectiveVoice = - settings?.voice || - voice || - (effectiveProvider === 'openai' - ? 'alloy' - : effectiveProvider === 'deepinfra' - ? 'af_bella' - : 'af_sarah'); - const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed; const effectiveFormat = settings?.format ?? format; const section = sections[chapterIndex]; @@ -615,16 +578,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { 'x-tts-provider': effectiveProvider, }; - const reqBody: TTSRequestPayload = { - text: trimmedText, - voice: effectiveVoice, - speed: effectiveNativeSpeed, - format: 'mp3', - model: effectiveModel, - instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined - }; - - // Allow one narrow client retry for transient browser->/api/tts transport failures. + // Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures. // HTTP failures are not retried client-side. const retryOptions: TTSRetryOptions = { maxRetries: 2, @@ -632,13 +586,20 @@ export function EPUBProvider({ children }: { children: ReactNode }) { maxDelay: 300, }; - const audioBuffer = await withRetry( + const chapter = await withRetry( async () => { if (signal?.aborted) { throw new DOMException('Aborted', 'AbortError'); } - return await generateTTS(reqBody, reqHeaders, signal); + return await createAudiobookChapter({ + chapterTitle, + text: trimmedText, + bookId, + format: effectiveFormat, + chapterIndex, + settings + }, reqHeaders, signal); }, retryOptions ); @@ -647,16 +608,6 @@ export function EPUBProvider({ children }: { children: ReactNode }) { throw new Error('Chapter regeneration cancelled'); } - // Send to server for conversion and storage - const chapter = await createAudiobookChapter({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format: effectiveFormat, - chapterIndex, - settings - }, signal); - return chapter; } catch (error) { @@ -666,7 +617,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { console.error('Error regenerating chapter:', error); throw error; } - }, [extractBookText, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions]); + }, [extractBookText, apiKey, baseUrl, ttsProvider]); const setRendition = useCallback((rendition: Rendition) => { bookRef.current = rendition.book; diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index a9241c6..50039c5 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -32,7 +32,7 @@ import { ensureCachedDocument } from '@/lib/client/cache/documents'; import { useTTS } from '@/contexts/TTSContext'; import { useConfig } from '@/contexts/ConfigContext'; import { normalizeTextForTts } from '@/lib/shared/nlp'; -import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client/api/audiobooks'; +import { withRetry, getAudiobookStatus, createAudiobookChapter } from '@/lib/client/api/audiobooks'; import { extractTextFromPDF, highlightPattern, @@ -43,13 +43,11 @@ import { import type { TTSSentenceAlignment, - TTSAudioBuffer, TTSAudiobookFormat, TTSAudiobookChapter, } from '@/types/tts'; import type { TTSRequestHeaders, - TTSRequestPayload, TTSRetryOptions, AudiobookGenerationSettings, } from '@/types/client'; @@ -130,11 +128,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { rightMargin, apiKey, baseUrl, - voiceSpeed, - voice, ttsProvider, - ttsModel, - ttsInstructions, smartSentenceSplitting, } = useConfig(); @@ -420,16 +414,6 @@ export function PDFProvider({ children }: { children: ReactNode }) { } const effectiveProvider = settings?.ttsProvider ?? ttsProvider; - const effectiveModel = settings?.ttsModel ?? ttsModel; - const effectiveVoice = - settings?.voice || - voice || - (effectiveProvider === 'openai' - ? 'alloy' - : effectiveProvider === 'deepinfra' - ? 'af_bella' - : 'af_sarah'); - const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed; const effectiveFormat = settings?.format ?? format; // First pass: extract and measure all text @@ -504,16 +488,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { 'x-tts-provider': effectiveProvider, }; - const reqBody: TTSRequestPayload = { - text, - voice: effectiveVoice, - speed: effectiveNativeSpeed, - format: 'mp3', - model: effectiveModel, - instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined - }; - - // Allow one narrow client retry for transient browser->/api/tts transport failures. + // Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures. // HTTP failures are not retried client-side. const retryOptions: TTSRetryOptions = { maxRetries: 2, @@ -522,20 +497,27 @@ export function PDFProvider({ children }: { children: ReactNode }) { }; try { - const audioBuffer = await withRetry( + const chapterTitle = `Page ${i + 1}`; + + const chapter = await withRetry( async () => { - // Check for abort before starting TTS request + // Check for abort before starting chapter request if (signal?.aborted) { throw new DOMException('Aborted', 'AbortError'); } - return await generateTTS(reqBody, reqHeaders, signal); + return await createAudiobookChapter({ + chapterTitle, + text, + bookId, + format: effectiveFormat, + chapterIndex: i, + settings + }, reqHeaders, signal); }, retryOptions ); - const chapterTitle = `Page ${i + 1}`; - // Check for abort before sending to server if (signal?.aborted) { console.log('Generation cancelled before saving page'); @@ -545,16 +527,6 @@ export function PDFProvider({ children }: { children: ReactNode }) { throw new Error('Audiobook generation cancelled'); } - // Send to server for conversion and storage - const chapter = await createAudiobookChapter({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format: effectiveFormat, - chapterIndex: i, - settings - }, signal); - if (!bookId) { bookId = chapter.bookId!; } @@ -599,7 +571,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { console.error('Error creating audiobook:', error); throw error; } - }, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions, smartSentenceSplitting]); + }, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, ttsProvider, smartSentenceSplitting]); /** * Regenerates a specific chapter (page) of the PDF audiobook @@ -617,16 +589,6 @@ export function PDFProvider({ children }: { children: ReactNode }) { } const effectiveProvider = settings?.ttsProvider ?? ttsProvider; - const effectiveModel = settings?.ttsModel ?? ttsModel; - const effectiveVoice = - settings?.voice || - voice || - (effectiveProvider === 'openai' - ? 'alloy' - : effectiveProvider === 'deepinfra' - ? 'af_bella' - : 'af_sarah'); - const effectiveNativeSpeed = settings?.nativeSpeed ?? voiceSpeed; const effectiveFormat = settings?.format ?? format; // IMPORTANT: Chapter indices are based on non-empty pages used during generation. @@ -678,16 +640,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { 'x-tts-provider': effectiveProvider, }; - const reqBody: TTSRequestPayload = { - text: textForTTS, - voice: effectiveVoice, - speed: effectiveNativeSpeed, - format: 'mp3', - model: effectiveModel, - instructions: effectiveModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined - }; - - // Allow one narrow client retry for transient browser->/api/tts transport failures. + // Allow one narrow client retry for transient browser->/api/audiobook/chapter transport failures. // HTTP failures are not retried client-side. const retryOptions: TTSRetryOptions = { maxRetries: 2, @@ -695,13 +648,20 @@ export function PDFProvider({ children }: { children: ReactNode }) { maxDelay: 300, }; - const audioBuffer: TTSAudioBuffer = await withRetry( + const chapter = await withRetry( async () => { if (signal?.aborted) { throw new DOMException('Aborted', 'AbortError'); } - return await generateTTS(reqBody, reqHeaders, signal); + return await createAudiobookChapter({ + chapterTitle, + text: textForTTS, + bookId, + format: effectiveFormat, + chapterIndex, + settings + }, reqHeaders, signal); }, retryOptions ); @@ -710,16 +670,6 @@ export function PDFProvider({ children }: { children: ReactNode }) { throw new Error('Page regeneration cancelled'); } - // Send to server for conversion and storage - const chapter = await createAudiobookChapter({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format: effectiveFormat, - chapterIndex, - settings - }, signal); - return chapter; } catch (error) { @@ -729,7 +679,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { console.error('Error regenerating page:', error); throw error; } - }, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, voice, voiceSpeed, ttsProvider, ttsModel, ttsInstructions, smartSentenceSplitting]); + }, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, ttsProvider, smartSentenceSplitting]); /** * Effect hook to initialize TTS as non-EPUB mode diff --git a/src/lib/client/api/audiobooks.ts b/src/lib/client/api/audiobooks.ts index 3a5c164..9705987 100644 --- a/src/lib/client/api/audiobooks.ts +++ b/src/lib/client/api/audiobooks.ts @@ -115,12 +115,14 @@ export const getAudiobookStatus = async (bookId: string): Promise => { const response = await fetch(`/api/audiobook/chapter`, { method: 'POST', headers: { 'Content-Type': 'application/json', + ...headers, }, body: JSON.stringify(payload), signal diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts new file mode 100644 index 0000000..611d7a7 --- /dev/null +++ b/src/lib/server/tts/generate.ts @@ -0,0 +1,296 @@ +import OpenAI from 'openai'; +import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs'; +import { isKokoroModel } from '@/lib/shared/kokoro'; +import { LRUCache } from 'lru-cache'; +import { createHash } from 'crypto'; +import { access, readFile } from 'fs/promises'; +import { resolve } from 'path'; + +export interface ServerTTSRequest { + text: string; + voice: string; + speed: number; + format?: string; + model?: string | null; + instructions?: string; + provider: string; + apiKey: string; + baseUrl?: string; + testNamespace?: string | null; +} + +type CustomVoice = string; +type ExtendedSpeechParams = Omit & { + voice: SpeechCreateParams['voice'] | CustomVoice; + instructions?: string; +}; + +type ResolvedServerTTSRequest = { + text: string; + voice: string; + speed: number; + format: string; + model: SpeechCreateParams['model']; + instructions?: string; + provider: string; + apiKey: string; + baseUrl?: string; + testNamespace?: string | null; +}; + +type InflightEntry = { + promise: Promise; + controller: AbortController; + consumers: number; +}; + +const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB +const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes + +const ttsAudioCache = new LRUCache({ + maxSize: TTS_CACHE_MAX_SIZE_BYTES, + sizeCalculation: (value) => value.byteLength, + ttl: TTS_CACHE_TTL_MS, +}); + +const inflightRequests = new Map(); + +const TEST_TTS_MOCK_PATH = resolve(process.cwd(), 'tests/files/sample.mp3'); +let testMockTtsBufferPromise: Promise | null = null; + +function sleep(ms: number) { + return new Promise((res) => setTimeout(res, ms)); +} + +function getUpstreamStatus(error: unknown): number | undefined { + if (typeof error !== 'object' || error === null) return undefined; + const rec = error as Record; + if (typeof rec.status === 'number') return rec.status; + if (typeof rec.statusCode === 'number') return rec.statusCode; + return undefined; +} + +function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest { + const provider = input.provider || 'openai'; + const rawModel = provider === 'deepinfra' && !input.model ? 'hexgrad/Kokoro-82M' : input.model; + const model = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model']; + + const normalizedVoice = ( + !isKokoroModel(model as string) && input.voice.includes('+') + ? input.voice.split('+')[0].trim() + : input.voice + ) as string; + + const format = input.format || 'mp3'; + const speed = Number.isFinite(Number(input.speed)) ? Number(input.speed) : 1; + const instructions = (model as string) === 'gpt-4o-mini-tts' && input.instructions + ? input.instructions + : undefined; + + return { + text: input.text, + voice: normalizedVoice, + speed, + format, + model, + instructions, + provider, + apiKey: input.apiKey, + baseUrl: input.baseUrl, + testNamespace: input.testNamespace || null, + }; +} + +function makeCacheKey(input: { + provider: string; + model: string | null | undefined; + voice: string | undefined; + speed: number; + format: string; + text: string; + instructions?: string; + testNamespace?: string | null; +}) { + const canonical = { + provider: input.provider, + model: input.model || '', + voice: input.voice || '', + speed: input.speed, + format: input.format, + text: input.text, + instructions: input.instructions || undefined, + testNamespace: input.testNamespace || undefined, + }; + return createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); +} + +export function buildTTSCacheKey(request: ServerTTSRequest): string { + const resolved = resolveTTSRequest(request); + return makeCacheKey({ + provider: resolved.provider, + model: resolved.model, + voice: resolved.voice, + speed: resolved.speed, + format: resolved.format, + text: resolved.text, + instructions: resolved.instructions, + testNamespace: resolved.testNamespace, + }); +} + +export function getCachedTTSBuffer(cacheKey: string): Buffer | undefined { + return ttsAudioCache.get(cacheKey); +} + +export function getTTSContentType(format: string | undefined): string { + return (format || 'mp3') === 'mp3' ? 'audio/mpeg' : 'application/octet-stream'; +} + +async function getTestMockTtsBuffer(testNamespace?: string | null): Promise { + if (!testNamespace) return null; + if (!testMockTtsBufferPromise) { + testMockTtsBufferPromise = (async () => { + try { + await access(TEST_TTS_MOCK_PATH); + } catch { + return null; + } + return readFile(TEST_TTS_MOCK_PATH); + })(); + } + return testMockTtsBufferPromise; +} + +async function fetchTTSBufferWithRetry( + openai: OpenAI, + createParams: ExtendedSpeechParams, + signal: AbortSignal +): Promise { + let attempt = 0; + const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2); + let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250); + const maxDelay = Number(process.env.TTS_RETRY_MAX_MS ?? 2000); + const backoff = Number(process.env.TTS_RETRY_BACKOFF ?? 2); + + for (; ;) { + try { + const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal }); + const buffer = await response.arrayBuffer(); + return Buffer.from(buffer); + } catch (error) { + if (signal.aborted || (error instanceof Error && error.name === 'AbortError')) { + throw error; + } + + const status = getUpstreamStatus(error) ?? 0; + const retryable = status === 429 || status >= 500; + if (!retryable || attempt >= maxRetries) { + throw error; + } + + await sleep(Math.min(delay, maxDelay)); + delay = Math.min(maxDelay, delay * backoff); + attempt += 1; + } + } +} + +async function runProviderRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise { + const mockBuffer = await getTestMockTtsBuffer(request.testNamespace); + if (mockBuffer) return mockBuffer; + + const openai = new OpenAI({ + apiKey: request.apiKey, + baseURL: request.baseUrl, + maxRetries: 0, + timeout: Number(process.env.TTS_UPSTREAM_TIMEOUT_MS ?? 45_000), + }); + + const createParams: ExtendedSpeechParams = { + model: request.model, + voice: request.voice as SpeechCreateParams['voice'], + input: request.text, + speed: request.speed, + response_format: request.format as SpeechCreateParams['response_format'], + }; + + if (request.instructions) { + createParams.instructions = request.instructions; + } + + return fetchTTSBufferWithRetry(openai, createParams, signal); +} + +export async function generateTTSBuffer( + request: ServerTTSRequest, + signal?: AbortSignal +): Promise { + const resolved = resolveTTSRequest(request); + const cacheKey = makeCacheKey({ + provider: resolved.provider, + model: resolved.model, + voice: resolved.voice, + speed: resolved.speed, + format: resolved.format, + text: resolved.text, + instructions: resolved.instructions, + testNamespace: resolved.testNamespace, + }); + + const cachedBuffer = ttsAudioCache.get(cacheKey); + if (cachedBuffer) return cachedBuffer; + + const existing = inflightRequests.get(cacheKey); + if (existing) { + existing.consumers += 1; + + const onAbort = () => { + existing.consumers = Math.max(0, existing.consumers - 1); + if (existing.consumers === 0) { + existing.controller.abort(); + } + }; + + signal?.addEventListener('abort', onAbort, { once: true }); + try { + return await existing.promise; + } finally { + try { + signal?.removeEventListener('abort', onAbort); + } catch { } + } + } + + const controller = new AbortController(); + const entry: InflightEntry = { + controller, + consumers: 1, + promise: (async () => { + try { + const buffer = await runProviderRequest(resolved, controller.signal); + ttsAudioCache.set(cacheKey, buffer); + return buffer; + } finally { + inflightRequests.delete(cacheKey); + } + })(), + }; + + inflightRequests.set(cacheKey, entry); + + const onAbort = () => { + entry.consumers = Math.max(0, entry.consumers - 1); + if (entry.consumers === 0) { + entry.controller.abort(); + } + }; + + signal?.addEventListener('abort', onAbort, { once: true }); + try { + return await entry.promise; + } finally { + try { + signal?.removeEventListener('abort', onAbort); + } catch { } + } +} diff --git a/src/types/client.ts b/src/types/client.ts index 7c741ec..1f3eab0 100644 --- a/src/types/client.ts +++ b/src/types/client.ts @@ -56,11 +56,12 @@ export interface AudiobookGenerationSettings { nativeSpeed: number; postSpeed: number; format: TTSAudiobookFormat; + ttsInstructions?: string; } export interface CreateChapterPayload { chapterTitle: string; - buffer: TTSAudioBytes; // Array.from(new Uint8Array(audioBuffer)) + text: string; bookId: string; format: TTSAudiobookFormat; chapterIndex: number;