From 9999cafec138aeb296585295532ed8a2e04b01f4 Mon Sep 17 00:00:00 2001 From: Richard R Date: Mon, 4 May 2026 10:18:56 -0600 Subject: [PATCH] feat(tts): add TTS segments API, schema, storage, and manifest endpoints Introduce TTS segment support with new database tables and migrations for both Postgres and SQLite. Implement segment ensure, audio, and manifest API endpoints, segment storage in S3, and segment authorization logic. Update TTS context and client API to use segment-based synthesis and retrieval. Add server-side helpers for segment ID, settings hash, locator normalization, and audio probing. Extend user data cleanup and teardown logic to remove TTS segment objects. Add unit tests for segment helpers. BREAKING CHANGE: TTS synthesis now uses segment-based storage and APIs; database migrations required. --- drizzle/postgres/0001_tts_segments.sql | 27 ++ drizzle/postgres/meta/_journal.json | 7 + drizzle/sqlite/0001_tts_segments.sql | 26 ++ drizzle/sqlite/meta/_journal.json | 7 + src/app/api/account/delete/route.ts | 13 + src/app/api/tts/segments/audio/route.ts | 70 ++++ src/app/api/tts/segments/ensure/route.ts | 435 +++++++++++++++++++++ src/app/api/tts/segments/manifest/route.ts | 81 ++++ src/app/api/whisper/route.ts | 422 +------------------- src/contexts/TTSContext.tsx | 283 +++++++------- src/db/schema.ts | 1 + src/db/schema_postgres.ts | 25 ++ src/db/schema_sqlite.ts | 25 ++ src/lib/client/api/audiobooks.ts | 59 ++- src/lib/server/tts/segments-auth.ts | 64 +++ src/lib/server/tts/segments-blobstore.ts | 104 +++++ src/lib/server/tts/segments.ts | 188 +++++++++ src/lib/server/user/data-cleanup.ts | 57 ++- src/lib/server/whisper/alignment.ts | 395 +++++++++++++++++++ src/types/client.ts | 43 ++ tests/global-teardown.ts | 3 + tests/unit/tts-segments.spec.ts | 84 ++++ 22 files changed, 1847 insertions(+), 572 deletions(-) create mode 100644 drizzle/postgres/0001_tts_segments.sql create mode 100644 drizzle/sqlite/0001_tts_segments.sql create mode 100644 src/app/api/tts/segments/audio/route.ts create mode 100644 src/app/api/tts/segments/ensure/route.ts create mode 100644 src/app/api/tts/segments/manifest/route.ts create mode 100644 src/lib/server/tts/segments-auth.ts create mode 100644 src/lib/server/tts/segments-blobstore.ts create mode 100644 src/lib/server/tts/segments.ts create mode 100644 src/lib/server/whisper/alignment.ts create mode 100644 tests/unit/tts-segments.spec.ts diff --git a/drizzle/postgres/0001_tts_segments.sql b/drizzle/postgres/0001_tts_segments.sql new file mode 100644 index 0000000..7ef400f --- /dev/null +++ b/drizzle/postgres/0001_tts_segments.sql @@ -0,0 +1,27 @@ +CREATE TABLE "tts_segments" ( + "segment_id" text NOT NULL, + "user_id" text NOT NULL, + "document_id" text NOT NULL, + "reader_type" text NOT NULL, + "document_version" bigint NOT NULL, + "segment_index" integer NOT NULL, + "locator_json" text, + "settings_hash" text NOT NULL, + "text_hash" text NOT NULL, + "text_length" integer DEFAULT 0 NOT NULL, + "audio_key" text, + "audio_format" text DEFAULT 'mp3' NOT NULL, + "duration_ms" integer, + "alignment_json" text, + "status" text DEFAULT 'pending' NOT NULL, + "error" text, + "created_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint, + "updated_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint, + CONSTRAINT "tts_segments_segment_id_user_id_pk" PRIMARY KEY("segment_id","user_id") +); +--> statement-breakpoint +ALTER TABLE "tts_segments" ADD CONSTRAINT "tts_segments_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +CREATE INDEX "idx_tts_segments_lookup" ON "tts_segments" USING btree ("user_id","document_id","document_version","settings_hash"); +--> statement-breakpoint +CREATE INDEX "idx_tts_segments_doc_index" ON "tts_segments" USING btree ("user_id","document_id","segment_index"); diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index f20e818..1121af8 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1771386480954, "tag": "0000_black_lucky_pierre", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1777843301471, + "tag": "0001_tts_segments", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/sqlite/0001_tts_segments.sql b/drizzle/sqlite/0001_tts_segments.sql new file mode 100644 index 0000000..64f2ce0 --- /dev/null +++ b/drizzle/sqlite/0001_tts_segments.sql @@ -0,0 +1,26 @@ +CREATE TABLE `tts_segments` ( + `segment_id` text NOT NULL, + `user_id` text NOT NULL, + `document_id` text NOT NULL, + `reader_type` text NOT NULL, + `document_version` integer NOT NULL, + `segment_index` integer NOT NULL, + `locator_json` text, + `settings_hash` text NOT NULL, + `text_hash` text NOT NULL, + `text_length` integer DEFAULT 0 NOT NULL, + `audio_key` text, + `audio_format` text DEFAULT 'mp3' NOT NULL, + `duration_ms` integer, + `alignment_json` text, + `status` text DEFAULT 'pending' NOT NULL, + `error` text, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)), + `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)), + PRIMARY KEY(`segment_id`, `user_id`), + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `idx_tts_segments_lookup` ON `tts_segments` (`user_id`,`document_id`,`document_version`,`settings_hash`); +--> statement-breakpoint +CREATE INDEX `idx_tts_segments_doc_index` ON `tts_segments` (`user_id`,`document_id`,`segment_index`); diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index 3536d28..6c3ee9d 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1771386480690, "tag": "0000_familiar_caretaker", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1777843301472, + "tag": "0001_tts_segments", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/api/account/delete/route.ts b/src/app/api/account/delete/route.ts index 76358f6..5e59e83 100644 --- a/src/app/api/account/delete/route.ts +++ b/src/app/api/account/delete/route.ts @@ -2,6 +2,8 @@ import { headers } from 'next/headers'; import { NextResponse } from 'next/server'; import { auth } from '@/lib/server/auth/auth'; import { isAuthEnabled } from '@/lib/server/auth/config'; +import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; +import { deleteUserStorageData } from '@/lib/server/user/data-cleanup'; export async function DELETE() { if (!isAuthEnabled() || !auth) { @@ -19,6 +21,17 @@ export async function DELETE() { } try { + // Best-effort cleanup for test namespaced storage using request context. + // The Better Auth beforeDelete hook still runs and handles non-namespaced data. + const testNamespace = getOpenReaderTestNamespace(reqHeaders); + if (testNamespace) { + try { + await deleteUserStorageData(session.user.id, testNamespace); + } catch (error) { + console.error('[account-delete] Failed to clean up namespaced user storage before deletion:', error); + } + } + // Use Better Auth's built-in deleteUser to handle cascading cleanup await auth.api.deleteUser({ headers: reqHeaders, diff --git a/src/app/api/tts/segments/audio/route.ts b/src/app/api/tts/segments/audio/route.ts new file mode 100644 index 0000000..e2e0a93 --- /dev/null +++ b/src/app/api/tts/segments/audio/route.ts @@ -0,0 +1,70 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq } from 'drizzle-orm'; +import { db } from '@/db'; +import { ttsSegments } from '@/db/schema'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import { getTtsSegmentAudioObject } from '@/lib/server/tts/segments-blobstore'; +import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'TTS segments storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function streamBuffer(buffer: Buffer): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(buffer)); + controller.close(); + }, + }); +} + +export async function GET(request: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const documentId = (request.nextUrl.searchParams.get('documentId') || '').trim().toLowerCase(); + const segmentId = (request.nextUrl.searchParams.get('segmentId') || '').trim().toLowerCase(); + if (!documentId || !segmentId) { + return NextResponse.json({ error: 'Missing documentId or segmentId' }, { status: 400 }); + } + + const scope = await resolveSegmentDocumentScope(request, documentId); + if (scope instanceof Response) return scope; + + const rows = (await db + .select({ + audioKey: ttsSegments.audioKey, + status: ttsSegments.status, + }) + .from(ttsSegments) + .where(and( + eq(ttsSegments.userId, scope.storageUserId), + eq(ttsSegments.documentId, documentId), + eq(ttsSegments.documentVersion, scope.documentVersion), + eq(ttsSegments.segmentId, segmentId), + ))) as Array<{ audioKey: string | null; status: string }>; + + const row = rows[0]; + if (!row || !row.audioKey || row.status !== 'completed') { + return NextResponse.json({ error: 'Segment audio not found' }, { status: 404 }); + } + + const audio = await getTtsSegmentAudioObject(row.audioKey); + + return new NextResponse(streamBuffer(audio), { + headers: { + 'Content-Type': 'audio/mpeg', + 'Cache-Control': 'private, max-age=86400', + }, + }); + } catch (error) { + console.error('Error serving TTS segment audio:', error); + return NextResponse.json({ error: 'Failed to load segment audio' }, { status: 500 }); + } +} diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts new file mode 100644 index 0000000..a64cd9c --- /dev/null +++ b/src/app/api/tts/segments/ensure/route.ts @@ -0,0 +1,435 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { ttsSegments } from '@/db/schema'; +import { isS3Configured, getS3Config } from '@/lib/server/storage/s3'; +import { generateTTSBuffer } from '@/lib/server/tts/generate'; +import { getTtsSegmentAudioObject, putTtsSegmentAudioObject } from '@/lib/server/tts/segments-blobstore'; +import { + buildTtsSegmentAudioKey, + buildTtsSegmentId, + buildTtsSegmentSettingsHash, + buildTtsSegmentTextHash, + locatorFingerprint, + normalizeLocator, + normalizeSegmentText, + probeAudioDurationMsFromBuffer, +} from '@/lib/server/tts/segments'; +import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; +import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { getClientIp } from '@/lib/server/rate-limit/request-ip'; +import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; +import { alignAudioWithText } from '@/lib/server/whisper/alignment'; +import type { + TTSSegmentInput, + TTSSegmentManifestItem, + TTSSegmentSettings, + TTSSegmentsEnsureRequest, +} from '@/types/client'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +function attachDeviceIdCookie(response: NextResponse, deviceId: string | null, didCreate: boolean) { + if (didCreate && deviceId) { + setDeviceIdCookie(response, deviceId); + } +} + +function formatLimitForHint(limit: number): string { + if (!Number.isFinite(limit) || limit <= 0) return String(limit); + if (limit >= 1_000_000) { + const m = limit / 1_000_000; + return `${m % 1 === 0 ? m.toFixed(0) : m.toFixed(1)}M`; + } + if (limit >= 1_000) return `${Math.round(limit / 1_000)}K`; + return String(limit); +} + +function parseSettings(value: unknown): TTSSegmentSettings | null { + if (!value || typeof value !== 'object') return null; + const rec = value as Record; + if (typeof rec.ttsProvider !== 'string') return null; + if (typeof rec.ttsModel !== 'string') return null; + if (typeof rec.voice !== 'string') return null; + if (!Number.isFinite(Number(rec.nativeSpeed))) return null; + if (rec.ttsInstructions !== undefined && typeof rec.ttsInstructions !== 'string') return null; + + return { + ttsProvider: rec.ttsProvider, + ttsModel: rec.ttsModel, + voice: rec.voice, + nativeSpeed: Number(rec.nativeSpeed), + ...(typeof rec.ttsInstructions === 'string' ? { ttsInstructions: rec.ttsInstructions } : {}), + }; +} + +function parseSegments(value: unknown): TTSSegmentInput[] | null { + if (!Array.isArray(value) || value.length === 0) return null; + const parsed: TTSSegmentInput[] = []; + for (const item of value) { + if (!item || typeof item !== 'object') return null; + const rec = item as Record; + if (!Number.isInteger(rec.segmentIndex) || Number(rec.segmentIndex) < 0) return null; + if (typeof rec.text !== 'string') return null; + parsed.push({ + segmentIndex: Number(rec.segmentIndex), + text: rec.text, + ...(rec.locator && typeof rec.locator === 'object' ? { locator: rec.locator as TTSSegmentInput['locator'] } : {}), + }); + } + return parsed; +} + +function parseBody(value: unknown): TTSSegmentsEnsureRequest | null { + if (!value || typeof value !== 'object') return null; + const rec = value as Record; + if (typeof rec.documentId !== 'string' || !rec.documentId.trim()) return null; + const settings = parseSettings(rec.settings); + const segments = parseSegments(rec.segments); + if (!settings || !segments) return null; + + return { + documentId: rec.documentId.trim().toLowerCase(), + settings, + segments, + }; +} + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'TTS segments storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function textHmacSecret(): string { + return process.env.AUTH_SECRET?.trim() + || 'openreader-default-tts-segment-secret'; +} + +export async function POST(request: NextRequest) { + let didCreateDeviceIdCookie = false; + let deviceIdToSet: string | null = null; + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const parsed = parseBody(await request.json().catch(() => null)); + if (!parsed) { + return NextResponse.json({ error: 'Invalid request payload' }, { status: 400 }); + } + + const scope = await resolveSegmentDocumentScope(request, parsed.documentId); + if (scope instanceof Response) return scope; + + const settingsHash = buildTtsSegmentSettingsHash(parsed.settings); + const nowMs = Date.now(); + const storagePrefix = getS3Config().prefix; + const secret = textHmacSecret(); + + const normalized = parsed.segments + .map((segment) => { + const text = normalizeSegmentText(segment.text); + if (!text) return null; + const locator = normalizeLocator(segment.locator); + const locatorHash = locatorFingerprint(locator); + const segmentId = buildTtsSegmentId({ + documentId: parsed.documentId, + documentVersion: scope.documentVersion, + settingsHash, + segmentIndex: segment.segmentIndex, + normalizedText: text, + locatorFingerprint: locatorHash, + }); + + return { + original: segment, + text, + locator, + segmentId, + textHash: buildTtsSegmentTextHash(text, secret), + }; + }) + .filter((value): value is NonNullable => Boolean(value)); + + if (normalized.length === 0) { + return NextResponse.json({ error: 'No valid non-empty segments provided' }, { status: 400 }); + } + + const ids = normalized.map((segment) => segment.segmentId); + const rows = (await db + .select() + .from(ttsSegments) + .where(and(eq(ttsSegments.userId, scope.storageUserId), inArray(ttsSegments.segmentId, ids)))) as Array<{ + segmentId: string; + userId: string; + documentId: string; + readerType: string; + documentVersion: number; + segmentIndex: number; + locatorJson: string | null; + settingsHash: string; + textHash: string; + textLength: number; + audioKey: string | null; + audioFormat: string; + durationMs: number | null; + alignmentJson: string | null; + status: string; + error: string | null; + createdAt: number | null; + updatedAt: number | null; + }>; + + const existingById = new Map(rows.map((row) => [row.segmentId, row])); + const manifest: TTSSegmentManifestItem[] = []; + + for (const segment of normalized) { + const existing = existingById.get(segment.segmentId); + + if (existing?.status === 'completed' && existing.audioKey) { + let alignment = existing.alignmentJson + ? (JSON.parse(existing.alignmentJson) as TTSSegmentManifestItem['alignment']) + : null; + const locator = existing.locatorJson + ? (JSON.parse(existing.locatorJson) as TTSSegmentManifestItem['locator']) + : null; + + // Self-heal transient Whisper failures: if audio exists but alignment was + // previously unavailable, retry alignment using the current segment text. + if (!alignment) { + try { + const audioBuffer = await getTtsSegmentAudioObject(existing.audioKey); + const whisperBytes = Uint8Array.from(audioBuffer); + const aligned = await alignAudioWithText( + whisperBytes.buffer, + segment.text, + undefined, + { engine: 'whisper.cpp' }, + ); + alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null; + + if (alignment) { + await db + .update(ttsSegments) + .set({ + alignmentJson: JSON.stringify(alignment), + updatedAt: Date.now(), + }) + .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); + } + } catch (alignError) { + console.warn('Whisper alignment still unavailable for completed segment; continuing without word highlights.', { + segmentId: segment.segmentId, + error: alignError instanceof Error ? alignError.message : String(alignError), + }); + alignment = null; + } + } + + manifest.push({ + segmentId: segment.segmentId, + segmentIndex: existing.segmentIndex, + audioUrl: `/api/tts/segments/audio?documentId=${encodeURIComponent(parsed.documentId)}&segmentId=${encodeURIComponent(segment.segmentId)}`, + durationMs: existing.durationMs ?? 0, + alignment, + locator, + status: 'completed', + }); + continue; + } + + const audioKey = existing?.audioKey || buildTtsSegmentAudioKey({ + storagePrefix, + namespace: scope.testNamespace, + userId: scope.storageUserId, + documentId: parsed.documentId, + documentVersion: scope.documentVersion, + settingsHash, + segmentId: segment.segmentId, + }); + + await db + .insert(ttsSegments) + .values({ + segmentId: segment.segmentId, + userId: scope.storageUserId, + documentId: parsed.documentId, + readerType: scope.readerType, + documentVersion: scope.documentVersion, + segmentIndex: segment.original.segmentIndex, + locatorJson: segment.locator ? JSON.stringify(segment.locator) : null, + settingsHash, + textHash: segment.textHash, + textLength: segment.text.length, + audioKey, + audioFormat: 'mp3', + status: 'pending', + error: null, + updatedAt: nowMs, + }) + .onConflictDoUpdate({ + target: [ttsSegments.segmentId, ttsSegments.userId], + set: { + documentId: parsed.documentId, + readerType: scope.readerType, + documentVersion: scope.documentVersion, + segmentIndex: segment.original.segmentIndex, + locatorJson: segment.locator ? JSON.stringify(segment.locator) : null, + settingsHash, + textHash: segment.textHash, + textLength: segment.text.length, + audioKey, + audioFormat: 'mp3', + status: 'pending', + error: null, + updatedAt: nowMs, + }, + }); + + try { + if (scope.authEnabled && scope.userId && isTtsRateLimitEnabled()) { + const charCount = segment.text.length; + const ip = getClientIp(request); + const device = scope.isAnonymousUser ? getOrCreateDeviceId(request) : null; + if (device?.didCreate) { + didCreateDeviceIdCookie = true; + deviceIdToSet = device.deviceId; + } + + const rateLimitResult = await rateLimiter.checkAndIncrementLimit( + { id: scope.userId, isAnonymous: scope.isAnonymousUser }, + charCount, + { + deviceId: device?.deviceId ?? null, + ip, + }, + ); + + if (!rateLimitResult.allowed) { + const resetTimeMs = rateLimitResult.resetTimeMs; + const retryAfterSeconds = Math.max(0, Math.ceil((resetTimeMs - Date.now()) / 1000)); + const response = new NextResponse(JSON.stringify({ + type: 'https://openreader.app/problems/daily-quota-exceeded', + title: 'Daily quota exceeded', + status: 429, + detail: 'Daily character limit exceeded', + code: 'USER_DAILY_QUOTA_EXCEEDED', + currentCount: rateLimitResult.currentCount, + limit: rateLimitResult.limit, + remainingChars: rateLimitResult.remainingChars, + resetTimeMs, + userType: scope.isAnonymousUser ? 'anonymous' : 'authenticated', + upgradeHint: scope.isAnonymousUser + ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + : undefined, + instance: request.nextUrl.pathname, + }), { + status: 429, + headers: { + 'Content-Type': 'application/problem+json', + 'Retry-After': String(retryAfterSeconds), + }, + }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } + } + + const ttsBuffer = await generateTTSBuffer({ + text: segment.text, + voice: parsed.settings.voice, + speed: parsed.settings.nativeSpeed, + format: 'mp3', + model: parsed.settings.ttsModel, + instructions: parsed.settings.ttsInstructions, + provider: parsed.settings.ttsProvider, + apiKey: request.headers.get('x-openai-key') || process.env.API_KEY || 'none', + baseUrl: request.headers.get('x-openai-base-url') || process.env.API_BASE, + testNamespace: scope.testNamespace, + }, request.signal); + + await putTtsSegmentAudioObject(audioKey, ttsBuffer); + + let persistedBuffer = ttsBuffer; + if (persistedBuffer.byteLength === 0) { + persistedBuffer = await getTtsSegmentAudioObject(audioKey); + } + + const durationMs = await probeAudioDurationMsFromBuffer(persistedBuffer, request.signal); + let alignment: TTSSegmentManifestItem['alignment'] = null; + try { + const whisperBytes = Uint8Array.from(persistedBuffer); + const aligned = await alignAudioWithText( + whisperBytes.buffer, + segment.text, + undefined, + { engine: 'whisper.cpp' }, + ); + alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null; + } catch (alignError) { + console.warn('Whisper alignment unavailable for segment; continuing without word highlights.', { + segmentId: segment.segmentId, + error: alignError instanceof Error ? alignError.message : String(alignError), + }); + alignment = null; + } + + await db + .update(ttsSegments) + .set({ + durationMs, + alignmentJson: alignment ? JSON.stringify(alignment) : null, + status: 'completed', + error: null, + updatedAt: Date.now(), + }) + .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); + + manifest.push({ + segmentId: segment.segmentId, + segmentIndex: segment.original.segmentIndex, + audioUrl: `/api/tts/segments/audio?documentId=${encodeURIComponent(parsed.documentId)}&segmentId=${encodeURIComponent(segment.segmentId)}`, + durationMs, + alignment, + locator: segment.locator, + status: 'completed', + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to generate segment'; + await db + .update(ttsSegments) + .set({ + status: 'error', + error: message, + updatedAt: Date.now(), + }) + .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); + + manifest.push({ + segmentId: segment.segmentId, + segmentIndex: segment.original.segmentIndex, + audioUrl: '', + durationMs: 0, + alignment: null, + locator: segment.locator, + status: 'error', + }); + } + } + + const response = NextResponse.json({ + documentId: parsed.documentId, + documentVersion: scope.documentVersion, + settingsHash, + segments: manifest, + }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } catch (error) { + console.error('Error ensuring TTS segments:', error); + const response = NextResponse.json({ error: 'Failed to ensure TTS segments' }, { status: 500 }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } +} diff --git a/src/app/api/tts/segments/manifest/route.ts b/src/app/api/tts/segments/manifest/route.ts new file mode 100644 index 0000000..ef01812 --- /dev/null +++ b/src/app/api/tts/segments/manifest/route.ts @@ -0,0 +1,81 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, asc, eq, gte, lte } from 'drizzle-orm'; +import { db } from '@/db'; +import { ttsSegments } from '@/db/schema'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; +import type { TTSSegmentManifestItem } from '@/types/client'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'TTS segments storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function parseIntOr(value: string | null, fallback: number): number { + const n = Number.parseInt(value || '', 10); + return Number.isFinite(n) ? n : fallback; +} + +export async function GET(request: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const documentId = (request.nextUrl.searchParams.get('documentId') || '').trim().toLowerCase(); + const settingsHash = (request.nextUrl.searchParams.get('settingsHash') || '').trim(); + if (!documentId || !settingsHash) { + return NextResponse.json({ error: 'Missing documentId or settingsHash' }, { status: 400 }); + } + + const scope = await resolveSegmentDocumentScope(request, documentId); + if (scope instanceof Response) return scope; + + const fromIndex = Math.max(0, parseIntOr(request.nextUrl.searchParams.get('fromIndex'), 0)); + const toIndex = Math.max(fromIndex, parseIntOr(request.nextUrl.searchParams.get('toIndex'), fromIndex + 8)); + + const rows = (await db + .select() + .from(ttsSegments) + .where(and( + eq(ttsSegments.userId, scope.storageUserId), + eq(ttsSegments.documentId, documentId), + eq(ttsSegments.documentVersion, scope.documentVersion), + eq(ttsSegments.settingsHash, settingsHash), + gte(ttsSegments.segmentIndex, fromIndex), + lte(ttsSegments.segmentIndex, toIndex), + )) + .orderBy(asc(ttsSegments.segmentIndex))) as Array<{ + segmentId: string; + segmentIndex: number; + locatorJson: string | null; + durationMs: number | null; + alignmentJson: string | null; + status: string; + }>; + + const segments: TTSSegmentManifestItem[] = rows.map((row) => ({ + segmentId: row.segmentId, + segmentIndex: row.segmentIndex, + audioUrl: row.status === 'completed' + ? `/api/tts/segments/audio?documentId=${encodeURIComponent(documentId)}&segmentId=${encodeURIComponent(row.segmentId)}` + : '', + durationMs: row.durationMs ?? 0, + alignment: row.alignmentJson ? JSON.parse(row.alignmentJson) : null, + locator: row.locatorJson ? JSON.parse(row.locatorJson) : null, + status: row.status === 'error' ? 'error' : row.status === 'completed' ? 'completed' : 'pending', + })); + + return NextResponse.json({ + documentId, + documentVersion: scope.documentVersion, + settingsHash, + segments, + }); + } catch (error) { + console.error('Error fetching TTS segment manifest:', error); + return NextResponse.json({ error: 'Failed to fetch TTS segment manifest' }, { status: 500 }); + } +} diff --git a/src/app/api/whisper/route.ts b/src/app/api/whisper/route.ts index d43f4b4..4c73ff3 100644 --- a/src/app/api/whisper/route.ts +++ b/src/app/api/whisper/route.ts @@ -1,424 +1,16 @@ import { NextRequest, NextResponse } from 'next/server'; -import { createHash, randomUUID } from 'crypto'; -import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; -import { tmpdir } from 'os'; -import { join } from 'path'; -import { spawn } from 'child_process'; -import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts'; -import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import type { TTSSentenceAlignment } from '@/types/tts'; import { auth } from '@/lib/server/auth/auth'; -import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin'; +import { + alignAudioWithText, + makeWhisperCacheKey, + type WhisperRequestBody, +} from '@/lib/server/whisper/alignment'; export const runtime = 'nodejs'; -interface WhisperRequestBody { - text: string; - audio: TTSAudioBytes; // raw bytes from Uint8Array - lang?: string; -} - -interface WhisperAlignmentOptions { - engine?: 'whisper.cpp'; - lang?: string; -} - -interface WhisperWord { - start: number; - end: number; - word: string; -} - -// Simple in-memory cache keyed by a hash of text+lang+audio length -const alignmentCache = new Map(); - -// Model management using a fixed tiny.en GGML model stored under docstore/model -const MODEL_NAME = 'ggml-tiny.en.bin'; -const MODEL_URL = - 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin'; -const DOCSTORE_DIR = join(process.cwd(), 'docstore'); -const MODEL_DIR = join(DOCSTORE_DIR, 'model'); -const MODEL_PATH = join(MODEL_DIR, MODEL_NAME); -const modelReadyPromises = new Map>(); - -async function ensureModelAvailable(): Promise { - // Fast path: model already present - try { - await access(MODEL_PATH); - return; - } catch { - // fall through to download - } - - const existing = modelReadyPromises.get(MODEL_PATH); - if (existing) return existing; - - const promise = (async () => { - try { - await access(MODEL_PATH); - return; - } catch { - // still missing - } - - await mkdir(MODEL_DIR, { recursive: true }); - - const res = await fetch(MODEL_URL); - if (!res.ok) { - throw new Error( - `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}` - ); - } - - const arrayBuffer = await res.arrayBuffer(); - await writeFile(MODEL_PATH, Buffer.from(arrayBuffer)); - })(); - - modelReadyPromises.set(MODEL_PATH, promise); - return promise; -} - -async function runWhisperCpp( - wavPath: string, - opts: WhisperAlignmentOptions -): Promise { - const binary = process.env.WHISPER_CPP_BIN; - if (!binary) { - throw new Error( - 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.' - ); - } - - await ensureModelAvailable(); - - return new Promise((resolve, reject) => { - const jsonBase = `${wavPath}.json_out`; - const jsonPath = `${jsonBase}.json`; - // Request full JSON output (including token-level details when available) - // and ask whisper-cli not to print anything to stdout. - const args = [ - '-m', - MODEL_PATH, - '-f', - wavPath, - '-of', - jsonBase, - '-ojf', - '-np', - ]; - - if (opts.lang) { - args.push('-l', opts.lang); - } - - const child = spawn(binary, args); - - let stdout = ''; - let stderr = ''; - - child.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - child.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - child.on('error', (err) => { - reject(err); - }); - - child.on('close', (code) => { - if (code !== 0) { - console.error('whisper-cli exited with error', { - code, - wavPath, - stdout: stdout.slice(0, 500), - stderr: stderr.slice(0, 500), - }); - return reject( - new Error( - `whisper.cpp exited with code ${code}: ${stderr || stdout}` - ) - ); - } - - readFile(jsonPath, 'utf-8') - .then((content: string) => { - const words: WhisperWord[] = []; - const parsed = JSON.parse(content) as { - transcription?: Array<{ - text?: string; - timestamps?: { from?: string; to?: string }; - offsets?: { from?: number; to?: number }; - tokens?: Array<{ - text?: string; - timestamps?: { from?: string; to?: string }; - offsets?: { from?: number; to?: number }; - }>; - }>; - }; - - const transcription = parsed.transcription; - - const parseTimecode = (value?: string): number | null => { - if (!value) return null; - const m = value.match(/(\d+):(\d+):(\d+),(\d+)/); - if (!m) return null; - const h = Number(m[1]); - const min = Number(m[2]); - const s = Number(m[3]); - const ms = Number(m[4]); - if ( - Number.isNaN(h) || - Number.isNaN(min) || - Number.isNaN(s) || - Number.isNaN(ms) - ) { - return null; - } - return h * 3600 + min * 60 + s + ms / 1000; - }; - - if (Array.isArray(transcription)) { - for (const seg of transcription) { - const segText = (seg.text || '').trim(); - const segStartSecFromTs = parseTimecode( - seg.timestamps?.from - ); - const segEndSecFromTs = parseTimecode(seg.timestamps?.to); - const segStartSecFromMs = - typeof seg.offsets?.from === 'number' - ? seg.offsets.from / 1000 - : null; - const segEndSecFromMs = - typeof seg.offsets?.to === 'number' - ? seg.offsets.to / 1000 - : null; - - const segStartSec = - segStartSecFromTs ?? - segStartSecFromMs ?? - 0; - const segEndSec = - segEndSecFromTs ?? - segEndSecFromMs ?? - segStartSec; - - const tokens = Array.isArray(seg.tokens) - ? seg.tokens - : []; - - if (tokens.length > 0) { - for (const token of tokens) { - const rawText = token.text || ''; - const tokenText = rawText.trim(); - // Skip special markers like [_BEG_] - if (!tokenText || /^\[.*\]$/.test(tokenText)) continue; - - const tokStartSecFromTs = parseTimecode( - token.timestamps?.from - ); - const tokEndSecFromTs = parseTimecode( - token.timestamps?.to - ); - const tokStartSecFromMs = - typeof token.offsets?.from === 'number' - ? token.offsets.from / 1000 - : null; - const tokEndSecFromMs = - typeof token.offsets?.to === 'number' - ? token.offsets.to / 1000 - : null; - - const startSec = - tokStartSecFromTs ?? - tokStartSecFromMs ?? - segStartSec; - const endSec = - tokEndSecFromTs ?? - tokEndSecFromMs ?? - segEndSec; - - words.push({ - word: tokenText, - start: startSec, - end: endSec, - }); - } - } else if (segText) { - // Fallback: no token list, approximate per-word timing within the segment - const segTokens = segText.split(/\s+/).filter(Boolean); - if (segTokens.length) { - const totalDur = Math.max(segEndSec - segStartSec, 0); - const step = - segTokens.length > 0 - ? totalDur / segTokens.length - : 0; - segTokens.forEach((token, index) => { - const wStart = - step > 0 - ? segStartSec + step * index - : segStartSec; - const wEnd = - step > 0 - ? index === segTokens.length - 1 - ? segEndSec - : segStartSec + step * (index + 1) - : segEndSec; - words.push({ - word: token, - start: wStart, - end: wEnd, - }); - }); - } - } - } - } - - resolve(words); - }) - .catch((err: unknown) => { - reject(err); - }); - }); - }); -} - -function mapWordsToSentenceOffsets( - sentence: string, - words: WhisperWord[] -): TTSSentenceAlignment { - const normalizedSentence = preprocessSentenceForAudio(sentence); - let cursor = 0; - - const alignedWords = words.map((w) => { - const token = w.word.trim(); - if (!token) { - return { - text: '', - startSec: w.start, - endSec: w.end, - charStart: cursor, - charEnd: cursor, - }; - } - - const idx = normalizedSentence - .toLowerCase() - .indexOf(token.toLowerCase(), cursor); - - const start = - idx !== -1 - ? idx - : cursor; - const end = start + token.length; - - cursor = end; - - return { - text: token, - startSec: w.start, - endSec: w.end, - charStart: start, - charEnd: end, - }; - }); - - return { - sentence, - sentenceIndex: 0, - words: alignedWords.filter((w) => w.text.length > 0), - }; -} - -async function alignAudioWithText( - audioBuffer: TTSAudioBuffer, - text: string, - cacheKey?: string, - opts: WhisperAlignmentOptions = {} -): Promise { - if (!text.trim()) { - return []; - } - - if (cacheKey && alignmentCache.has(cacheKey)) { - return alignmentCache.get(cacheKey)!; - } - - const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); - const inputPath = join(tmpBase, `${randomUUID()}-input.bin`); - const wavPath = join(tmpBase, `${randomUUID()}-input.wav`); - - try { - await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); - - await new Promise((resolve, reject) => { - const ffmpeg = spawn(getFFmpegPath(), [ - '-y', - '-i', - inputPath, - '-ar', - '16000', - '-ac', - '1', - wavPath, - ]); - - let stderr = ''; - ffmpeg.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - ffmpeg.on('error', (err) => { - reject(err); - }); - - ffmpeg.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - reject( - new Error(`ffmpeg failed with code ${code}: ${stderr}`) - ); - } - }); - }); - - const words = await runWhisperCpp(wavPath, opts); - const alignment = mapWordsToSentenceOffsets(text, words); - const result: TTSSentenceAlignment[] = [alignment]; - - if (cacheKey) { - alignmentCache.set(cacheKey, result); - } - - return result; - } finally { - try { - await rm(tmpBase, { recursive: true, force: true }); - } catch { - // ignore - } - } -} - -function makeCacheKey(input: WhisperRequestBody) { - const hash = createHash('sha256') - .update( - JSON.stringify({ - text: input.text, - lang: input.lang || '', - audioLen: input.audio?.length || 0, - }) - ) - .digest('hex'); - return hash; -} - export async function POST(req: NextRequest) { try { - // Auth check - require session const session = await auth?.api.getSession({ headers: req.headers }); if (auth && !session?.user) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); @@ -434,7 +26,7 @@ export async function POST(req: NextRequest) { ); } - const cacheKey = makeCacheKey(body); + const cacheKey = makeWhisperCacheKey(body); const audioBuffer = new Uint8Array(audio).buffer; const alignments: TTSSentenceAlignment[] = await alignAudioWithText( diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index e771de2..e5cb55c 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -36,7 +36,7 @@ import { useMediaSession } from '@/hooks/audio/useMediaSession'; import { useAudioContext } from '@/hooks/audio/useAudioContext'; import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/client/dexie'; import { getDocumentProgress, scheduleDocumentProgressSync } from '@/lib/client/api/user-state'; -import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks'; +import { withRetry, ensureTtsSegments } from '@/lib/client/api/audiobooks'; import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp'; import { isKokoroModel } from '@/lib/shared/kokoro'; import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog'; @@ -50,9 +50,9 @@ import type { TTSAudioBuffer, } from '@/types/tts'; import type { - TTSRequestPayload, TTSRequestHeaders, TTSRetryOptions, + TTSSegmentManifestItem, } from '@/types/client'; import type { ReaderType } from '@/types/user-state'; @@ -105,6 +105,7 @@ const MAX_CONTINUATION_CARRY_CHARS = 220; const MAX_CONTINUATION_CARRY_WORDS = 40; const LOOP_GUARD_MIN_INDEX = 2; const LOOP_GUARD_MIN_PROGRESS = 0.6; +const AUDIO_CACHE_MAX_ITEMS = 25; const SENTENCE_ENDING = /[.?!…]["'”’)\]]*\s*$/; const wordHighlightFeatureEnabled = process.env.NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT?.toLowerCase() !== 'false'; @@ -322,7 +323,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Audio and voice management hooks const audioContext = useAudioContext(); - const audioCache = useAudioCache(25); + const audioCache = useAudioCache(AUDIO_CACHE_MAX_ITEMS); const { availableVoices, fetchVoices } = useVoiceManagement(openApiKey, openApiBaseUrl, configTTSProvider, configTTSModel); const { authEnabled, @@ -360,6 +361,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Get document ID from URL params const { id } = useParams(); const pathname = usePathname(); + const documentId = useMemo(() => { + if (typeof id === 'string') return id; + if (Array.isArray(id)) return id[0]; + return ''; + }, [id]); const currentReaderType: ReaderType = useMemo(() => { if (pathname.startsWith('/epub/')) return 'epub'; @@ -406,11 +412,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const restartSeqRef = useRef(0); // Track continuation slices for PDF/EPUB page transitions const continuationCarryRef = useRef>(new Map()); + // Preserve autoplay intent across location changes. Some browsers can emit pause + // events while we stop/unload between pages, which momentarily flips `isPlaying` + // false and can prevent automatic resume on the next page. + const resumeAfterLocationChangeRef = useRef(false); const epubContinuationRef = useRef(null); const pageTurnEstimateRef = useRef(null); const pageTurnTimeoutRef = useRef | null>(null); const pageFirstBlockFingerprintRef = useRef>(new Map()); const sentenceAlignmentCacheRef = useRef>(new Map()); + const segmentManifestCacheRef = useRef>(new Map()); const [currentSentenceAlignment, setCurrentSentenceAlignment] = useState(); const [currentWordIndex, setCurrentWordIndex] = useState(null); const isPlayingRef = useRef(false); @@ -509,6 +520,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } }, [audioContext]); + const setSegmentManifestCache = useCallback((key: string, item: TTSSegmentManifestItem) => { + const cache = segmentManifestCacheRef.current; + if (cache.has(key)) { + cache.delete(key); + } + cache.set(key, item); + while (cache.size > AUDIO_CACHE_MAX_ITEMS) { + const oldestKey = cache.keys().next().value; + if (typeof oldestKey !== 'string') break; + cache.delete(oldestKey); + } + }, []); + const isAutoplayBlockedError = useCallback((err: unknown) => { const msg = (() => { if (typeof err === 'string') return err; @@ -620,6 +644,12 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {boolean} shouldPause - Whether to pause playback */ const skipToLocation = useCallback((location: TTSLocation, shouldPause = false) => { + if (shouldPause) { + resumeAfterLocationChangeRef.current = false; + } else if (isPlayingRef.current) { + resumeAfterLocationChangeRef.current = true; + } + // Reset state for new content in correct order abortAudio(); if (shouldPause) setIsPlaying(false); @@ -836,9 +866,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement if (handleBlankSection(workingText)) return; const shouldPause = normalizedOptions.shouldPause ?? false; + const pendingAutoResume = resumeAfterLocationChangeRef.current; + const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume); + if (shouldPause || pendingAutoResume) { + resumeAfterLocationChangeRef.current = false; + } // Keep track of previous state and pause playback - const wasPlaying = isPlaying; setIsPlaying(false); abortAudio(true); // Clear pending requests since text is changing setIsProcessing(true); // Set processing state before text processing starts @@ -917,7 +951,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setIsProcessing(false); // Restore playback state if needed - if (!shouldPause && wasPlaying) { + if (shouldResumePlayback) { setIsPlaying(true); } }) @@ -1053,54 +1087,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {string} sentence - The sentence to generate audio for * @returns {Promise} The generated audio buffer */ - const getAudio = useCallback(async (sentence: string, preload = false): Promise => { + const getAudio = useCallback(async ( + sentence: string, + sentenceIndex: number, + preload = false, + ): Promise => { const alignmentEnabledForCurrentDoc = wordHighlightFeatureEnabled && ((!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) || (isEPUB && epubHighlightEnabled && epubWordHighlightEnabled)); - // Helper to ensure we have an alignment for a given - // sentence/audio pair, even when the audio itself is - // served from the local cache. - const ensureAlignment = (arrayBuffer: TTSAudioBuffer) => { - if (!alignmentEnabledForCurrentDoc) return; - const alignmentKey = buildCacheKey( - sentence, - voice, - effectiveNativeSpeed, - configTTSProvider, - ttsModel, - ); - if (sentenceAlignmentCacheRef.current.has(alignmentKey)) return; - - try { - const audioBytes = Array.from(new Uint8Array(arrayBuffer)); - const alignmentBody = { - text: sentence, - audio: audioBytes, - }; - - void alignAudio(alignmentBody) - .then(async (data) => { - if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) { - return; - } - const alignment = data.alignments[0] as TTSSentenceAlignment; - sentenceAlignmentCacheRef.current.set(alignmentKey, alignment); - - const currentSentence = sentencesRef.current[currentIndexRef.current]; - if (currentSentence === sentence) { - setCurrentSentenceAlignment(alignment); - setCurrentWordIndex(null); - } - }) - .catch((err) => { - console.warn('Alignment request failed:', err); - }); - } catch (err) { - console.warn('Failed to start alignment request:', err); - } - }; - const audioCacheKey = buildCacheKey( sentence, voice, @@ -1109,83 +1104,94 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, ); - // Check if the audio is already cached const cachedAudio = audioCache.get(audioCacheKey); if (cachedAudio) { - console.log('Using cached audio for sentence:', sentence.substring(0, 20)); - // If we have audio but no alignment (e.g. after a - // navigation or TTS reset), kick off a fresh alignment - // request using the cached audio buffer. - ensureAlignment(cachedAudio); + const cachedManifest = segmentManifestCacheRef.current.get(audioCacheKey); + if (alignmentEnabledForCurrentDoc && cachedManifest?.alignment) { + sentenceAlignmentCacheRef.current.set(audioCacheKey, cachedManifest.alignment); + } return cachedAudio; } - // If the user is already out of quota, avoid spamming requests. - // Cached audio above is still allowed to play. - if (isAtLimit) { - if (!preload) { - setIsPlaying(false); - } + if (!documentId) { + if (!preload) setIsPlaying(false); return undefined; } + if (isAtLimit) { + if (!preload) setIsPlaying(false); + return undefined; + } + + const controller = new AbortController(); + activeAbortControllers.current.add(controller); + + const reqHeaders: TTSRequestHeaders = { + 'Content-Type': 'application/json', + 'x-openai-key': openApiKey || '', + 'x-tts-provider': configTTSProvider, + }; + if (openApiBaseUrl) { + reqHeaders['x-openai-base-url'] = openApiBaseUrl; + } + + const locator = isEPUB + ? { location: String(currDocPage), readerType: currentReaderType } + : { page: Number(currDocPageNumber || 1), readerType: currentReaderType }; + + const retryOptions: TTSRetryOptions = { + maxRetries: 2, + initialDelay: 300, + maxDelay: 300, + }; + try { - console.log('Requesting audio for sentence:', sentence); - - // Create an AbortController for this request - const controller = new AbortController(); - activeAbortControllers.current.add(controller); - - const reqHeaders: TTSRequestHeaders = { - 'Content-Type': 'application/json', - 'x-openai-key': openApiKey || '', - 'x-tts-provider': configTTSProvider, - }; - if (openApiBaseUrl) { - reqHeaders['x-openai-base-url'] = openApiBaseUrl; - } - - const reqBody: TTSRequestPayload = { - text: sentence, - voice, - speed: effectiveNativeSpeed, - format: 'mp3', - model: ttsModel, - instructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined, - }; - - // Allow one narrow client retry for transient browser->/api/tts transport failures. - // HTTP failures are not retried client-side. - const retryOptions: TTSRetryOptions = { - maxRetries: 2, - initialDelay: 300, - maxDelay: 300, - }; - onTTSStart(); - let arrayBuffer: TTSAudioBuffer; - try { - arrayBuffer = await withRetry( - async () => { - return await generateTTS(reqBody, reqHeaders, controller.signal); + const ensured = await withRetry( + async () => ensureTtsSegments({ + documentId, + settings: { + ttsProvider: configTTSProvider, + ttsModel, + voice, + nativeSpeed: effectiveNativeSpeed, + ...(supportsTtsInstructions(ttsModel) && ttsInstructions ? { ttsInstructions } : {}), }, - retryOptions - ); - } finally { - onTTSComplete(); + segments: [ + { + segmentIndex: sentenceIndex, + text: sentence, + locator, + }, + ], + }, reqHeaders, controller.signal), + retryOptions, + ); + + const segment = ensured.segments[0]; + if (!segment || segment.status !== 'completed' || !segment.audioUrl) { + throw new Error('Failed to prepare segment audio'); } - // Remove the controller once the request is complete - activeAbortControllers.current.delete(controller); + const response = await fetch(segment.audioUrl, { signal: controller.signal }); + if (!response.ok) { + throw new Error(`Segment audio fetch failed: ${response.status}`); + } + const arrayBuffer = await response.arrayBuffer(); + if (arrayBuffer.byteLength === 0) { + throw new Error('Received empty segment audio buffer'); + } - // Cache the array buffer audioCache.set(audioCacheKey, arrayBuffer); - - // Fire-and-forget alignment request; do not block audio playback - ensureAlignment(arrayBuffer); - + setSegmentManifestCache(audioCacheKey, segment); + if (alignmentEnabledForCurrentDoc && segment.alignment) { + sentenceAlignmentCacheRef.current.set(audioCacheKey, segment.alignment); + } return arrayBuffer; } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + return undefined; + } const status = (() => { if (typeof error === 'object' && error !== null && 'status' in error) { const maybe = (error as { status?: unknown }).status; @@ -1193,7 +1199,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } return undefined; })(); - const code = (() => { if (typeof error === 'object' && error !== null && 'code' in error) { const maybe = (error as { code?: unknown }).code; @@ -1201,23 +1206,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } return undefined; })(); - - // Check if this was an abort error - if (error instanceof Error && error.name === 'AbortError') { - console.log('TTS request aborted:', sentence.substring(0, 20)); - return; - } - - // If a preload request fails, we should not flip the global playback state. - // Otherwise the UI can lose the pause button while the current sentence - // continues playing. - if (!preload) { - setIsPlaying(false); - } - - // Handle daily quota exceeded (429 + Problem Details code) if (status === 429 && code === 'USER_DAILY_QUOTA_EXCEEDED') { - // Avoid noisy toasts from background preloading; keep the user-facing error for active playback. if (!preload) { toast.error('Daily TTS limit reached.', { id: 'tts-limit-error', @@ -1226,18 +1215,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } triggerRateLimit(); refreshRateLimit().catch(console.error); - // Do NOT re-throw, just return undefined to stop playback gracefully return undefined; } - - // Avoid noisy toasts from background preloading. if (!preload) { + setIsPlaying(false); toast.error('TTS failed. Skipped sentence and paused.', { id: 'tts-api-error', duration: 7000, }); } throw error; + } finally { + activeAbortControllers.current.delete(controller); + onTTSComplete(); } }, [ voice, @@ -1253,11 +1243,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement pdfWordHighlightEnabled, epubHighlightEnabled, epubWordHighlightEnabled, - triggerRateLimit, - refreshRateLimit, onTTSComplete, onTTSStart, - isAtLimit + isAtLimit, + documentId, + currDocPage, + currDocPageNumber, + currentReaderType, + refreshRateLimit, + triggerRateLimit, + setSegmentManifestCache, ]); /** @@ -1267,17 +1262,22 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {boolean} [preload=false] - Whether this is a preload request * @returns {Promise} The URL of the processed audio */ - const processSentence = useCallback(async (sentence: string, preload = false): Promise => { + const processSentence = useCallback(async ( + sentence: string, + sentenceIndex: number, + preload = false, + ): Promise => { if (!audioContext) throw new Error('Audio context not initialized'); + const requestKey = `${sentenceIndex}::${sentence}`; // Check if there's a pending preload request for this sentence - const pendingRequest = preloadRequests.current.get(sentence); + const pendingRequest = preloadRequests.current.get(requestKey); if (pendingRequest) { console.log('Using pending preload request for:', sentence.substring(0, 20)); setIsProcessing(true); // Show processing state when using pending request // If this is not a preload request, remove it from the pending map if (!preload) { - preloadRequests.current.delete(sentence); + preloadRequests.current.delete(requestKey); } return pendingRequest; } @@ -1288,7 +1288,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Create the audio processing promise const processPromise = (async () => { try { - const audioBuffer = await getAudio(sentence, preload); + const audioBuffer = await getAudio(sentence, sentenceIndex, preload); if (!audioBuffer) { // If quota or other handled error returns undefined, ensure we don't throw "No audio data" // Just return empty string to signal graceful failure/skip @@ -1308,11 +1308,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // If this is a preload request, store it in the map if (preload) { - preloadRequests.current.set(sentence, processPromise); + preloadRequests.current.set(requestKey, processPromise); // Clean up the map entry once the promise resolves or rejects void processPromise .finally(() => { - preloadRequests.current.delete(sentence); + preloadRequests.current.delete(requestKey); }) .catch(() => { // Prevent unhandled rejections from the cleanup-only chained promise. @@ -1341,7 +1341,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const createHowl = async (retryCount = 0): Promise => { let playErrorAttempts = 0; // Get the processed audio data URI directly from processSentence - const audioDataUri = await processSentence(sentence); + const audioDataUri = await processSentence(sentence, sentenceIndex); if (!audioDataUri) { // Graceful exit for rate limit / abort / intentionally skipped sentence console.log('Skipping playback for sentence (no audio generated)'); @@ -1698,7 +1698,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement return; } - const pending = preloadRequests.current.get(nextSentence); + const preloadKey = `${sentenceIndex}::${nextSentence}`; + const pending = preloadRequests.current.get(preloadKey); if (pending) { void pending .finally(() => { @@ -1710,7 +1711,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement return; } - void processSentence(nextSentence, true) + void processSentence(nextSentence, sentenceIndex, true) .catch((error) => { const status = (() => { if (typeof error === 'object' && error !== null && 'status' in error) { diff --git a/src/db/schema.ts b/src/db/schema.ts index 429e5a2..975b1e5 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -13,3 +13,4 @@ export const userTtsChars = usePostgres ? postgresSchema.userTtsChars : sqliteSc export const userPreferences = usePostgres ? postgresSchema.userPreferences : sqliteSchema.userPreferences; export const userDocumentProgress = usePostgres ? postgresSchema.userDocumentProgress : sqliteSchema.userDocumentProgress; export const documentPreviews = usePostgres ? postgresSchema.documentPreviews : sqliteSchema.documentPreviews; +export const ttsSegments = usePostgres ? postgresSchema.ttsSegments : sqliteSchema.ttsSegments; diff --git a/src/db/schema_postgres.ts b/src/db/schema_postgres.ts index b91792a..3652dc8 100644 --- a/src/db/schema_postgres.ts +++ b/src/db/schema_postgres.ts @@ -108,3 +108,28 @@ export const documentPreviews = pgTable('document_previews', { primaryKey({ columns: [table.documentId, table.namespace, table.variant] }), index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs), ]); + +export const ttsSegments = pgTable('tts_segments', { + segmentId: text('segment_id').notNull(), + userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), + documentId: text('document_id').notNull(), + readerType: text('reader_type').notNull(), + documentVersion: bigint('document_version', { mode: 'number' }).notNull(), + segmentIndex: integer('segment_index').notNull(), + locatorJson: text('locator_json'), + settingsHash: text('settings_hash').notNull(), + textHash: text('text_hash').notNull(), + textLength: integer('text_length').notNull().default(0), + audioKey: text('audio_key'), + audioFormat: text('audio_format').notNull().default('mp3'), + durationMs: integer('duration_ms'), + alignmentJson: text('alignment_json'), + status: text('status').notNull().default('pending'), + error: text('error'), + createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS), + updatedAt: bigint('updated_at', { mode: 'number' }).default(PG_NOW_MS), +}, (table) => [ + primaryKey({ columns: [table.segmentId, table.userId] }), + index('idx_tts_segments_lookup').on(table.userId, table.documentId, table.documentVersion, table.settingsHash), + index('idx_tts_segments_doc_index').on(table.userId, table.documentId, table.segmentIndex), +]); diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index d133a10..83af6ec 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -108,3 +108,28 @@ export const documentPreviews = sqliteTable('document_previews', { primaryKey({ columns: [table.documentId, table.namespace, table.variant] }), index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs), ]); + +export const ttsSegments = sqliteTable('tts_segments', { + segmentId: text('segment_id').notNull(), + userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), + documentId: text('document_id').notNull(), + readerType: text('reader_type').notNull(), + documentVersion: integer('document_version').notNull(), + segmentIndex: integer('segment_index').notNull(), + locatorJson: text('locator_json'), + settingsHash: text('settings_hash').notNull(), + textHash: text('text_hash').notNull(), + textLength: integer('text_length').notNull().default(0), + audioKey: text('audio_key'), + audioFormat: text('audio_format').notNull().default('mp3'), + durationMs: integer('duration_ms'), + alignmentJson: text('alignment_json'), + status: text('status').notNull().default('pending'), + error: text('error'), + createdAt: integer('created_at').default(SQLITE_NOW_MS), + updatedAt: integer('updated_at').default(SQLITE_NOW_MS), +}, (table) => [ + primaryKey({ columns: [table.segmentId, table.userId] }), + index('idx_tts_segments_lookup').on(table.userId, table.documentId, table.documentVersion, table.settingsHash), + index('idx_tts_segments_doc_index').on(table.userId, table.documentId, table.segmentIndex), +]); diff --git a/src/lib/client/api/audiobooks.ts b/src/lib/client/api/audiobooks.ts index 9705987..2395308 100644 --- a/src/lib/client/api/audiobooks.ts +++ b/src/lib/client/api/audiobooks.ts @@ -7,7 +7,9 @@ import type { CreateChapterPayload, VoicesResponse, AlignmentPayload, - AlignmentResponse + AlignmentResponse, + TTSSegmentsEnsureRequest, + TTSSegmentsEnsureResponse, } from '@/types/client'; import type { TTSAudiobookChapter, TTSAudioBuffer } from '@/types/tts'; @@ -251,3 +253,58 @@ export const alignAudio = async (payload: AlignmentPayload): Promise => { + const response = await fetch('/api/tts/segments/ensure', { + method: 'POST', + headers: headers as HeadersInit, + body: JSON.stringify(payload), + signal, + }); + + if (!response.ok) { + let problem: unknown = undefined; + const contentType = response.headers.get('content-type') || ''; + if (contentType.includes('application/problem+json') || contentType.includes('application/json')) { + problem = await response.json().catch(() => null); + } + + const err = new Error(`TTS segment ensure failed with status ${response.status}`) as TTSRequestError; + err.status = response.status; + if (typeof problem === 'object' && problem !== null) { + const rec = problem as Record; + if (typeof rec.code === 'string') err.code = rec.code; + if (typeof rec.type === 'string') err.type = rec.type; + if (typeof rec.title === 'string') err.title = rec.title; + if (typeof rec.detail === 'string') err.detail = rec.detail; + } + + throw err; + } + + return await response.json(); +}; + +export const fetchTtsSegmentManifest = async (params: { + documentId: string; + settingsHash: string; + fromIndex: number; + toIndex: number; +}): Promise => { + const query = new URLSearchParams({ + documentId: params.documentId, + settingsHash: params.settingsHash, + fromIndex: String(params.fromIndex), + toIndex: String(params.toIndex), + }); + const response = await fetch(`/api/tts/segments/manifest?${query.toString()}`); + if (!response.ok) { + const data = await response.json().catch(() => null) as { error?: string } | null; + throw new Error(data?.error || 'Failed to fetch segment manifest'); + } + return await response.json(); +}; diff --git a/src/lib/server/tts/segments-auth.ts b/src/lib/server/tts/segments-auth.ts new file mode 100644 index 0000000..e68b68b --- /dev/null +++ b/src/lib/server/tts/segments-auth.ts @@ -0,0 +1,64 @@ +import { and, eq, inArray } from 'drizzle-orm'; +import type { NextRequest } from 'next/server'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth/auth'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; +import type { ReaderType } from '@/types/user-state'; + +export type ResolvedSegmentDocumentScope = { + testNamespace: string | null; + storageUserId: string; + authEnabled: boolean; + userId: string | null; + isAnonymousUser: boolean; + documentVersion: number; + readerType: ReaderType; +}; + +function toReaderType(documentType: string): ReaderType { + if (documentType === 'pdf') return 'pdf'; + if (documentType === 'epub') return 'epub'; + return 'html'; +} + +export async function resolveSegmentDocumentScope( + request: NextRequest, + documentId: string, +): Promise { + const ctxOrRes = await requireAuthContext(request); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(request.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const rows = (await db + .select({ + userId: documents.userId, + lastModified: documents.lastModified, + type: documents.type, + }) + .from(documents) + .where(and(eq(documents.id, documentId), inArray(documents.userId, allowedUserIds)))) as Array<{ + userId: string; + lastModified: number; + type: string; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return Response.json({ error: 'Document not found' }, { status: 404 }); + } + + return { + testNamespace, + storageUserId: doc.userId, + authEnabled: ctxOrRes.authEnabled, + userId: ctxOrRes.userId, + isAnonymousUser: Boolean(ctxOrRes.user?.isAnonymous), + documentVersion: Number(doc.lastModified), + readerType: toReaderType(doc.type), + }; +} diff --git a/src/lib/server/tts/segments-blobstore.ts b/src/lib/server/tts/segments-blobstore.ts new file mode 100644 index 0000000..6bb3c8a --- /dev/null +++ b/src/lib/server/tts/segments-blobstore.ts @@ -0,0 +1,104 @@ +import { + DeleteObjectsCommand, + GetObjectCommand, + ListObjectsV2Command, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; + +function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream { + return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function'; +} + +async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) { + chunks.push(chunk); + } else if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk)); + } else { + chunks.push(Buffer.from(chunk as Uint8Array)); + } + } + return Buffer.concat(chunks); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + + if (isNodeReadableStream(body)) { + return streamToBuffer(body); + } + + throw new Error('Unsupported S3 response body type'); +} + +export async function putTtsSegmentAudioObject(key: string, buffer: Buffer): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send(new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: buffer, + ContentType: 'audio/mpeg', + ServerSideEncryption: 'AES256', + })); +} + +export async function getTtsSegmentAudioObject(key: string): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const res = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: key })); + return bodyToBuffer(res.Body); +} + +export async function deleteTtsSegmentPrefix(prefix: string): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const cleanedPrefix = prefix.replace(/^\/+/, ''); + let deleted = 0; + let continuationToken: string | undefined; + + do { + const listRes = await client.send( + new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: cleanedPrefix, + ContinuationToken: continuationToken, + }), + ); + + const keys = (listRes.Contents ?? []) + .map((item) => item.Key) + .filter((value): value is string => typeof value === 'string' && value.length > 0); + + if (keys.length > 0) { + const deleteRes = await client.send( + new DeleteObjectsCommand({ + Bucket: cfg.bucket, + Delete: { + Objects: keys.map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + deleted += deleteRes.Deleted?.length ?? 0; + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + + return deleted; +} diff --git a/src/lib/server/tts/segments.ts b/src/lib/server/tts/segments.ts new file mode 100644 index 0000000..03e2bec --- /dev/null +++ b/src/lib/server/tts/segments.ts @@ -0,0 +1,188 @@ +import { createHash, createHmac } from 'crypto'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import { ffprobeAudio } from '@/lib/server/audiobooks/chapters'; +import type { + TTSSegmentLocator, + TTSSegmentSettings, +} from '@/types/client'; +import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; + +function stableStringify(value: unknown): string { + if (value === null || typeof value !== 'object') { + return JSON.stringify(value); + } + + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item)).join(',')}]`; + } + + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}`; +} + +function settingsCanonical(settings: TTSSegmentSettings): string { + return stableStringify({ + provider: settings.ttsProvider, + model: settings.ttsModel, + voice: settings.voice, + speed: Number.isFinite(Number(settings.nativeSpeed)) ? Number(settings.nativeSpeed) : 1, + instructions: settings.ttsInstructions || '', + format: 'mp3', + }); +} + +export function buildTtsSegmentSettingsHash(settings: TTSSegmentSettings): string { + return createHash('sha256').update(settingsCanonical(settings)).digest('hex'); +} + +export function normalizeSegmentText(text: string): string { + return preprocessSentenceForAudio(text || '').trim(); +} + +export function normalizeLocator(locator: TTSSegmentLocator | undefined): TTSSegmentLocator | null { + if (!locator) return null; + const normalized: TTSSegmentLocator = {}; + if (typeof locator.page === 'number' && Number.isFinite(locator.page)) { + normalized.page = Math.max(1, Math.floor(locator.page)); + } + if (typeof locator.location === 'string' && locator.location.trim()) { + normalized.location = locator.location.trim(); + } + if (locator.readerType === 'pdf' || locator.readerType === 'epub' || locator.readerType === 'html') { + normalized.readerType = locator.readerType; + } + if (Object.keys(normalized).length === 0) return null; + return normalized; +} + +export function locatorFingerprint(locator: TTSSegmentLocator | null): string { + if (!locator) return ''; + return createHash('sha256').update(stableStringify(locator)).digest('hex'); +} + +export function buildTtsSegmentId(input: { + documentId: string; + documentVersion: number; + settingsHash: string; + segmentIndex: number; + normalizedText: string; + locatorFingerprint: string; +}): string { + const canonical = stableStringify({ + d: input.documentId, + v: input.documentVersion, + s: input.settingsHash, + i: input.segmentIndex, + t: input.normalizedText, + l: input.locatorFingerprint, + }); + return createHash('sha256').update(canonical).digest('hex'); +} + +export function buildTtsSegmentTextHash(text: string, secret: string): string { + return createHmac('sha256', secret).update(text).digest('hex'); +} + +export function buildTtsSegmentAudioKey(input: { + storagePrefix: string; + namespace: string | null; + userId: string; + documentId: string; + documentVersion: number; + settingsHash: string; + segmentId: string; +}): string { + const nsSegment = input.namespace ? `ns/${input.namespace}/` : ''; + return `${input.storagePrefix}/tts_segments_v1/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`; +} + +export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise { + let workDir: string | null = null; + try { + workDir = await mkdtemp(join(tmpdir(), 'openreader-tts-segment-')); + const audioPath = join(workDir, 'segment.mp3'); + await writeFile(audioPath, buffer); + const probe = await ffprobeAudio(audioPath, signal); + const sec = Number(probe.durationSec ?? 0); + if (!Number.isFinite(sec) || sec <= 0) { + return 0; + } + return Math.max(0, Math.floor(sec * 1000)); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +function alignWordsToText(sentence: string): Array<{ text: string; charStart: number; charEnd: number }> { + const words = sentence.match(/\S+/g) || []; + const aligned: Array<{ text: string; charStart: number; charEnd: number }> = []; + let cursor = 0; + const lowerSentence = sentence.toLowerCase(); + + for (const token of words) { + const clean = token.trim(); + if (!clean) continue; + const idx = lowerSentence.indexOf(clean.toLowerCase(), cursor); + const start = idx >= 0 ? idx : cursor; + const end = Math.min(sentence.length, start + clean.length); + cursor = Math.max(cursor, end); + aligned.push({ + text: clean, + charStart: start, + charEnd: end, + }); + } + + return aligned; +} + +export function buildProportionalAlignment(input: { + sentence: string; + sentenceIndex: number; + durationMs: number; +}): TTSSentenceAlignment { + const wordsWithOffsets = alignWordsToText(input.sentence); + if (wordsWithOffsets.length === 0 || input.durationMs <= 0) { + return { + sentence: input.sentence, + sentenceIndex: input.sentenceIndex, + words: [], + }; + } + + const weighted = wordsWithOffsets.map((word) => ({ + ...word, + weight: Math.max(1, word.text.replace(/[^a-zA-Z0-9]/g, '').length), + })); + const totalWeight = weighted.reduce((sum, word) => sum + word.weight, 0); + + let consumedMs = 0; + const alignedWords: TTSSentenceWord[] = weighted.map((word, index) => { + const remainingMs = Math.max(0, input.durationMs - consumedMs); + const sliceMs = index === weighted.length - 1 + ? remainingMs + : Math.max(1, Math.round((input.durationMs * word.weight) / Math.max(1, totalWeight))); + + const startMs = consumedMs; + consumedMs += Math.min(sliceMs, remainingMs); + + return { + text: word.text, + startSec: startMs / 1000, + endSec: consumedMs / 1000, + charStart: word.charStart, + charEnd: word.charEnd, + }; + }); + + return { + sentence: input.sentence, + sentenceIndex: input.sentenceIndex, + words: alignedWords, + }; +} diff --git a/src/lib/server/user/data-cleanup.ts b/src/lib/server/user/data-cleanup.ts index 187fc4a..9b74c67 100644 --- a/src/lib/server/user/data-cleanup.ts +++ b/src/lib/server/user/data-cleanup.ts @@ -7,10 +7,12 @@ import { db } from '@/db'; import { documents, audiobooks } from '@/db/schema'; import { eq } from 'drizzle-orm'; -import { isS3Configured } from '@/lib/server/storage/s3'; +import { getS3Config, isS3Configured } from '@/lib/server/storage/s3'; import { deleteDocumentBlob } from '@/lib/server/documents/blobstore'; import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; +import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews'; import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore'; +import { deleteTtsSegmentPrefix } from '@/lib/server/tts/segments-blobstore'; type DocumentRow = { id: string }; type AudiobookRow = { id: string }; @@ -29,7 +31,7 @@ export async function deleteUserStorageData( userId: string, namespace: string | null, ): Promise { - if (!isS3Configured()) return; + const s3Enabled = isS3Configured(); // eslint-disable-next-line @typescript-eslint/no-explicit-any const database = db as any; @@ -42,25 +44,36 @@ export async function deleteUserStorageData( let docsDeleted = 0; for (const doc of userDocs) { - try { - await deleteDocumentBlob(doc.id, namespace); - docsDeleted++; - } catch (error) { - console.error(`[user-data-cleanup] Failed to delete document blob ${doc.id}:`, error); + if (s3Enabled) { + try { + await deleteDocumentBlob(doc.id, namespace); + docsDeleted++; + } catch (error) { + console.error(`[user-data-cleanup] Failed to delete document blob ${doc.id}:`, error); + } + + try { + await deleteDocumentPreviewArtifacts(doc.id, namespace); + } catch (error) { + console.error(`[user-data-cleanup] Failed to delete preview for ${doc.id}:`, error); + } } + // Always clean up DB rows — documentPreviews has no FK cascade on user. try { - await deleteDocumentPreviewArtifacts(doc.id, namespace); + await deleteDocumentPreviewRows(doc.id, namespace); } catch (error) { - console.error(`[user-data-cleanup] Failed to delete preview for ${doc.id}:`, error); + console.error(`[user-data-cleanup] Failed to delete preview rows for ${doc.id}:`, error); } } // --- Audiobooks --- - const userBooks: AudiobookRow[] = await database - .select({ id: audiobooks.id }) - .from(audiobooks) - .where(eq(audiobooks.userId, userId)); + const userBooks: AudiobookRow[] = s3Enabled + ? await database + .select({ id: audiobooks.id }) + .from(audiobooks) + .where(eq(audiobooks.userId, userId)) + : []; let booksDeleted = 0; for (const book of userBooks) { @@ -73,11 +86,25 @@ export async function deleteUserStorageData( } } - if (docsDeleted > 0 || booksDeleted > 0) { + // --- TTS segments --- + let segmentsDeleted = 0; + if (s3Enabled) { + try { + const cfg = getS3Config(); + const nsSegment = namespace ? `ns/${namespace}/` : ''; + const ttsPrefix = `${cfg.prefix}/tts_segments_v1/${nsSegment}users/${encodeURIComponent(userId)}/`; + segmentsDeleted = await deleteTtsSegmentPrefix(ttsPrefix); + } catch (error) { + console.error(`[user-data-cleanup] Failed to delete TTS segment blobs for user ${userId}:`, error); + } + } + + if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) { console.log( `[user-data-cleanup] Cleaned up S3 data for user ${userId}: ` + `${docsDeleted}/${userDocs.length} document(s), ` + - `${booksDeleted}/${userBooks.length} audiobook(s)`, + `${booksDeleted}/${userBooks.length} audiobook(s), ` + + `${segmentsDeleted} tts segment object(s)`, ); } } diff --git a/src/lib/server/whisper/alignment.ts b/src/lib/server/whisper/alignment.ts new file mode 100644 index 0000000..488b3c5 --- /dev/null +++ b/src/lib/server/whisper/alignment.ts @@ -0,0 +1,395 @@ +import { createHash, randomUUID } from 'crypto'; +import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { spawn } from 'child_process'; +import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts'; +import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin'; + +interface WhisperAlignmentOptions { + engine?: 'whisper.cpp'; + lang?: string; +} + +interface WhisperWord { + start: number; + end: number; + word: string; +} + +export interface WhisperRequestBody { + text: string; + audio: TTSAudioBytes; + lang?: string; +} + +const alignmentCache = new Map(); + +const MODEL_NAME = 'ggml-tiny.en.bin'; +const MODEL_URL = + 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin'; +const DOCSTORE_DIR = join(process.cwd(), 'docstore'); +const MODEL_DIR = join(DOCSTORE_DIR, 'model'); +const MODEL_PATH = join(MODEL_DIR, MODEL_NAME); +const modelReadyPromises = new Map>(); + +async function ensureModelAvailable(): Promise { + try { + await access(MODEL_PATH); + return; + } catch { + // continue + } + + const existing = modelReadyPromises.get(MODEL_PATH); + if (existing) return existing; + + const promise = (async () => { + try { + await access(MODEL_PATH); + return; + } catch { + // still missing + } + + await mkdir(MODEL_DIR, { recursive: true }); + + const res = await fetch(MODEL_URL); + if (!res.ok) { + throw new Error( + `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}` + ); + } + + const arrayBuffer = await res.arrayBuffer(); + await writeFile(MODEL_PATH, Buffer.from(arrayBuffer)); + })(); + + modelReadyPromises.set(MODEL_PATH, promise); + return promise; +} + +async function runWhisperCpp( + wavPath: string, + opts: WhisperAlignmentOptions +): Promise { + const binary = process.env.WHISPER_CPP_BIN; + if (!binary) { + throw new Error( + 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.' + ); + } + + await ensureModelAvailable(); + + return new Promise((resolve, reject) => { + const jsonBase = `${wavPath}.json_out`; + const jsonPath = `${jsonBase}.json`; + const args = [ + '-m', + MODEL_PATH, + '-f', + wavPath, + '-of', + jsonBase, + '-ojf', + '-np', + ]; + + if (opts.lang) { + args.push('-l', opts.lang); + } + + const child = spawn(binary, args); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('error', (err) => { + reject(err); + }); + + child.on('close', (code) => { + if (code !== 0) { + return reject( + new Error( + `whisper.cpp exited with code ${code}: ${stderr || stdout}` + ) + ); + } + + readFile(jsonPath, 'utf-8') + .then((content: string) => { + const words: WhisperWord[] = []; + const parsed = JSON.parse(content) as { + transcription?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + tokens?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + }>; + }>; + }; + + const transcription = parsed.transcription; + + const parseTimecode = (value?: string): number | null => { + if (!value) return null; + const m = value.match(/(\d+):(\d+):(\d+),(\d+)/); + if (!m) return null; + const h = Number(m[1]); + const min = Number(m[2]); + const s = Number(m[3]); + const ms = Number(m[4]); + if ( + Number.isNaN(h) + || Number.isNaN(min) + || Number.isNaN(s) + || Number.isNaN(ms) + ) { + return null; + } + return h * 3600 + min * 60 + s + ms / 1000; + }; + + if (Array.isArray(transcription)) { + for (const seg of transcription) { + const segText = (seg.text || '').trim(); + const segStartSecFromTs = parseTimecode( + seg.timestamps?.from + ); + const segEndSecFromTs = parseTimecode(seg.timestamps?.to); + const segStartSecFromMs = + typeof seg.offsets?.from === 'number' + ? seg.offsets.from / 1000 + : null; + const segEndSecFromMs = + typeof seg.offsets?.to === 'number' + ? seg.offsets.to / 1000 + : null; + + const segStartSec = + segStartSecFromTs + ?? segStartSecFromMs + ?? 0; + const segEndSec = + segEndSecFromTs + ?? segEndSecFromMs + ?? segStartSec; + + const tokens = Array.isArray(seg.tokens) + ? seg.tokens + : []; + + if (tokens.length > 0) { + for (const token of tokens) { + const rawText = token.text || ''; + const tokenText = rawText.trim(); + if (!tokenText || /^\[.*\]$/.test(tokenText)) continue; + + const tokStartSecFromTs = parseTimecode( + token.timestamps?.from + ); + const tokEndSecFromTs = parseTimecode( + token.timestamps?.to + ); + const tokStartSecFromMs = + typeof token.offsets?.from === 'number' + ? token.offsets.from / 1000 + : null; + const tokEndSecFromMs = + typeof token.offsets?.to === 'number' + ? token.offsets.to / 1000 + : null; + + const startSec = + tokStartSecFromTs + ?? tokStartSecFromMs + ?? segStartSec; + const endSec = + tokEndSecFromTs + ?? tokEndSecFromMs + ?? segEndSec; + + words.push({ + word: tokenText, + start: startSec, + end: endSec, + }); + } + } else if (segText) { + const segTokens = segText.split(/\s+/).filter(Boolean); + if (segTokens.length) { + const totalDur = Math.max(segEndSec - segStartSec, 0); + const step = + segTokens.length > 0 + ? totalDur / segTokens.length + : 0; + segTokens.forEach((token, index) => { + const wStart = + step > 0 + ? segStartSec + step * index + : segStartSec; + const wEnd = + step > 0 + ? index === segTokens.length - 1 + ? segEndSec + : segStartSec + step * (index + 1) + : segEndSec; + words.push({ + word: token, + start: wStart, + end: wEnd, + }); + }); + } + } + } + } + + resolve(words); + }) + .catch((err: unknown) => { + reject(err); + }); + }); + }); +} + +function mapWordsToSentenceOffsets( + sentence: string, + words: WhisperWord[] +): TTSSentenceAlignment { + const normalizedSentence = preprocessSentenceForAudio(sentence); + let cursor = 0; + + const alignedWords = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = normalizedSentence + .toLowerCase() + .indexOf(token.toLowerCase(), cursor); + + const start = + idx !== -1 + ? idx + : cursor; + const end = start + token.length; + + cursor = end; + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords.filter((w) => w.text.length > 0), + }; +} + +export async function alignAudioWithText( + audioBuffer: TTSAudioBuffer, + text: string, + cacheKey?: string, + opts: WhisperAlignmentOptions = {} +): Promise { + if (!text.trim()) { + return []; + } + + if (cacheKey && alignmentCache.has(cacheKey)) { + return alignmentCache.get(cacheKey)!; + } + + const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); + const inputPath = join(tmpBase, `${randomUUID()}-input.bin`); + const wavPath = join(tmpBase, `${randomUUID()}-input.wav`); + + try { + await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); + + await new Promise((resolve, reject) => { + const ffmpeg = spawn(getFFmpegPath(), [ + '-y', + '-i', + inputPath, + '-ar', + '16000', + '-ac', + '1', + wavPath, + ]); + + let stderr = ''; + ffmpeg.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('error', (err) => { + reject(err); + }); + + ffmpeg.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject( + new Error(`ffmpeg failed with code ${code}: ${stderr}`) + ); + } + }); + }); + + const words = await runWhisperCpp(wavPath, opts); + const alignment = mapWordsToSentenceOffsets(text, words); + const result: TTSSentenceAlignment[] = [alignment]; + + if (cacheKey) { + alignmentCache.set(cacheKey, result); + } + + return result; + } finally { + await rm(tmpBase, { recursive: true, force: true }).catch(() => {}); + } +} + +export function makeWhisperCacheKey(input: WhisperRequestBody): string { + return createHash('sha256') + .update( + JSON.stringify({ + text: input.text, + lang: input.lang || '', + audioLen: input.audio?.length || 0, + }) + ) + .digest('hex'); +} diff --git a/src/types/client.ts b/src/types/client.ts index 1f3eab0..a1985fb 100644 --- a/src/types/client.ts +++ b/src/types/client.ts @@ -85,3 +85,46 @@ export interface AlignmentPayload { export interface AlignmentResponse { alignments: TTSSentenceAlignment[]; } + +export interface TTSSegmentSettings { + ttsProvider: string; + ttsModel: string; + voice: string; + nativeSpeed: number; + ttsInstructions?: string; +} + +export interface TTSSegmentLocator { + page?: number; + location?: string; + readerType?: 'pdf' | 'epub' | 'html'; +} + +export interface TTSSegmentInput { + segmentIndex: number; + text: string; + locator?: TTSSegmentLocator; +} + +export interface TTSSegmentsEnsureRequest { + documentId: string; + segments: TTSSegmentInput[]; + settings: TTSSegmentSettings; +} + +export interface TTSSegmentManifestItem { + segmentId: string; + segmentIndex: number; + audioUrl: string; + durationMs: number; + alignment: TTSSentenceAlignment | null; + locator: TTSSegmentLocator | null; + status: 'pending' | 'completed' | 'error'; +} + +export interface TTSSegmentsEnsureResponse { + documentId: string; + documentVersion: number; + settingsHash: string; + segments: TTSSegmentManifestItem[]; +} diff --git a/tests/global-teardown.ts b/tests/global-teardown.ts index f47f32a..f77bdd5 100644 --- a/tests/global-teardown.ts +++ b/tests/global-teardown.ts @@ -4,6 +4,7 @@ import { db } from '../src/db'; import { audiobooks, audiobookChapters, documents } from '../src/db/schema'; import { deleteDocumentPrefix } from '../src/lib/server/documents/blobstore'; import { deleteAudiobookPrefix } from '../src/lib/server/audiobooks/blobstore'; +import { deleteTtsSegmentPrefix } from '../src/lib/server/tts/segments-blobstore'; import { getS3Client, getS3Config, isS3Configured } from '../src/lib/server/storage/s3'; function chunk(items: T[], size: number): T[][] { @@ -75,6 +76,7 @@ export default async function globalTeardown(): Promise { const config = getS3Config(); const docsNsRootPrefix = `${config.prefix}/documents_v1/ns/`; const audiobooksNsRootPrefix = `${config.prefix}/audiobooks_v1/ns/`; + const ttsSegmentsNsRootPrefix = `${config.prefix}/tts_segments_v1/ns/`; // Remove SQL audiobook rows for namespaced objects (covers auth claim flows too). const audiobookKeys = await listKeysByPrefix(audiobooksNsRootPrefix); @@ -103,4 +105,5 @@ export default async function globalTeardown(): Promise { await deleteDocumentPrefix(docsNsRootPrefix); await deleteAudiobookPrefix(audiobooksNsRootPrefix); + await deleteTtsSegmentPrefix(ttsSegmentsNsRootPrefix); } diff --git a/tests/unit/tts-segments.spec.ts b/tests/unit/tts-segments.spec.ts new file mode 100644 index 0000000..1c5c3bb --- /dev/null +++ b/tests/unit/tts-segments.spec.ts @@ -0,0 +1,84 @@ +import { expect, test } from '@playwright/test'; +import { + buildProportionalAlignment, + buildTtsSegmentId, + buildTtsSegmentSettingsHash, + buildTtsSegmentTextHash, + locatorFingerprint, + normalizeLocator, + normalizeSegmentText, +} from '../../src/lib/server/tts/segments'; + +test.describe('tts segment helpers', () => { + test('builds stable settings hash', () => { + const a = buildTtsSegmentSettingsHash({ + ttsProvider: 'openai', + ttsModel: 'gpt-4o-mini-tts', + voice: 'alloy', + nativeSpeed: 1, + ttsInstructions: 'calm', + }); + const b = buildTtsSegmentSettingsHash({ + ttsProvider: 'openai', + ttsModel: 'gpt-4o-mini-tts', + voice: 'alloy', + nativeSpeed: 1, + ttsInstructions: 'calm', + }); + expect(a).toBe(b); + }); + + test('builds deterministic segment id', () => { + const id1 = buildTtsSegmentId({ + documentId: 'doc', + documentVersion: 1, + settingsHash: 'abc', + segmentIndex: 2, + normalizedText: 'hello world', + locatorFingerprint: 'loc', + }); + const id2 = buildTtsSegmentId({ + documentId: 'doc', + documentVersion: 1, + settingsHash: 'abc', + segmentIndex: 2, + normalizedText: 'hello world', + locatorFingerprint: 'loc', + }); + const id3 = buildTtsSegmentId({ + documentId: 'doc', + documentVersion: 1, + settingsHash: 'abc', + segmentIndex: 3, + normalizedText: 'hello world', + locatorFingerprint: 'loc', + }); + expect(id1).toBe(id2); + expect(id1).not.toBe(id3); + }); + + test('does not leak plaintext via text hash', () => { + const hash = buildTtsSegmentTextHash('plain sentence', 'secret'); + expect(hash).not.toContain('plain'); + expect(hash).toHaveLength(64); + }); + + test('normalizes locators and creates fingerprints', () => { + const locator = normalizeLocator({ page: 2.9, location: ' cfi(1) ', readerType: 'epub' }); + expect(locator).toEqual({ page: 2, location: 'cfi(1)', readerType: 'epub' }); + expect(locatorFingerprint(locator)).toHaveLength(64); + }); + + test('builds proportional alignment preserving order', () => { + const alignment = buildProportionalAlignment({ + sentence: normalizeSegmentText('Hello world again'), + sentenceIndex: 5, + durationMs: 1500, + }); + expect(alignment.sentenceIndex).toBe(5); + expect(alignment.words.length).toBe(3); + expect(alignment.words[0].startSec).toBe(0); + expect(alignment.words[2].endSec).toBeGreaterThan(1.4); + expect(alignment.words[1].charStart).toBeGreaterThan(alignment.words[0].charStart); + }); +});