diff --git a/drizzle/postgres/0001_tts_segments.sql b/drizzle/postgres/0001_tts_segments.sql new file mode 100644 index 0000000..7ef400f --- /dev/null +++ b/drizzle/postgres/0001_tts_segments.sql @@ -0,0 +1,27 @@ +CREATE TABLE "tts_segments" ( + "segment_id" text NOT NULL, + "user_id" text NOT NULL, + "document_id" text NOT NULL, + "reader_type" text NOT NULL, + "document_version" bigint NOT NULL, + "segment_index" integer NOT NULL, + "locator_json" text, + "settings_hash" text NOT NULL, + "text_hash" text NOT NULL, + "text_length" integer DEFAULT 0 NOT NULL, + "audio_key" text, + "audio_format" text DEFAULT 'mp3' NOT NULL, + "duration_ms" integer, + "alignment_json" text, + "status" text DEFAULT 'pending' NOT NULL, + "error" text, + "created_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint, + "updated_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint, + CONSTRAINT "tts_segments_segment_id_user_id_pk" PRIMARY KEY("segment_id","user_id") +); +--> statement-breakpoint +ALTER TABLE "tts_segments" ADD CONSTRAINT "tts_segments_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +CREATE INDEX "idx_tts_segments_lookup" ON "tts_segments" USING btree ("user_id","document_id","document_version","settings_hash"); +--> statement-breakpoint +CREATE INDEX "idx_tts_segments_doc_index" ON "tts_segments" USING btree ("user_id","document_id","segment_index"); diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index f20e818..1121af8 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1771386480954, "tag": "0000_black_lucky_pierre", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1777843301471, + "tag": "0001_tts_segments", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/sqlite/0001_tts_segments.sql b/drizzle/sqlite/0001_tts_segments.sql new file mode 100644 index 0000000..64f2ce0 --- /dev/null +++ b/drizzle/sqlite/0001_tts_segments.sql @@ -0,0 +1,26 @@ +CREATE TABLE `tts_segments` ( + `segment_id` text NOT NULL, + `user_id` text NOT NULL, + `document_id` text NOT NULL, + `reader_type` text NOT NULL, + `document_version` integer NOT NULL, + `segment_index` integer NOT NULL, + `locator_json` text, + `settings_hash` text NOT NULL, + `text_hash` text NOT NULL, + `text_length` integer DEFAULT 0 NOT NULL, + `audio_key` text, + `audio_format` text DEFAULT 'mp3' NOT NULL, + `duration_ms` integer, + `alignment_json` text, + `status` text DEFAULT 'pending' NOT NULL, + `error` text, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)), + `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)), + PRIMARY KEY(`segment_id`, `user_id`), + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `idx_tts_segments_lookup` ON `tts_segments` (`user_id`,`document_id`,`document_version`,`settings_hash`); +--> statement-breakpoint +CREATE INDEX `idx_tts_segments_doc_index` ON `tts_segments` (`user_id`,`document_id`,`segment_index`); diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index 3536d28..6c3ee9d 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1771386480690, "tag": "0000_familiar_caretaker", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1777843301472, + "tag": "0001_tts_segments", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/api/account/delete/route.ts b/src/app/api/account/delete/route.ts index 76358f6..5e59e83 100644 --- a/src/app/api/account/delete/route.ts +++ b/src/app/api/account/delete/route.ts @@ -2,6 +2,8 @@ import { headers } from 'next/headers'; import { NextResponse } from 'next/server'; import { auth } from '@/lib/server/auth/auth'; import { isAuthEnabled } from '@/lib/server/auth/config'; +import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; +import { deleteUserStorageData } from '@/lib/server/user/data-cleanup'; export async function DELETE() { if (!isAuthEnabled() || !auth) { @@ -19,6 +21,17 @@ export async function DELETE() { } try { + // Best-effort cleanup for test namespaced storage using request context. + // The Better Auth beforeDelete hook still runs and handles non-namespaced data. + const testNamespace = getOpenReaderTestNamespace(reqHeaders); + if (testNamespace) { + try { + await deleteUserStorageData(session.user.id, testNamespace); + } catch (error) { + console.error('[account-delete] Failed to clean up namespaced user storage before deletion:', error); + } + } + // Use Better Auth's built-in deleteUser to handle cascading cleanup await auth.api.deleteUser({ headers: reqHeaders, diff --git a/src/app/api/tts/segments/audio/route.ts b/src/app/api/tts/segments/audio/route.ts new file mode 100644 index 0000000..e2e0a93 --- /dev/null +++ b/src/app/api/tts/segments/audio/route.ts @@ -0,0 +1,70 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq } from 'drizzle-orm'; +import { db } from '@/db'; +import { ttsSegments } from '@/db/schema'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import { getTtsSegmentAudioObject } from '@/lib/server/tts/segments-blobstore'; +import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'TTS segments storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function streamBuffer(buffer: Buffer): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(buffer)); + controller.close(); + }, + }); +} + +export async function GET(request: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const documentId = (request.nextUrl.searchParams.get('documentId') || '').trim().toLowerCase(); + const segmentId = (request.nextUrl.searchParams.get('segmentId') || '').trim().toLowerCase(); + if (!documentId || !segmentId) { + return NextResponse.json({ error: 'Missing documentId or segmentId' }, { status: 400 }); + } + + const scope = await resolveSegmentDocumentScope(request, documentId); + if (scope instanceof Response) return scope; + + const rows = (await db + .select({ + audioKey: ttsSegments.audioKey, + status: ttsSegments.status, + }) + .from(ttsSegments) + .where(and( + eq(ttsSegments.userId, scope.storageUserId), + eq(ttsSegments.documentId, documentId), + eq(ttsSegments.documentVersion, scope.documentVersion), + eq(ttsSegments.segmentId, segmentId), + ))) as Array<{ audioKey: string | null; status: string }>; + + const row = rows[0]; + if (!row || !row.audioKey || row.status !== 'completed') { + return NextResponse.json({ error: 'Segment audio not found' }, { status: 404 }); + } + + const audio = await getTtsSegmentAudioObject(row.audioKey); + + return new NextResponse(streamBuffer(audio), { + headers: { + 'Content-Type': 'audio/mpeg', + 'Cache-Control': 'private, max-age=86400', + }, + }); + } catch (error) { + console.error('Error serving TTS segment audio:', error); + return NextResponse.json({ error: 'Failed to load segment audio' }, { status: 500 }); + } +} diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts new file mode 100644 index 0000000..a64cd9c --- /dev/null +++ b/src/app/api/tts/segments/ensure/route.ts @@ -0,0 +1,435 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { ttsSegments } from '@/db/schema'; +import { isS3Configured, getS3Config } from '@/lib/server/storage/s3'; +import { generateTTSBuffer } from '@/lib/server/tts/generate'; +import { getTtsSegmentAudioObject, putTtsSegmentAudioObject } from '@/lib/server/tts/segments-blobstore'; +import { + buildTtsSegmentAudioKey, + buildTtsSegmentId, + buildTtsSegmentSettingsHash, + buildTtsSegmentTextHash, + locatorFingerprint, + normalizeLocator, + normalizeSegmentText, + probeAudioDurationMsFromBuffer, +} from '@/lib/server/tts/segments'; +import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; +import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { getClientIp } from '@/lib/server/rate-limit/request-ip'; +import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; +import { alignAudioWithText } from '@/lib/server/whisper/alignment'; +import type { + TTSSegmentInput, + TTSSegmentManifestItem, + TTSSegmentSettings, + TTSSegmentsEnsureRequest, +} from '@/types/client'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +function attachDeviceIdCookie(response: NextResponse, deviceId: string | null, didCreate: boolean) { + if (didCreate && deviceId) { + setDeviceIdCookie(response, deviceId); + } +} + +function formatLimitForHint(limit: number): string { + if (!Number.isFinite(limit) || limit <= 0) return String(limit); + if (limit >= 1_000_000) { + const m = limit / 1_000_000; + return `${m % 1 === 0 ? m.toFixed(0) : m.toFixed(1)}M`; + } + if (limit >= 1_000) return `${Math.round(limit / 1_000)}K`; + return String(limit); +} + +function parseSettings(value: unknown): TTSSegmentSettings | null { + if (!value || typeof value !== 'object') return null; + const rec = value as Record; + if (typeof rec.ttsProvider !== 'string') return null; + if (typeof rec.ttsModel !== 'string') return null; + if (typeof rec.voice !== 'string') return null; + if (!Number.isFinite(Number(rec.nativeSpeed))) return null; + if (rec.ttsInstructions !== undefined && typeof rec.ttsInstructions !== 'string') return null; + + return { + ttsProvider: rec.ttsProvider, + ttsModel: rec.ttsModel, + voice: rec.voice, + nativeSpeed: Number(rec.nativeSpeed), + ...(typeof rec.ttsInstructions === 'string' ? { ttsInstructions: rec.ttsInstructions } : {}), + }; +} + +function parseSegments(value: unknown): TTSSegmentInput[] | null { + if (!Array.isArray(value) || value.length === 0) return null; + const parsed: TTSSegmentInput[] = []; + for (const item of value) { + if (!item || typeof item !== 'object') return null; + const rec = item as Record; + if (!Number.isInteger(rec.segmentIndex) || Number(rec.segmentIndex) < 0) return null; + if (typeof rec.text !== 'string') return null; + parsed.push({ + segmentIndex: Number(rec.segmentIndex), + text: rec.text, + ...(rec.locator && typeof rec.locator === 'object' ? { locator: rec.locator as TTSSegmentInput['locator'] } : {}), + }); + } + return parsed; +} + +function parseBody(value: unknown): TTSSegmentsEnsureRequest | null { + if (!value || typeof value !== 'object') return null; + const rec = value as Record; + if (typeof rec.documentId !== 'string' || !rec.documentId.trim()) return null; + const settings = parseSettings(rec.settings); + const segments = parseSegments(rec.segments); + if (!settings || !segments) return null; + + return { + documentId: rec.documentId.trim().toLowerCase(), + settings, + segments, + }; +} + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'TTS segments storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function textHmacSecret(): string { + return process.env.AUTH_SECRET?.trim() + || 'openreader-default-tts-segment-secret'; +} + +export async function POST(request: NextRequest) { + let didCreateDeviceIdCookie = false; + let deviceIdToSet: string | null = null; + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const parsed = parseBody(await request.json().catch(() => null)); + if (!parsed) { + return NextResponse.json({ error: 'Invalid request payload' }, { status: 400 }); + } + + const scope = await resolveSegmentDocumentScope(request, parsed.documentId); + if (scope instanceof Response) return scope; + + const settingsHash = buildTtsSegmentSettingsHash(parsed.settings); + const nowMs = Date.now(); + const storagePrefix = getS3Config().prefix; + const secret = textHmacSecret(); + + const normalized = parsed.segments + .map((segment) => { + const text = normalizeSegmentText(segment.text); + if (!text) return null; + const locator = normalizeLocator(segment.locator); + const locatorHash = locatorFingerprint(locator); + const segmentId = buildTtsSegmentId({ + documentId: parsed.documentId, + documentVersion: scope.documentVersion, + settingsHash, + segmentIndex: segment.segmentIndex, + normalizedText: text, + locatorFingerprint: locatorHash, + }); + + return { + original: segment, + text, + locator, + segmentId, + textHash: buildTtsSegmentTextHash(text, secret), + }; + }) + .filter((value): value is NonNullable => Boolean(value)); + + if (normalized.length === 0) { + return NextResponse.json({ error: 'No valid non-empty segments provided' }, { status: 400 }); + } + + const ids = normalized.map((segment) => segment.segmentId); + const rows = (await db + .select() + .from(ttsSegments) + .where(and(eq(ttsSegments.userId, scope.storageUserId), inArray(ttsSegments.segmentId, ids)))) as Array<{ + segmentId: string; + userId: string; + documentId: string; + readerType: string; + documentVersion: number; + segmentIndex: number; + locatorJson: string | null; + settingsHash: string; + textHash: string; + textLength: number; + audioKey: string | null; + audioFormat: string; + durationMs: number | null; + alignmentJson: string | null; + status: string; + error: string | null; + createdAt: number | null; + updatedAt: number | null; + }>; + + const existingById = new Map(rows.map((row) => [row.segmentId, row])); + const manifest: TTSSegmentManifestItem[] = []; + + for (const segment of normalized) { + const existing = existingById.get(segment.segmentId); + + if (existing?.status === 'completed' && existing.audioKey) { + let alignment = existing.alignmentJson + ? (JSON.parse(existing.alignmentJson) as TTSSegmentManifestItem['alignment']) + : null; + const locator = existing.locatorJson + ? (JSON.parse(existing.locatorJson) as TTSSegmentManifestItem['locator']) + : null; + + // Self-heal transient Whisper failures: if audio exists but alignment was + // previously unavailable, retry alignment using the current segment text. + if (!alignment) { + try { + const audioBuffer = await getTtsSegmentAudioObject(existing.audioKey); + const whisperBytes = Uint8Array.from(audioBuffer); + const aligned = await alignAudioWithText( + whisperBytes.buffer, + segment.text, + undefined, + { engine: 'whisper.cpp' }, + ); + alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null; + + if (alignment) { + await db + .update(ttsSegments) + .set({ + alignmentJson: JSON.stringify(alignment), + updatedAt: Date.now(), + }) + .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); + } + } catch (alignError) { + console.warn('Whisper alignment still unavailable for completed segment; continuing without word highlights.', { + segmentId: segment.segmentId, + error: alignError instanceof Error ? alignError.message : String(alignError), + }); + alignment = null; + } + } + + manifest.push({ + segmentId: segment.segmentId, + segmentIndex: existing.segmentIndex, + audioUrl: `/api/tts/segments/audio?documentId=${encodeURIComponent(parsed.documentId)}&segmentId=${encodeURIComponent(segment.segmentId)}`, + durationMs: existing.durationMs ?? 0, + alignment, + locator, + status: 'completed', + }); + continue; + } + + const audioKey = existing?.audioKey || buildTtsSegmentAudioKey({ + storagePrefix, + namespace: scope.testNamespace, + userId: scope.storageUserId, + documentId: parsed.documentId, + documentVersion: scope.documentVersion, + settingsHash, + segmentId: segment.segmentId, + }); + + await db + .insert(ttsSegments) + .values({ + segmentId: segment.segmentId, + userId: scope.storageUserId, + documentId: parsed.documentId, + readerType: scope.readerType, + documentVersion: scope.documentVersion, + segmentIndex: segment.original.segmentIndex, + locatorJson: segment.locator ? JSON.stringify(segment.locator) : null, + settingsHash, + textHash: segment.textHash, + textLength: segment.text.length, + audioKey, + audioFormat: 'mp3', + status: 'pending', + error: null, + updatedAt: nowMs, + }) + .onConflictDoUpdate({ + target: [ttsSegments.segmentId, ttsSegments.userId], + set: { + documentId: parsed.documentId, + readerType: scope.readerType, + documentVersion: scope.documentVersion, + segmentIndex: segment.original.segmentIndex, + locatorJson: segment.locator ? JSON.stringify(segment.locator) : null, + settingsHash, + textHash: segment.textHash, + textLength: segment.text.length, + audioKey, + audioFormat: 'mp3', + status: 'pending', + error: null, + updatedAt: nowMs, + }, + }); + + try { + if (scope.authEnabled && scope.userId && isTtsRateLimitEnabled()) { + const charCount = segment.text.length; + const ip = getClientIp(request); + const device = scope.isAnonymousUser ? getOrCreateDeviceId(request) : null; + if (device?.didCreate) { + didCreateDeviceIdCookie = true; + deviceIdToSet = device.deviceId; + } + + const rateLimitResult = await rateLimiter.checkAndIncrementLimit( + { id: scope.userId, isAnonymous: scope.isAnonymousUser }, + charCount, + { + deviceId: device?.deviceId ?? null, + ip, + }, + ); + + if (!rateLimitResult.allowed) { + const resetTimeMs = rateLimitResult.resetTimeMs; + const retryAfterSeconds = Math.max(0, Math.ceil((resetTimeMs - Date.now()) / 1000)); + const response = new NextResponse(JSON.stringify({ + type: 'https://openreader.app/problems/daily-quota-exceeded', + title: 'Daily quota exceeded', + status: 429, + detail: 'Daily character limit exceeded', + code: 'USER_DAILY_QUOTA_EXCEEDED', + currentCount: rateLimitResult.currentCount, + limit: rateLimitResult.limit, + remainingChars: rateLimitResult.remainingChars, + resetTimeMs, + userType: scope.isAnonymousUser ? 'anonymous' : 'authenticated', + upgradeHint: scope.isAnonymousUser + ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + : undefined, + instance: request.nextUrl.pathname, + }), { + status: 429, + headers: { + 'Content-Type': 'application/problem+json', + 'Retry-After': String(retryAfterSeconds), + }, + }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } + } + + const ttsBuffer = await generateTTSBuffer({ + text: segment.text, + voice: parsed.settings.voice, + speed: parsed.settings.nativeSpeed, + format: 'mp3', + model: parsed.settings.ttsModel, + instructions: parsed.settings.ttsInstructions, + provider: parsed.settings.ttsProvider, + apiKey: request.headers.get('x-openai-key') || process.env.API_KEY || 'none', + baseUrl: request.headers.get('x-openai-base-url') || process.env.API_BASE, + testNamespace: scope.testNamespace, + }, request.signal); + + await putTtsSegmentAudioObject(audioKey, ttsBuffer); + + let persistedBuffer = ttsBuffer; + if (persistedBuffer.byteLength === 0) { + persistedBuffer = await getTtsSegmentAudioObject(audioKey); + } + + const durationMs = await probeAudioDurationMsFromBuffer(persistedBuffer, request.signal); + let alignment: TTSSegmentManifestItem['alignment'] = null; + try { + const whisperBytes = Uint8Array.from(persistedBuffer); + const aligned = await alignAudioWithText( + whisperBytes.buffer, + segment.text, + undefined, + { engine: 'whisper.cpp' }, + ); + alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null; + } catch (alignError) { + console.warn('Whisper alignment unavailable for segment; continuing without word highlights.', { + segmentId: segment.segmentId, + error: alignError instanceof Error ? alignError.message : String(alignError), + }); + alignment = null; + } + + await db + .update(ttsSegments) + .set({ + durationMs, + alignmentJson: alignment ? JSON.stringify(alignment) : null, + status: 'completed', + error: null, + updatedAt: Date.now(), + }) + .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); + + manifest.push({ + segmentId: segment.segmentId, + segmentIndex: segment.original.segmentIndex, + audioUrl: `/api/tts/segments/audio?documentId=${encodeURIComponent(parsed.documentId)}&segmentId=${encodeURIComponent(segment.segmentId)}`, + durationMs, + alignment, + locator: segment.locator, + status: 'completed', + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to generate segment'; + await db + .update(ttsSegments) + .set({ + status: 'error', + error: message, + updatedAt: Date.now(), + }) + .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); + + manifest.push({ + segmentId: segment.segmentId, + segmentIndex: segment.original.segmentIndex, + audioUrl: '', + durationMs: 0, + alignment: null, + locator: segment.locator, + status: 'error', + }); + } + } + + const response = NextResponse.json({ + documentId: parsed.documentId, + documentVersion: scope.documentVersion, + settingsHash, + segments: manifest, + }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } catch (error) { + console.error('Error ensuring TTS segments:', error); + const response = NextResponse.json({ error: 'Failed to ensure TTS segments' }, { status: 500 }); + attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); + return response; + } +} diff --git a/src/app/api/tts/segments/manifest/route.ts b/src/app/api/tts/segments/manifest/route.ts new file mode 100644 index 0000000..ef01812 --- /dev/null +++ b/src/app/api/tts/segments/manifest/route.ts @@ -0,0 +1,81 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, asc, eq, gte, lte } from 'drizzle-orm'; +import { db } from '@/db'; +import { ttsSegments } from '@/db/schema'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; +import type { TTSSegmentManifestItem } from '@/types/client'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'TTS segments storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function parseIntOr(value: string | null, fallback: number): number { + const n = Number.parseInt(value || '', 10); + return Number.isFinite(n) ? n : fallback; +} + +export async function GET(request: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const documentId = (request.nextUrl.searchParams.get('documentId') || '').trim().toLowerCase(); + const settingsHash = (request.nextUrl.searchParams.get('settingsHash') || '').trim(); + if (!documentId || !settingsHash) { + return NextResponse.json({ error: 'Missing documentId or settingsHash' }, { status: 400 }); + } + + const scope = await resolveSegmentDocumentScope(request, documentId); + if (scope instanceof Response) return scope; + + const fromIndex = Math.max(0, parseIntOr(request.nextUrl.searchParams.get('fromIndex'), 0)); + const toIndex = Math.max(fromIndex, parseIntOr(request.nextUrl.searchParams.get('toIndex'), fromIndex + 8)); + + const rows = (await db + .select() + .from(ttsSegments) + .where(and( + eq(ttsSegments.userId, scope.storageUserId), + eq(ttsSegments.documentId, documentId), + eq(ttsSegments.documentVersion, scope.documentVersion), + eq(ttsSegments.settingsHash, settingsHash), + gte(ttsSegments.segmentIndex, fromIndex), + lte(ttsSegments.segmentIndex, toIndex), + )) + .orderBy(asc(ttsSegments.segmentIndex))) as Array<{ + segmentId: string; + segmentIndex: number; + locatorJson: string | null; + durationMs: number | null; + alignmentJson: string | null; + status: string; + }>; + + const segments: TTSSegmentManifestItem[] = rows.map((row) => ({ + segmentId: row.segmentId, + segmentIndex: row.segmentIndex, + audioUrl: row.status === 'completed' + ? `/api/tts/segments/audio?documentId=${encodeURIComponent(documentId)}&segmentId=${encodeURIComponent(row.segmentId)}` + : '', + durationMs: row.durationMs ?? 0, + alignment: row.alignmentJson ? JSON.parse(row.alignmentJson) : null, + locator: row.locatorJson ? JSON.parse(row.locatorJson) : null, + status: row.status === 'error' ? 'error' : row.status === 'completed' ? 'completed' : 'pending', + })); + + return NextResponse.json({ + documentId, + documentVersion: scope.documentVersion, + settingsHash, + segments, + }); + } catch (error) { + console.error('Error fetching TTS segment manifest:', error); + return NextResponse.json({ error: 'Failed to fetch TTS segment manifest' }, { status: 500 }); + } +} diff --git a/src/app/api/whisper/route.ts b/src/app/api/whisper/route.ts index d43f4b4..4c73ff3 100644 --- a/src/app/api/whisper/route.ts +++ b/src/app/api/whisper/route.ts @@ -1,424 +1,16 @@ import { NextRequest, NextResponse } from 'next/server'; -import { createHash, randomUUID } from 'crypto'; -import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; -import { tmpdir } from 'os'; -import { join } from 'path'; -import { spawn } from 'child_process'; -import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts'; -import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import type { TTSSentenceAlignment } from '@/types/tts'; import { auth } from '@/lib/server/auth/auth'; -import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin'; +import { + alignAudioWithText, + makeWhisperCacheKey, + type WhisperRequestBody, +} from '@/lib/server/whisper/alignment'; export const runtime = 'nodejs'; -interface WhisperRequestBody { - text: string; - audio: TTSAudioBytes; // raw bytes from Uint8Array - lang?: string; -} - -interface WhisperAlignmentOptions { - engine?: 'whisper.cpp'; - lang?: string; -} - -interface WhisperWord { - start: number; - end: number; - word: string; -} - -// Simple in-memory cache keyed by a hash of text+lang+audio length -const alignmentCache = new Map(); - -// Model management using a fixed tiny.en GGML model stored under docstore/model -const MODEL_NAME = 'ggml-tiny.en.bin'; -const MODEL_URL = - 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin'; -const DOCSTORE_DIR = join(process.cwd(), 'docstore'); -const MODEL_DIR = join(DOCSTORE_DIR, 'model'); -const MODEL_PATH = join(MODEL_DIR, MODEL_NAME); -const modelReadyPromises = new Map>(); - -async function ensureModelAvailable(): Promise { - // Fast path: model already present - try { - await access(MODEL_PATH); - return; - } catch { - // fall through to download - } - - const existing = modelReadyPromises.get(MODEL_PATH); - if (existing) return existing; - - const promise = (async () => { - try { - await access(MODEL_PATH); - return; - } catch { - // still missing - } - - await mkdir(MODEL_DIR, { recursive: true }); - - const res = await fetch(MODEL_URL); - if (!res.ok) { - throw new Error( - `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}` - ); - } - - const arrayBuffer = await res.arrayBuffer(); - await writeFile(MODEL_PATH, Buffer.from(arrayBuffer)); - })(); - - modelReadyPromises.set(MODEL_PATH, promise); - return promise; -} - -async function runWhisperCpp( - wavPath: string, - opts: WhisperAlignmentOptions -): Promise { - const binary = process.env.WHISPER_CPP_BIN; - if (!binary) { - throw new Error( - 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.' - ); - } - - await ensureModelAvailable(); - - return new Promise((resolve, reject) => { - const jsonBase = `${wavPath}.json_out`; - const jsonPath = `${jsonBase}.json`; - // Request full JSON output (including token-level details when available) - // and ask whisper-cli not to print anything to stdout. - const args = [ - '-m', - MODEL_PATH, - '-f', - wavPath, - '-of', - jsonBase, - '-ojf', - '-np', - ]; - - if (opts.lang) { - args.push('-l', opts.lang); - } - - const child = spawn(binary, args); - - let stdout = ''; - let stderr = ''; - - child.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - child.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - child.on('error', (err) => { - reject(err); - }); - - child.on('close', (code) => { - if (code !== 0) { - console.error('whisper-cli exited with error', { - code, - wavPath, - stdout: stdout.slice(0, 500), - stderr: stderr.slice(0, 500), - }); - return reject( - new Error( - `whisper.cpp exited with code ${code}: ${stderr || stdout}` - ) - ); - } - - readFile(jsonPath, 'utf-8') - .then((content: string) => { - const words: WhisperWord[] = []; - const parsed = JSON.parse(content) as { - transcription?: Array<{ - text?: string; - timestamps?: { from?: string; to?: string }; - offsets?: { from?: number; to?: number }; - tokens?: Array<{ - text?: string; - timestamps?: { from?: string; to?: string }; - offsets?: { from?: number; to?: number }; - }>; - }>; - }; - - const transcription = parsed.transcription; - - const parseTimecode = (value?: string): number | null => { - if (!value) return null; - const m = value.match(/(\d+):(\d+):(\d+),(\d+)/); - if (!m) return null; - const h = Number(m[1]); - const min = Number(m[2]); - const s = Number(m[3]); - const ms = Number(m[4]); - if ( - Number.isNaN(h) || - Number.isNaN(min) || - Number.isNaN(s) || - Number.isNaN(ms) - ) { - return null; - } - return h * 3600 + min * 60 + s + ms / 1000; - }; - - if (Array.isArray(transcription)) { - for (const seg of transcription) { - const segText = (seg.text || '').trim(); - const segStartSecFromTs = parseTimecode( - seg.timestamps?.from - ); - const segEndSecFromTs = parseTimecode(seg.timestamps?.to); - const segStartSecFromMs = - typeof seg.offsets?.from === 'number' - ? seg.offsets.from / 1000 - : null; - const segEndSecFromMs = - typeof seg.offsets?.to === 'number' - ? seg.offsets.to / 1000 - : null; - - const segStartSec = - segStartSecFromTs ?? - segStartSecFromMs ?? - 0; - const segEndSec = - segEndSecFromTs ?? - segEndSecFromMs ?? - segStartSec; - - const tokens = Array.isArray(seg.tokens) - ? seg.tokens - : []; - - if (tokens.length > 0) { - for (const token of tokens) { - const rawText = token.text || ''; - const tokenText = rawText.trim(); - // Skip special markers like [_BEG_] - if (!tokenText || /^\[.*\]$/.test(tokenText)) continue; - - const tokStartSecFromTs = parseTimecode( - token.timestamps?.from - ); - const tokEndSecFromTs = parseTimecode( - token.timestamps?.to - ); - const tokStartSecFromMs = - typeof token.offsets?.from === 'number' - ? token.offsets.from / 1000 - : null; - const tokEndSecFromMs = - typeof token.offsets?.to === 'number' - ? token.offsets.to / 1000 - : null; - - const startSec = - tokStartSecFromTs ?? - tokStartSecFromMs ?? - segStartSec; - const endSec = - tokEndSecFromTs ?? - tokEndSecFromMs ?? - segEndSec; - - words.push({ - word: tokenText, - start: startSec, - end: endSec, - }); - } - } else if (segText) { - // Fallback: no token list, approximate per-word timing within the segment - const segTokens = segText.split(/\s+/).filter(Boolean); - if (segTokens.length) { - const totalDur = Math.max(segEndSec - segStartSec, 0); - const step = - segTokens.length > 0 - ? totalDur / segTokens.length - : 0; - segTokens.forEach((token, index) => { - const wStart = - step > 0 - ? segStartSec + step * index - : segStartSec; - const wEnd = - step > 0 - ? index === segTokens.length - 1 - ? segEndSec - : segStartSec + step * (index + 1) - : segEndSec; - words.push({ - word: token, - start: wStart, - end: wEnd, - }); - }); - } - } - } - } - - resolve(words); - }) - .catch((err: unknown) => { - reject(err); - }); - }); - }); -} - -function mapWordsToSentenceOffsets( - sentence: string, - words: WhisperWord[] -): TTSSentenceAlignment { - const normalizedSentence = preprocessSentenceForAudio(sentence); - let cursor = 0; - - const alignedWords = words.map((w) => { - const token = w.word.trim(); - if (!token) { - return { - text: '', - startSec: w.start, - endSec: w.end, - charStart: cursor, - charEnd: cursor, - }; - } - - const idx = normalizedSentence - .toLowerCase() - .indexOf(token.toLowerCase(), cursor); - - const start = - idx !== -1 - ? idx - : cursor; - const end = start + token.length; - - cursor = end; - - return { - text: token, - startSec: w.start, - endSec: w.end, - charStart: start, - charEnd: end, - }; - }); - - return { - sentence, - sentenceIndex: 0, - words: alignedWords.filter((w) => w.text.length > 0), - }; -} - -async function alignAudioWithText( - audioBuffer: TTSAudioBuffer, - text: string, - cacheKey?: string, - opts: WhisperAlignmentOptions = {} -): Promise { - if (!text.trim()) { - return []; - } - - if (cacheKey && alignmentCache.has(cacheKey)) { - return alignmentCache.get(cacheKey)!; - } - - const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); - const inputPath = join(tmpBase, `${randomUUID()}-input.bin`); - const wavPath = join(tmpBase, `${randomUUID()}-input.wav`); - - try { - await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); - - await new Promise((resolve, reject) => { - const ffmpeg = spawn(getFFmpegPath(), [ - '-y', - '-i', - inputPath, - '-ar', - '16000', - '-ac', - '1', - wavPath, - ]); - - let stderr = ''; - ffmpeg.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - ffmpeg.on('error', (err) => { - reject(err); - }); - - ffmpeg.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - reject( - new Error(`ffmpeg failed with code ${code}: ${stderr}`) - ); - } - }); - }); - - const words = await runWhisperCpp(wavPath, opts); - const alignment = mapWordsToSentenceOffsets(text, words); - const result: TTSSentenceAlignment[] = [alignment]; - - if (cacheKey) { - alignmentCache.set(cacheKey, result); - } - - return result; - } finally { - try { - await rm(tmpBase, { recursive: true, force: true }); - } catch { - // ignore - } - } -} - -function makeCacheKey(input: WhisperRequestBody) { - const hash = createHash('sha256') - .update( - JSON.stringify({ - text: input.text, - lang: input.lang || '', - audioLen: input.audio?.length || 0, - }) - ) - .digest('hex'); - return hash; -} - export async function POST(req: NextRequest) { try { - // Auth check - require session const session = await auth?.api.getSession({ headers: req.headers }); if (auth && !session?.user) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); @@ -434,7 +26,7 @@ export async function POST(req: NextRequest) { ); } - const cacheKey = makeCacheKey(body); + const cacheKey = makeWhisperCacheKey(body); const audioBuffer = new Uint8Array(audio).buffer; const alignments: TTSSentenceAlignment[] = await alignAudioWithText( diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index e771de2..e5cb55c 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -36,7 +36,7 @@ import { useMediaSession } from '@/hooks/audio/useMediaSession'; import { useAudioContext } from '@/hooks/audio/useAudioContext'; import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/client/dexie'; import { getDocumentProgress, scheduleDocumentProgressSync } from '@/lib/client/api/user-state'; -import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks'; +import { withRetry, ensureTtsSegments } from '@/lib/client/api/audiobooks'; import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp'; import { isKokoroModel } from '@/lib/shared/kokoro'; import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog'; @@ -50,9 +50,9 @@ import type { TTSAudioBuffer, } from '@/types/tts'; import type { - TTSRequestPayload, TTSRequestHeaders, TTSRetryOptions, + TTSSegmentManifestItem, } from '@/types/client'; import type { ReaderType } from '@/types/user-state'; @@ -105,6 +105,7 @@ const MAX_CONTINUATION_CARRY_CHARS = 220; const MAX_CONTINUATION_CARRY_WORDS = 40; const LOOP_GUARD_MIN_INDEX = 2; const LOOP_GUARD_MIN_PROGRESS = 0.6; +const AUDIO_CACHE_MAX_ITEMS = 25; const SENTENCE_ENDING = /[.?!…]["'”’)\]]*\s*$/; const wordHighlightFeatureEnabled = process.env.NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT?.toLowerCase() !== 'false'; @@ -322,7 +323,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Audio and voice management hooks const audioContext = useAudioContext(); - const audioCache = useAudioCache(25); + const audioCache = useAudioCache(AUDIO_CACHE_MAX_ITEMS); const { availableVoices, fetchVoices } = useVoiceManagement(openApiKey, openApiBaseUrl, configTTSProvider, configTTSModel); const { authEnabled, @@ -360,6 +361,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Get document ID from URL params const { id } = useParams(); const pathname = usePathname(); + const documentId = useMemo(() => { + if (typeof id === 'string') return id; + if (Array.isArray(id)) return id[0]; + return ''; + }, [id]); const currentReaderType: ReaderType = useMemo(() => { if (pathname.startsWith('/epub/')) return 'epub'; @@ -406,11 +412,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const restartSeqRef = useRef(0); // Track continuation slices for PDF/EPUB page transitions const continuationCarryRef = useRef>(new Map()); + // Preserve autoplay intent across location changes. Some browsers can emit pause + // events while we stop/unload between pages, which momentarily flips `isPlaying` + // false and can prevent automatic resume on the next page. + const resumeAfterLocationChangeRef = useRef(false); const epubContinuationRef = useRef(null); const pageTurnEstimateRef = useRef(null); const pageTurnTimeoutRef = useRef | null>(null); const pageFirstBlockFingerprintRef = useRef>(new Map()); const sentenceAlignmentCacheRef = useRef>(new Map()); + const segmentManifestCacheRef = useRef>(new Map()); const [currentSentenceAlignment, setCurrentSentenceAlignment] = useState(); const [currentWordIndex, setCurrentWordIndex] = useState(null); const isPlayingRef = useRef(false); @@ -509,6 +520,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } }, [audioContext]); + const setSegmentManifestCache = useCallback((key: string, item: TTSSegmentManifestItem) => { + const cache = segmentManifestCacheRef.current; + if (cache.has(key)) { + cache.delete(key); + } + cache.set(key, item); + while (cache.size > AUDIO_CACHE_MAX_ITEMS) { + const oldestKey = cache.keys().next().value; + if (typeof oldestKey !== 'string') break; + cache.delete(oldestKey); + } + }, []); + const isAutoplayBlockedError = useCallback((err: unknown) => { const msg = (() => { if (typeof err === 'string') return err; @@ -620,6 +644,12 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {boolean} shouldPause - Whether to pause playback */ const skipToLocation = useCallback((location: TTSLocation, shouldPause = false) => { + if (shouldPause) { + resumeAfterLocationChangeRef.current = false; + } else if (isPlayingRef.current) { + resumeAfterLocationChangeRef.current = true; + } + // Reset state for new content in correct order abortAudio(); if (shouldPause) setIsPlaying(false); @@ -836,9 +866,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement if (handleBlankSection(workingText)) return; const shouldPause = normalizedOptions.shouldPause ?? false; + const pendingAutoResume = resumeAfterLocationChangeRef.current; + const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume); + if (shouldPause || pendingAutoResume) { + resumeAfterLocationChangeRef.current = false; + } // Keep track of previous state and pause playback - const wasPlaying = isPlaying; setIsPlaying(false); abortAudio(true); // Clear pending requests since text is changing setIsProcessing(true); // Set processing state before text processing starts @@ -917,7 +951,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setIsProcessing(false); // Restore playback state if needed - if (!shouldPause && wasPlaying) { + if (shouldResumePlayback) { setIsPlaying(true); } }) @@ -1053,54 +1087,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {string} sentence - The sentence to generate audio for * @returns {Promise} The generated audio buffer */ - const getAudio = useCallback(async (sentence: string, preload = false): Promise => { + const getAudio = useCallback(async ( + sentence: string, + sentenceIndex: number, + preload = false, + ): Promise => { const alignmentEnabledForCurrentDoc = wordHighlightFeatureEnabled && ((!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) || (isEPUB && epubHighlightEnabled && epubWordHighlightEnabled)); - // Helper to ensure we have an alignment for a given - // sentence/audio pair, even when the audio itself is - // served from the local cache. - const ensureAlignment = (arrayBuffer: TTSAudioBuffer) => { - if (!alignmentEnabledForCurrentDoc) return; - const alignmentKey = buildCacheKey( - sentence, - voice, - effectiveNativeSpeed, - configTTSProvider, - ttsModel, - ); - if (sentenceAlignmentCacheRef.current.has(alignmentKey)) return; - - try { - const audioBytes = Array.from(new Uint8Array(arrayBuffer)); - const alignmentBody = { - text: sentence, - audio: audioBytes, - }; - - void alignAudio(alignmentBody) - .then(async (data) => { - if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) { - return; - } - const alignment = data.alignments[0] as TTSSentenceAlignment; - sentenceAlignmentCacheRef.current.set(alignmentKey, alignment); - - const currentSentence = sentencesRef.current[currentIndexRef.current]; - if (currentSentence === sentence) { - setCurrentSentenceAlignment(alignment); - setCurrentWordIndex(null); - } - }) - .catch((err) => { - console.warn('Alignment request failed:', err); - }); - } catch (err) { - console.warn('Failed to start alignment request:', err); - } - }; - const audioCacheKey = buildCacheKey( sentence, voice, @@ -1109,83 +1104,94 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, ); - // Check if the audio is already cached const cachedAudio = audioCache.get(audioCacheKey); if (cachedAudio) { - console.log('Using cached audio for sentence:', sentence.substring(0, 20)); - // If we have audio but no alignment (e.g. after a - // navigation or TTS reset), kick off a fresh alignment - // request using the cached audio buffer. - ensureAlignment(cachedAudio); + const cachedManifest = segmentManifestCacheRef.current.get(audioCacheKey); + if (alignmentEnabledForCurrentDoc && cachedManifest?.alignment) { + sentenceAlignmentCacheRef.current.set(audioCacheKey, cachedManifest.alignment); + } return cachedAudio; } - // If the user is already out of quota, avoid spamming requests. - // Cached audio above is still allowed to play. - if (isAtLimit) { - if (!preload) { - setIsPlaying(false); - } + if (!documentId) { + if (!preload) setIsPlaying(false); return undefined; } + if (isAtLimit) { + if (!preload) setIsPlaying(false); + return undefined; + } + + const controller = new AbortController(); + activeAbortControllers.current.add(controller); + + const reqHeaders: TTSRequestHeaders = { + 'Content-Type': 'application/json', + 'x-openai-key': openApiKey || '', + 'x-tts-provider': configTTSProvider, + }; + if (openApiBaseUrl) { + reqHeaders['x-openai-base-url'] = openApiBaseUrl; + } + + const locator = isEPUB + ? { location: String(currDocPage), readerType: currentReaderType } + : { page: Number(currDocPageNumber || 1), readerType: currentReaderType }; + + const retryOptions: TTSRetryOptions = { + maxRetries: 2, + initialDelay: 300, + maxDelay: 300, + }; + try { - console.log('Requesting audio for sentence:', sentence); - - // Create an AbortController for this request - const controller = new AbortController(); - activeAbortControllers.current.add(controller); - - const reqHeaders: TTSRequestHeaders = { - 'Content-Type': 'application/json', - 'x-openai-key': openApiKey || '', - 'x-tts-provider': configTTSProvider, - }; - if (openApiBaseUrl) { - reqHeaders['x-openai-base-url'] = openApiBaseUrl; - } - - const reqBody: TTSRequestPayload = { - text: sentence, - voice, - speed: effectiveNativeSpeed, - format: 'mp3', - model: ttsModel, - instructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined, - }; - - // Allow one narrow client retry for transient browser->/api/tts transport failures. - // HTTP failures are not retried client-side. - const retryOptions: TTSRetryOptions = { - maxRetries: 2, - initialDelay: 300, - maxDelay: 300, - }; - onTTSStart(); - let arrayBuffer: TTSAudioBuffer; - try { - arrayBuffer = await withRetry( - async () => { - return await generateTTS(reqBody, reqHeaders, controller.signal); + const ensured = await withRetry( + async () => ensureTtsSegments({ + documentId, + settings: { + ttsProvider: configTTSProvider, + ttsModel, + voice, + nativeSpeed: effectiveNativeSpeed, + ...(supportsTtsInstructions(ttsModel) && ttsInstructions ? { ttsInstructions } : {}), }, - retryOptions - ); - } finally { - onTTSComplete(); + segments: [ + { + segmentIndex: sentenceIndex, + text: sentence, + locator, + }, + ], + }, reqHeaders, controller.signal), + retryOptions, + ); + + const segment = ensured.segments[0]; + if (!segment || segment.status !== 'completed' || !segment.audioUrl) { + throw new Error('Failed to prepare segment audio'); } - // Remove the controller once the request is complete - activeAbortControllers.current.delete(controller); + const response = await fetch(segment.audioUrl, { signal: controller.signal }); + if (!response.ok) { + throw new Error(`Segment audio fetch failed: ${response.status}`); + } + const arrayBuffer = await response.arrayBuffer(); + if (arrayBuffer.byteLength === 0) { + throw new Error('Received empty segment audio buffer'); + } - // Cache the array buffer audioCache.set(audioCacheKey, arrayBuffer); - - // Fire-and-forget alignment request; do not block audio playback - ensureAlignment(arrayBuffer); - + setSegmentManifestCache(audioCacheKey, segment); + if (alignmentEnabledForCurrentDoc && segment.alignment) { + sentenceAlignmentCacheRef.current.set(audioCacheKey, segment.alignment); + } return arrayBuffer; } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + return undefined; + } const status = (() => { if (typeof error === 'object' && error !== null && 'status' in error) { const maybe = (error as { status?: unknown }).status; @@ -1193,7 +1199,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } return undefined; })(); - const code = (() => { if (typeof error === 'object' && error !== null && 'code' in error) { const maybe = (error as { code?: unknown }).code; @@ -1201,23 +1206,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } return undefined; })(); - - // Check if this was an abort error - if (error instanceof Error && error.name === 'AbortError') { - console.log('TTS request aborted:', sentence.substring(0, 20)); - return; - } - - // If a preload request fails, we should not flip the global playback state. - // Otherwise the UI can lose the pause button while the current sentence - // continues playing. - if (!preload) { - setIsPlaying(false); - } - - // Handle daily quota exceeded (429 + Problem Details code) if (status === 429 && code === 'USER_DAILY_QUOTA_EXCEEDED') { - // Avoid noisy toasts from background preloading; keep the user-facing error for active playback. if (!preload) { toast.error('Daily TTS limit reached.', { id: 'tts-limit-error', @@ -1226,18 +1215,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } triggerRateLimit(); refreshRateLimit().catch(console.error); - // Do NOT re-throw, just return undefined to stop playback gracefully return undefined; } - - // Avoid noisy toasts from background preloading. if (!preload) { + setIsPlaying(false); toast.error('TTS failed. Skipped sentence and paused.', { id: 'tts-api-error', duration: 7000, }); } throw error; + } finally { + activeAbortControllers.current.delete(controller); + onTTSComplete(); } }, [ voice, @@ -1253,11 +1243,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement pdfWordHighlightEnabled, epubHighlightEnabled, epubWordHighlightEnabled, - triggerRateLimit, - refreshRateLimit, onTTSComplete, onTTSStart, - isAtLimit + isAtLimit, + documentId, + currDocPage, + currDocPageNumber, + currentReaderType, + refreshRateLimit, + triggerRateLimit, + setSegmentManifestCache, ]); /** @@ -1267,17 +1262,22 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {boolean} [preload=false] - Whether this is a preload request * @returns {Promise} The URL of the processed audio */ - const processSentence = useCallback(async (sentence: string, preload = false): Promise => { + const processSentence = useCallback(async ( + sentence: string, + sentenceIndex: number, + preload = false, + ): Promise => { if (!audioContext) throw new Error('Audio context not initialized'); + const requestKey = `${sentenceIndex}::${sentence}`; // Check if there's a pending preload request for this sentence - const pendingRequest = preloadRequests.current.get(sentence); + const pendingRequest = preloadRequests.current.get(requestKey); if (pendingRequest) { console.log('Using pending preload request for:', sentence.substring(0, 20)); setIsProcessing(true); // Show processing state when using pending request // If this is not a preload request, remove it from the pending map if (!preload) { - preloadRequests.current.delete(sentence); + preloadRequests.current.delete(requestKey); } return pendingRequest; } @@ -1288,7 +1288,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Create the audio processing promise const processPromise = (async () => { try { - const audioBuffer = await getAudio(sentence, preload); + const audioBuffer = await getAudio(sentence, sentenceIndex, preload); if (!audioBuffer) { // If quota or other handled error returns undefined, ensure we don't throw "No audio data" // Just return empty string to signal graceful failure/skip @@ -1308,11 +1308,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // If this is a preload request, store it in the map if (preload) { - preloadRequests.current.set(sentence, processPromise); + preloadRequests.current.set(requestKey, processPromise); // Clean up the map entry once the promise resolves or rejects void processPromise .finally(() => { - preloadRequests.current.delete(sentence); + preloadRequests.current.delete(requestKey); }) .catch(() => { // Prevent unhandled rejections from the cleanup-only chained promise. @@ -1341,7 +1341,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const createHowl = async (retryCount = 0): Promise => { let playErrorAttempts = 0; // Get the processed audio data URI directly from processSentence - const audioDataUri = await processSentence(sentence); + const audioDataUri = await processSentence(sentence, sentenceIndex); if (!audioDataUri) { // Graceful exit for rate limit / abort / intentionally skipped sentence console.log('Skipping playback for sentence (no audio generated)'); @@ -1698,7 +1698,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement return; } - const pending = preloadRequests.current.get(nextSentence); + const preloadKey = `${sentenceIndex}::${nextSentence}`; + const pending = preloadRequests.current.get(preloadKey); if (pending) { void pending .finally(() => { @@ -1710,7 +1711,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement return; } - void processSentence(nextSentence, true) + void processSentence(nextSentence, sentenceIndex, true) .catch((error) => { const status = (() => { if (typeof error === 'object' && error !== null && 'status' in error) { diff --git a/src/db/schema.ts b/src/db/schema.ts index 429e5a2..975b1e5 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -13,3 +13,4 @@ export const userTtsChars = usePostgres ? postgresSchema.userTtsChars : sqliteSc export const userPreferences = usePostgres ? postgresSchema.userPreferences : sqliteSchema.userPreferences; export const userDocumentProgress = usePostgres ? postgresSchema.userDocumentProgress : sqliteSchema.userDocumentProgress; export const documentPreviews = usePostgres ? postgresSchema.documentPreviews : sqliteSchema.documentPreviews; +export const ttsSegments = usePostgres ? postgresSchema.ttsSegments : sqliteSchema.ttsSegments; diff --git a/src/db/schema_postgres.ts b/src/db/schema_postgres.ts index b91792a..3652dc8 100644 --- a/src/db/schema_postgres.ts +++ b/src/db/schema_postgres.ts @@ -108,3 +108,28 @@ export const documentPreviews = pgTable('document_previews', { primaryKey({ columns: [table.documentId, table.namespace, table.variant] }), index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs), ]); + +export const ttsSegments = pgTable('tts_segments', { + segmentId: text('segment_id').notNull(), + userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), + documentId: text('document_id').notNull(), + readerType: text('reader_type').notNull(), + documentVersion: bigint('document_version', { mode: 'number' }).notNull(), + segmentIndex: integer('segment_index').notNull(), + locatorJson: text('locator_json'), + settingsHash: text('settings_hash').notNull(), + textHash: text('text_hash').notNull(), + textLength: integer('text_length').notNull().default(0), + audioKey: text('audio_key'), + audioFormat: text('audio_format').notNull().default('mp3'), + durationMs: integer('duration_ms'), + alignmentJson: text('alignment_json'), + status: text('status').notNull().default('pending'), + error: text('error'), + createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS), + updatedAt: bigint('updated_at', { mode: 'number' }).default(PG_NOW_MS), +}, (table) => [ + primaryKey({ columns: [table.segmentId, table.userId] }), + index('idx_tts_segments_lookup').on(table.userId, table.documentId, table.documentVersion, table.settingsHash), + index('idx_tts_segments_doc_index').on(table.userId, table.documentId, table.segmentIndex), +]); diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index d133a10..83af6ec 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -108,3 +108,28 @@ export const documentPreviews = sqliteTable('document_previews', { primaryKey({ columns: [table.documentId, table.namespace, table.variant] }), index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs), ]); + +export const ttsSegments = sqliteTable('tts_segments', { + segmentId: text('segment_id').notNull(), + userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), + documentId: text('document_id').notNull(), + readerType: text('reader_type').notNull(), + documentVersion: integer('document_version').notNull(), + segmentIndex: integer('segment_index').notNull(), + locatorJson: text('locator_json'), + settingsHash: text('settings_hash').notNull(), + textHash: text('text_hash').notNull(), + textLength: integer('text_length').notNull().default(0), + audioKey: text('audio_key'), + audioFormat: text('audio_format').notNull().default('mp3'), + durationMs: integer('duration_ms'), + alignmentJson: text('alignment_json'), + status: text('status').notNull().default('pending'), + error: text('error'), + createdAt: integer('created_at').default(SQLITE_NOW_MS), + updatedAt: integer('updated_at').default(SQLITE_NOW_MS), +}, (table) => [ + primaryKey({ columns: [table.segmentId, table.userId] }), + index('idx_tts_segments_lookup').on(table.userId, table.documentId, table.documentVersion, table.settingsHash), + index('idx_tts_segments_doc_index').on(table.userId, table.documentId, table.segmentIndex), +]); diff --git a/src/lib/client/api/audiobooks.ts b/src/lib/client/api/audiobooks.ts index 9705987..2395308 100644 --- a/src/lib/client/api/audiobooks.ts +++ b/src/lib/client/api/audiobooks.ts @@ -7,7 +7,9 @@ import type { CreateChapterPayload, VoicesResponse, AlignmentPayload, - AlignmentResponse + AlignmentResponse, + TTSSegmentsEnsureRequest, + TTSSegmentsEnsureResponse, } from '@/types/client'; import type { TTSAudiobookChapter, TTSAudioBuffer } from '@/types/tts'; @@ -251,3 +253,58 @@ export const alignAudio = async (payload: AlignmentPayload): Promise => { + const response = await fetch('/api/tts/segments/ensure', { + method: 'POST', + headers: headers as HeadersInit, + body: JSON.stringify(payload), + signal, + }); + + if (!response.ok) { + let problem: unknown = undefined; + const contentType = response.headers.get('content-type') || ''; + if (contentType.includes('application/problem+json') || contentType.includes('application/json')) { + problem = await response.json().catch(() => null); + } + + const err = new Error(`TTS segment ensure failed with status ${response.status}`) as TTSRequestError; + err.status = response.status; + if (typeof problem === 'object' && problem !== null) { + const rec = problem as Record; + if (typeof rec.code === 'string') err.code = rec.code; + if (typeof rec.type === 'string') err.type = rec.type; + if (typeof rec.title === 'string') err.title = rec.title; + if (typeof rec.detail === 'string') err.detail = rec.detail; + } + + throw err; + } + + return await response.json(); +}; + +export const fetchTtsSegmentManifest = async (params: { + documentId: string; + settingsHash: string; + fromIndex: number; + toIndex: number; +}): Promise => { + const query = new URLSearchParams({ + documentId: params.documentId, + settingsHash: params.settingsHash, + fromIndex: String(params.fromIndex), + toIndex: String(params.toIndex), + }); + const response = await fetch(`/api/tts/segments/manifest?${query.toString()}`); + if (!response.ok) { + const data = await response.json().catch(() => null) as { error?: string } | null; + throw new Error(data?.error || 'Failed to fetch segment manifest'); + } + return await response.json(); +}; diff --git a/src/lib/server/tts/segments-auth.ts b/src/lib/server/tts/segments-auth.ts new file mode 100644 index 0000000..e68b68b --- /dev/null +++ b/src/lib/server/tts/segments-auth.ts @@ -0,0 +1,64 @@ +import { and, eq, inArray } from 'drizzle-orm'; +import type { NextRequest } from 'next/server'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth/auth'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; +import type { ReaderType } from '@/types/user-state'; + +export type ResolvedSegmentDocumentScope = { + testNamespace: string | null; + storageUserId: string; + authEnabled: boolean; + userId: string | null; + isAnonymousUser: boolean; + documentVersion: number; + readerType: ReaderType; +}; + +function toReaderType(documentType: string): ReaderType { + if (documentType === 'pdf') return 'pdf'; + if (documentType === 'epub') return 'epub'; + return 'html'; +} + +export async function resolveSegmentDocumentScope( + request: NextRequest, + documentId: string, +): Promise { + const ctxOrRes = await requireAuthContext(request); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(request.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const rows = (await db + .select({ + userId: documents.userId, + lastModified: documents.lastModified, + type: documents.type, + }) + .from(documents) + .where(and(eq(documents.id, documentId), inArray(documents.userId, allowedUserIds)))) as Array<{ + userId: string; + lastModified: number; + type: string; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return Response.json({ error: 'Document not found' }, { status: 404 }); + } + + return { + testNamespace, + storageUserId: doc.userId, + authEnabled: ctxOrRes.authEnabled, + userId: ctxOrRes.userId, + isAnonymousUser: Boolean(ctxOrRes.user?.isAnonymous), + documentVersion: Number(doc.lastModified), + readerType: toReaderType(doc.type), + }; +} diff --git a/src/lib/server/tts/segments-blobstore.ts b/src/lib/server/tts/segments-blobstore.ts new file mode 100644 index 0000000..6bb3c8a --- /dev/null +++ b/src/lib/server/tts/segments-blobstore.ts @@ -0,0 +1,104 @@ +import { + DeleteObjectsCommand, + GetObjectCommand, + ListObjectsV2Command, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; + +function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream { + return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function'; +} + +async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) { + chunks.push(chunk); + } else if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk)); + } else { + chunks.push(Buffer.from(chunk as Uint8Array)); + } + } + return Buffer.concat(chunks); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + + if (isNodeReadableStream(body)) { + return streamToBuffer(body); + } + + throw new Error('Unsupported S3 response body type'); +} + +export async function putTtsSegmentAudioObject(key: string, buffer: Buffer): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send(new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: buffer, + ContentType: 'audio/mpeg', + ServerSideEncryption: 'AES256', + })); +} + +export async function getTtsSegmentAudioObject(key: string): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const res = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: key })); + return bodyToBuffer(res.Body); +} + +export async function deleteTtsSegmentPrefix(prefix: string): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const cleanedPrefix = prefix.replace(/^\/+/, ''); + let deleted = 0; + let continuationToken: string | undefined; + + do { + const listRes = await client.send( + new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: cleanedPrefix, + ContinuationToken: continuationToken, + }), + ); + + const keys = (listRes.Contents ?? []) + .map((item) => item.Key) + .filter((value): value is string => typeof value === 'string' && value.length > 0); + + if (keys.length > 0) { + const deleteRes = await client.send( + new DeleteObjectsCommand({ + Bucket: cfg.bucket, + Delete: { + Objects: keys.map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + deleted += deleteRes.Deleted?.length ?? 0; + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + + return deleted; +} diff --git a/src/lib/server/tts/segments.ts b/src/lib/server/tts/segments.ts new file mode 100644 index 0000000..03e2bec --- /dev/null +++ b/src/lib/server/tts/segments.ts @@ -0,0 +1,188 @@ +import { createHash, createHmac } from 'crypto'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import { ffprobeAudio } from '@/lib/server/audiobooks/chapters'; +import type { + TTSSegmentLocator, + TTSSegmentSettings, +} from '@/types/client'; +import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; + +function stableStringify(value: unknown): string { + if (value === null || typeof value !== 'object') { + return JSON.stringify(value); + } + + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item)).join(',')}]`; + } + + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}`; +} + +function settingsCanonical(settings: TTSSegmentSettings): string { + return stableStringify({ + provider: settings.ttsProvider, + model: settings.ttsModel, + voice: settings.voice, + speed: Number.isFinite(Number(settings.nativeSpeed)) ? Number(settings.nativeSpeed) : 1, + instructions: settings.ttsInstructions || '', + format: 'mp3', + }); +} + +export function buildTtsSegmentSettingsHash(settings: TTSSegmentSettings): string { + return createHash('sha256').update(settingsCanonical(settings)).digest('hex'); +} + +export function normalizeSegmentText(text: string): string { + return preprocessSentenceForAudio(text || '').trim(); +} + +export function normalizeLocator(locator: TTSSegmentLocator | undefined): TTSSegmentLocator | null { + if (!locator) return null; + const normalized: TTSSegmentLocator = {}; + if (typeof locator.page === 'number' && Number.isFinite(locator.page)) { + normalized.page = Math.max(1, Math.floor(locator.page)); + } + if (typeof locator.location === 'string' && locator.location.trim()) { + normalized.location = locator.location.trim(); + } + if (locator.readerType === 'pdf' || locator.readerType === 'epub' || locator.readerType === 'html') { + normalized.readerType = locator.readerType; + } + if (Object.keys(normalized).length === 0) return null; + return normalized; +} + +export function locatorFingerprint(locator: TTSSegmentLocator | null): string { + if (!locator) return ''; + return createHash('sha256').update(stableStringify(locator)).digest('hex'); +} + +export function buildTtsSegmentId(input: { + documentId: string; + documentVersion: number; + settingsHash: string; + segmentIndex: number; + normalizedText: string; + locatorFingerprint: string; +}): string { + const canonical = stableStringify({ + d: input.documentId, + v: input.documentVersion, + s: input.settingsHash, + i: input.segmentIndex, + t: input.normalizedText, + l: input.locatorFingerprint, + }); + return createHash('sha256').update(canonical).digest('hex'); +} + +export function buildTtsSegmentTextHash(text: string, secret: string): string { + return createHmac('sha256', secret).update(text).digest('hex'); +} + +export function buildTtsSegmentAudioKey(input: { + storagePrefix: string; + namespace: string | null; + userId: string; + documentId: string; + documentVersion: number; + settingsHash: string; + segmentId: string; +}): string { + const nsSegment = input.namespace ? `ns/${input.namespace}/` : ''; + return `${input.storagePrefix}/tts_segments_v1/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`; +} + +export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise { + let workDir: string | null = null; + try { + workDir = await mkdtemp(join(tmpdir(), 'openreader-tts-segment-')); + const audioPath = join(workDir, 'segment.mp3'); + await writeFile(audioPath, buffer); + const probe = await ffprobeAudio(audioPath, signal); + const sec = Number(probe.durationSec ?? 0); + if (!Number.isFinite(sec) || sec <= 0) { + return 0; + } + return Math.max(0, Math.floor(sec * 1000)); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +function alignWordsToText(sentence: string): Array<{ text: string; charStart: number; charEnd: number }> { + const words = sentence.match(/\S+/g) || []; + const aligned: Array<{ text: string; charStart: number; charEnd: number }> = []; + let cursor = 0; + const lowerSentence = sentence.toLowerCase(); + + for (const token of words) { + const clean = token.trim(); + if (!clean) continue; + const idx = lowerSentence.indexOf(clean.toLowerCase(), cursor); + const start = idx >= 0 ? idx : cursor; + const end = Math.min(sentence.length, start + clean.length); + cursor = Math.max(cursor, end); + aligned.push({ + text: clean, + charStart: start, + charEnd: end, + }); + } + + return aligned; +} + +export function buildProportionalAlignment(input: { + sentence: string; + sentenceIndex: number; + durationMs: number; +}): TTSSentenceAlignment { + const wordsWithOffsets = alignWordsToText(input.sentence); + if (wordsWithOffsets.length === 0 || input.durationMs <= 0) { + return { + sentence: input.sentence, + sentenceIndex: input.sentenceIndex, + words: [], + }; + } + + const weighted = wordsWithOffsets.map((word) => ({ + ...word, + weight: Math.max(1, word.text.replace(/[^a-zA-Z0-9]/g, '').length), + })); + const totalWeight = weighted.reduce((sum, word) => sum + word.weight, 0); + + let consumedMs = 0; + const alignedWords: TTSSentenceWord[] = weighted.map((word, index) => { + const remainingMs = Math.max(0, input.durationMs - consumedMs); + const sliceMs = index === weighted.length - 1 + ? remainingMs + : Math.max(1, Math.round((input.durationMs * word.weight) / Math.max(1, totalWeight))); + + const startMs = consumedMs; + consumedMs += Math.min(sliceMs, remainingMs); + + return { + text: word.text, + startSec: startMs / 1000, + endSec: consumedMs / 1000, + charStart: word.charStart, + charEnd: word.charEnd, + }; + }); + + return { + sentence: input.sentence, + sentenceIndex: input.sentenceIndex, + words: alignedWords, + }; +} diff --git a/src/lib/server/user/data-cleanup.ts b/src/lib/server/user/data-cleanup.ts index 187fc4a..9b74c67 100644 --- a/src/lib/server/user/data-cleanup.ts +++ b/src/lib/server/user/data-cleanup.ts @@ -7,10 +7,12 @@ import { db } from '@/db'; import { documents, audiobooks } from '@/db/schema'; import { eq } from 'drizzle-orm'; -import { isS3Configured } from '@/lib/server/storage/s3'; +import { getS3Config, isS3Configured } from '@/lib/server/storage/s3'; import { deleteDocumentBlob } from '@/lib/server/documents/blobstore'; import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; +import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews'; import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore'; +import { deleteTtsSegmentPrefix } from '@/lib/server/tts/segments-blobstore'; type DocumentRow = { id: string }; type AudiobookRow = { id: string }; @@ -29,7 +31,7 @@ export async function deleteUserStorageData( userId: string, namespace: string | null, ): Promise { - if (!isS3Configured()) return; + const s3Enabled = isS3Configured(); // eslint-disable-next-line @typescript-eslint/no-explicit-any const database = db as any; @@ -42,25 +44,36 @@ export async function deleteUserStorageData( let docsDeleted = 0; for (const doc of userDocs) { - try { - await deleteDocumentBlob(doc.id, namespace); - docsDeleted++; - } catch (error) { - console.error(`[user-data-cleanup] Failed to delete document blob ${doc.id}:`, error); + if (s3Enabled) { + try { + await deleteDocumentBlob(doc.id, namespace); + docsDeleted++; + } catch (error) { + console.error(`[user-data-cleanup] Failed to delete document blob ${doc.id}:`, error); + } + + try { + await deleteDocumentPreviewArtifacts(doc.id, namespace); + } catch (error) { + console.error(`[user-data-cleanup] Failed to delete preview for ${doc.id}:`, error); + } } + // Always clean up DB rows — documentPreviews has no FK cascade on user. try { - await deleteDocumentPreviewArtifacts(doc.id, namespace); + await deleteDocumentPreviewRows(doc.id, namespace); } catch (error) { - console.error(`[user-data-cleanup] Failed to delete preview for ${doc.id}:`, error); + console.error(`[user-data-cleanup] Failed to delete preview rows for ${doc.id}:`, error); } } // --- Audiobooks --- - const userBooks: AudiobookRow[] = await database - .select({ id: audiobooks.id }) - .from(audiobooks) - .where(eq(audiobooks.userId, userId)); + const userBooks: AudiobookRow[] = s3Enabled + ? await database + .select({ id: audiobooks.id }) + .from(audiobooks) + .where(eq(audiobooks.userId, userId)) + : []; let booksDeleted = 0; for (const book of userBooks) { @@ -73,11 +86,25 @@ export async function deleteUserStorageData( } } - if (docsDeleted > 0 || booksDeleted > 0) { + // --- TTS segments --- + let segmentsDeleted = 0; + if (s3Enabled) { + try { + const cfg = getS3Config(); + const nsSegment = namespace ? `ns/${namespace}/` : ''; + const ttsPrefix = `${cfg.prefix}/tts_segments_v1/${nsSegment}users/${encodeURIComponent(userId)}/`; + segmentsDeleted = await deleteTtsSegmentPrefix(ttsPrefix); + } catch (error) { + console.error(`[user-data-cleanup] Failed to delete TTS segment blobs for user ${userId}:`, error); + } + } + + if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) { console.log( `[user-data-cleanup] Cleaned up S3 data for user ${userId}: ` + `${docsDeleted}/${userDocs.length} document(s), ` + - `${booksDeleted}/${userBooks.length} audiobook(s)`, + `${booksDeleted}/${userBooks.length} audiobook(s), ` + + `${segmentsDeleted} tts segment object(s)`, ); } } diff --git a/src/lib/server/whisper/alignment.ts b/src/lib/server/whisper/alignment.ts new file mode 100644 index 0000000..488b3c5 --- /dev/null +++ b/src/lib/server/whisper/alignment.ts @@ -0,0 +1,395 @@ +import { createHash, randomUUID } from 'crypto'; +import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { spawn } from 'child_process'; +import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts'; +import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin'; + +interface WhisperAlignmentOptions { + engine?: 'whisper.cpp'; + lang?: string; +} + +interface WhisperWord { + start: number; + end: number; + word: string; +} + +export interface WhisperRequestBody { + text: string; + audio: TTSAudioBytes; + lang?: string; +} + +const alignmentCache = new Map(); + +const MODEL_NAME = 'ggml-tiny.en.bin'; +const MODEL_URL = + 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin'; +const DOCSTORE_DIR = join(process.cwd(), 'docstore'); +const MODEL_DIR = join(DOCSTORE_DIR, 'model'); +const MODEL_PATH = join(MODEL_DIR, MODEL_NAME); +const modelReadyPromises = new Map>(); + +async function ensureModelAvailable(): Promise { + try { + await access(MODEL_PATH); + return; + } catch { + // continue + } + + const existing = modelReadyPromises.get(MODEL_PATH); + if (existing) return existing; + + const promise = (async () => { + try { + await access(MODEL_PATH); + return; + } catch { + // still missing + } + + await mkdir(MODEL_DIR, { recursive: true }); + + const res = await fetch(MODEL_URL); + if (!res.ok) { + throw new Error( + `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}` + ); + } + + const arrayBuffer = await res.arrayBuffer(); + await writeFile(MODEL_PATH, Buffer.from(arrayBuffer)); + })(); + + modelReadyPromises.set(MODEL_PATH, promise); + return promise; +} + +async function runWhisperCpp( + wavPath: string, + opts: WhisperAlignmentOptions +): Promise { + const binary = process.env.WHISPER_CPP_BIN; + if (!binary) { + throw new Error( + 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.' + ); + } + + await ensureModelAvailable(); + + return new Promise((resolve, reject) => { + const jsonBase = `${wavPath}.json_out`; + const jsonPath = `${jsonBase}.json`; + const args = [ + '-m', + MODEL_PATH, + '-f', + wavPath, + '-of', + jsonBase, + '-ojf', + '-np', + ]; + + if (opts.lang) { + args.push('-l', opts.lang); + } + + const child = spawn(binary, args); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('error', (err) => { + reject(err); + }); + + child.on('close', (code) => { + if (code !== 0) { + return reject( + new Error( + `whisper.cpp exited with code ${code}: ${stderr || stdout}` + ) + ); + } + + readFile(jsonPath, 'utf-8') + .then((content: string) => { + const words: WhisperWord[] = []; + const parsed = JSON.parse(content) as { + transcription?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + tokens?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + }>; + }>; + }; + + const transcription = parsed.transcription; + + const parseTimecode = (value?: string): number | null => { + if (!value) return null; + const m = value.match(/(\d+):(\d+):(\d+),(\d+)/); + if (!m) return null; + const h = Number(m[1]); + const min = Number(m[2]); + const s = Number(m[3]); + const ms = Number(m[4]); + if ( + Number.isNaN(h) + || Number.isNaN(min) + || Number.isNaN(s) + || Number.isNaN(ms) + ) { + return null; + } + return h * 3600 + min * 60 + s + ms / 1000; + }; + + if (Array.isArray(transcription)) { + for (const seg of transcription) { + const segText = (seg.text || '').trim(); + const segStartSecFromTs = parseTimecode( + seg.timestamps?.from + ); + const segEndSecFromTs = parseTimecode(seg.timestamps?.to); + const segStartSecFromMs = + typeof seg.offsets?.from === 'number' + ? seg.offsets.from / 1000 + : null; + const segEndSecFromMs = + typeof seg.offsets?.to === 'number' + ? seg.offsets.to / 1000 + : null; + + const segStartSec = + segStartSecFromTs + ?? segStartSecFromMs + ?? 0; + const segEndSec = + segEndSecFromTs + ?? segEndSecFromMs + ?? segStartSec; + + const tokens = Array.isArray(seg.tokens) + ? seg.tokens + : []; + + if (tokens.length > 0) { + for (const token of tokens) { + const rawText = token.text || ''; + const tokenText = rawText.trim(); + if (!tokenText || /^\[.*\]$/.test(tokenText)) continue; + + const tokStartSecFromTs = parseTimecode( + token.timestamps?.from + ); + const tokEndSecFromTs = parseTimecode( + token.timestamps?.to + ); + const tokStartSecFromMs = + typeof token.offsets?.from === 'number' + ? token.offsets.from / 1000 + : null; + const tokEndSecFromMs = + typeof token.offsets?.to === 'number' + ? token.offsets.to / 1000 + : null; + + const startSec = + tokStartSecFromTs + ?? tokStartSecFromMs + ?? segStartSec; + const endSec = + tokEndSecFromTs + ?? tokEndSecFromMs + ?? segEndSec; + + words.push({ + word: tokenText, + start: startSec, + end: endSec, + }); + } + } else if (segText) { + const segTokens = segText.split(/\s+/).filter(Boolean); + if (segTokens.length) { + const totalDur = Math.max(segEndSec - segStartSec, 0); + const step = + segTokens.length > 0 + ? totalDur / segTokens.length + : 0; + segTokens.forEach((token, index) => { + const wStart = + step > 0 + ? segStartSec + step * index + : segStartSec; + const wEnd = + step > 0 + ? index === segTokens.length - 1 + ? segEndSec + : segStartSec + step * (index + 1) + : segEndSec; + words.push({ + word: token, + start: wStart, + end: wEnd, + }); + }); + } + } + } + } + + resolve(words); + }) + .catch((err: unknown) => { + reject(err); + }); + }); + }); +} + +function mapWordsToSentenceOffsets( + sentence: string, + words: WhisperWord[] +): TTSSentenceAlignment { + const normalizedSentence = preprocessSentenceForAudio(sentence); + let cursor = 0; + + const alignedWords = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = normalizedSentence + .toLowerCase() + .indexOf(token.toLowerCase(), cursor); + + const start = + idx !== -1 + ? idx + : cursor; + const end = start + token.length; + + cursor = end; + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords.filter((w) => w.text.length > 0), + }; +} + +export async function alignAudioWithText( + audioBuffer: TTSAudioBuffer, + text: string, + cacheKey?: string, + opts: WhisperAlignmentOptions = {} +): Promise { + if (!text.trim()) { + return []; + } + + if (cacheKey && alignmentCache.has(cacheKey)) { + return alignmentCache.get(cacheKey)!; + } + + const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); + const inputPath = join(tmpBase, `${randomUUID()}-input.bin`); + const wavPath = join(tmpBase, `${randomUUID()}-input.wav`); + + try { + await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); + + await new Promise((resolve, reject) => { + const ffmpeg = spawn(getFFmpegPath(), [ + '-y', + '-i', + inputPath, + '-ar', + '16000', + '-ac', + '1', + wavPath, + ]); + + let stderr = ''; + ffmpeg.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('error', (err) => { + reject(err); + }); + + ffmpeg.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject( + new Error(`ffmpeg failed with code ${code}: ${stderr}`) + ); + } + }); + }); + + const words = await runWhisperCpp(wavPath, opts); + const alignment = mapWordsToSentenceOffsets(text, words); + const result: TTSSentenceAlignment[] = [alignment]; + + if (cacheKey) { + alignmentCache.set(cacheKey, result); + } + + return result; + } finally { + await rm(tmpBase, { recursive: true, force: true }).catch(() => {}); + } +} + +export function makeWhisperCacheKey(input: WhisperRequestBody): string { + return createHash('sha256') + .update( + JSON.stringify({ + text: input.text, + lang: input.lang || '', + audioLen: input.audio?.length || 0, + }) + ) + .digest('hex'); +} diff --git a/src/types/client.ts b/src/types/client.ts index 1f3eab0..a1985fb 100644 --- a/src/types/client.ts +++ b/src/types/client.ts @@ -85,3 +85,46 @@ export interface AlignmentPayload { export interface AlignmentResponse { alignments: TTSSentenceAlignment[]; } + +export interface TTSSegmentSettings { + ttsProvider: string; + ttsModel: string; + voice: string; + nativeSpeed: number; + ttsInstructions?: string; +} + +export interface TTSSegmentLocator { + page?: number; + location?: string; + readerType?: 'pdf' | 'epub' | 'html'; +} + +export interface TTSSegmentInput { + segmentIndex: number; + text: string; + locator?: TTSSegmentLocator; +} + +export interface TTSSegmentsEnsureRequest { + documentId: string; + segments: TTSSegmentInput[]; + settings: TTSSegmentSettings; +} + +export interface TTSSegmentManifestItem { + segmentId: string; + segmentIndex: number; + audioUrl: string; + durationMs: number; + alignment: TTSSentenceAlignment | null; + locator: TTSSegmentLocator | null; + status: 'pending' | 'completed' | 'error'; +} + +export interface TTSSegmentsEnsureResponse { + documentId: string; + documentVersion: number; + settingsHash: string; + segments: TTSSegmentManifestItem[]; +} diff --git a/tests/global-teardown.ts b/tests/global-teardown.ts index f47f32a..f77bdd5 100644 --- a/tests/global-teardown.ts +++ b/tests/global-teardown.ts @@ -4,6 +4,7 @@ import { db } from '../src/db'; import { audiobooks, audiobookChapters, documents } from '../src/db/schema'; import { deleteDocumentPrefix } from '../src/lib/server/documents/blobstore'; import { deleteAudiobookPrefix } from '../src/lib/server/audiobooks/blobstore'; +import { deleteTtsSegmentPrefix } from '../src/lib/server/tts/segments-blobstore'; import { getS3Client, getS3Config, isS3Configured } from '../src/lib/server/storage/s3'; function chunk(items: T[], size: number): T[][] { @@ -75,6 +76,7 @@ export default async function globalTeardown(): Promise { const config = getS3Config(); const docsNsRootPrefix = `${config.prefix}/documents_v1/ns/`; const audiobooksNsRootPrefix = `${config.prefix}/audiobooks_v1/ns/`; + const ttsSegmentsNsRootPrefix = `${config.prefix}/tts_segments_v1/ns/`; // Remove SQL audiobook rows for namespaced objects (covers auth claim flows too). const audiobookKeys = await listKeysByPrefix(audiobooksNsRootPrefix); @@ -103,4 +105,5 @@ export default async function globalTeardown(): Promise { await deleteDocumentPrefix(docsNsRootPrefix); await deleteAudiobookPrefix(audiobooksNsRootPrefix); + await deleteTtsSegmentPrefix(ttsSegmentsNsRootPrefix); } diff --git a/tests/unit/tts-segments.spec.ts b/tests/unit/tts-segments.spec.ts new file mode 100644 index 0000000..1c5c3bb --- /dev/null +++ b/tests/unit/tts-segments.spec.ts @@ -0,0 +1,84 @@ +import { expect, test } from '@playwright/test'; +import { + buildProportionalAlignment, + buildTtsSegmentId, + buildTtsSegmentSettingsHash, + buildTtsSegmentTextHash, + locatorFingerprint, + normalizeLocator, + normalizeSegmentText, +} from '../../src/lib/server/tts/segments'; + +test.describe('tts segment helpers', () => { + test('builds stable settings hash', () => { + const a = buildTtsSegmentSettingsHash({ + ttsProvider: 'openai', + ttsModel: 'gpt-4o-mini-tts', + voice: 'alloy', + nativeSpeed: 1, + ttsInstructions: 'calm', + }); + const b = buildTtsSegmentSettingsHash({ + ttsProvider: 'openai', + ttsModel: 'gpt-4o-mini-tts', + voice: 'alloy', + nativeSpeed: 1, + ttsInstructions: 'calm', + }); + expect(a).toBe(b); + }); + + test('builds deterministic segment id', () => { + const id1 = buildTtsSegmentId({ + documentId: 'doc', + documentVersion: 1, + settingsHash: 'abc', + segmentIndex: 2, + normalizedText: 'hello world', + locatorFingerprint: 'loc', + }); + const id2 = buildTtsSegmentId({ + documentId: 'doc', + documentVersion: 1, + settingsHash: 'abc', + segmentIndex: 2, + normalizedText: 'hello world', + locatorFingerprint: 'loc', + }); + const id3 = buildTtsSegmentId({ + documentId: 'doc', + documentVersion: 1, + settingsHash: 'abc', + segmentIndex: 3, + normalizedText: 'hello world', + locatorFingerprint: 'loc', + }); + expect(id1).toBe(id2); + expect(id1).not.toBe(id3); + }); + + test('does not leak plaintext via text hash', () => { + const hash = buildTtsSegmentTextHash('plain sentence', 'secret'); + expect(hash).not.toContain('plain'); + expect(hash).toHaveLength(64); + }); + + test('normalizes locators and creates fingerprints', () => { + const locator = normalizeLocator({ page: 2.9, location: ' cfi(1) ', readerType: 'epub' }); + expect(locator).toEqual({ page: 2, location: 'cfi(1)', readerType: 'epub' }); + expect(locatorFingerprint(locator)).toHaveLength(64); + }); + + test('builds proportional alignment preserving order', () => { + const alignment = buildProportionalAlignment({ + sentence: normalizeSegmentText('Hello world again'), + sentenceIndex: 5, + durationMs: 1500, + }); + expect(alignment.sentenceIndex).toBe(5); + expect(alignment.words.length).toBe(3); + expect(alignment.words[0].startSec).toBe(0); + expect(alignment.words[2].endSec).toBeGreaterThan(1.4); + expect(alignment.words[1].charStart).toBeGreaterThan(alignment.words[0].charStart); + }); +});