From 265e1cb56a767ed677969dc624b102b070b3b4bb Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 7 May 2026 12:26:08 -0600 Subject: [PATCH] feat(tts,config,ui): add advanced segment preloading and block length controls Introduce configurable segment preloading depth, sentence lookahead, and TTS segment max block length for both PDF and EPUB readers. Add new settings to user preferences, config context, and document settings UI. Refactor TTS segment splitting logic to support per-user max block length and propagate these options through PDF/EPUB adapters and TTS segment generation. Update API, manifest pagination, and segment deduplication for improved performance and correctness. Add tests for new NLP options and manifest logic. BREAKING CHANGE: TTS segment splitting and manifest APIs now require max block length and preloading parameters; user config schema updated. --- src/app/api/tts/segments/clear/route.ts | 9 +- src/app/api/tts/segments/ensure/route.ts | 45 +- src/app/api/tts/segments/manifest/route.ts | 98 +- src/app/api/user/state/preferences/route.ts | 3 + src/components/AudiobookExportModal.tsx | 5 +- src/components/documents/DocumentSettings.tsx | 583 +++++----- src/components/reader/ReaderSidebarShell.tsx | 3 + src/components/reader/SegmentsSidebar.tsx | 417 +++++-- src/components/views/EPUBViewer.tsx | 15 +- src/contexts/ConfigContext.tsx | 9 + src/contexts/EPUBContext.tsx | 166 ++- src/contexts/PDFContext.tsx | 31 +- src/contexts/TTSContext.tsx | 1013 +++++++++++++++-- src/hooks/epub/useEPUBTheme.ts | 42 +- src/lib/client/api/audiobooks.ts | 6 +- src/lib/client/audiobooks/adapters/pdf.ts | 4 +- .../client/epub/rendered-location-walker.ts | 302 +++++ src/lib/server/tts/segments-blobstore.ts | 5 +- src/lib/server/tts/segments-manifest.ts | 77 ++ src/lib/server/tts/segments.ts | 16 + src/lib/shared/nlp.ts | 31 +- src/lib/shared/tts-locator.ts | 18 + src/types/client.ts | 2 + src/types/config.ts | 29 + src/types/tts.ts | 6 + src/types/user-state.ts | 4 +- tests/navigation.spec.ts | 8 +- tests/unit/nlp.spec.ts | 16 + tests/unit/tts-segments-manifest.spec.ts | 114 ++ 29 files changed, 2475 insertions(+), 602 deletions(-) create mode 100644 src/lib/client/epub/rendered-location-walker.ts create mode 100644 src/lib/server/tts/segments-manifest.ts create mode 100644 src/lib/shared/tts-locator.ts create mode 100644 tests/unit/tts-segments-manifest.spec.ts diff --git a/src/app/api/tts/segments/clear/route.ts b/src/app/api/tts/segments/clear/route.ts index 41d7e34..2f6fdea 100644 --- a/src/app/api/tts/segments/clear/route.ts +++ b/src/app/api/tts/segments/clear/route.ts @@ -48,12 +48,16 @@ export async function POST(request: NextRequest) { const audioKeys = rows .map((row) => row.audioKey) .filter((key): key is string => Boolean(key)); + const uniqueAudioKeys = Array.from(new Set(audioKeys)); let deletedAudioObjects = 0; let warning: string | undefined; - if (audioKeys.length > 0) { + if (uniqueAudioKeys.length > 0) { try { - deletedAudioObjects = await deleteTtsSegmentAudioObjects(audioKeys); + deletedAudioObjects = await deleteTtsSegmentAudioObjects(uniqueAudioKeys); + if (deletedAudioObjects < uniqueAudioKeys.length) { + warning = `Deleted ${deletedAudioObjects} of ${uniqueAudioKeys.length} audio objects.`; + } } catch (error) { warning = error instanceof Error ? error.message : 'Failed deleting some audio objects'; console.warn('Failed clearing some TTS segment audio objects:', { @@ -67,6 +71,7 @@ export async function POST(request: NextRequest) { return NextResponse.json({ documentId: parsed.documentId, deletedSegments: rows.length, + requestedAudioObjects: uniqueAudioKeys.length, deletedAudioObjects, ...(warning ? { warning } : {}), }); diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index bab9a18..948693a 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -15,6 +15,8 @@ import { buildTtsSegmentSettingsHash, buildTtsSegmentSettingsJson, buildTtsSegmentTextHash, + canonicalLocatorJson, + canonicalizeLocatorJsonString, locatorFingerprint, normalizeLocator, normalizeSegmentText, @@ -125,11 +127,22 @@ function buildSegmentAudioUrls(documentId: string, segmentId: string): { }; } -async function cleanupStaleErroredVariants(input: { +function isAbortLikeError(error: unknown): boolean { + const message = error instanceof Error + ? error.message + : typeof error === 'string' + ? error + : ''; + if (!message) return false; + return /abort/i.test(message); +} + +async function cleanupStaleCanonicalVariants(input: { userId: string; documentId: string; documentVersion: number; segmentIndex: number; + activeLocatorJson: string | null; activeSegmentId: string; activeSettingsHash: string; }): Promise { @@ -138,6 +151,7 @@ async function cleanupStaleErroredVariants(input: { segmentId: ttsSegments.segmentId, settingsHash: ttsSegments.settingsHash, audioKey: ttsSegments.audioKey, + locatorJson: ttsSegments.locatorJson, }) .from(ttsSegments) .where(and( @@ -150,9 +164,14 @@ async function cleanupStaleErroredVariants(input: { segmentId: string; settingsHash: string; audioKey: string | null; + locatorJson: string | null; }>; - const staleRowsForSettings = staleRows.filter((row) => row.settingsHash === input.activeSettingsHash); + const activeCanonicalLocator = canonicalizeLocatorJsonString(input.activeLocatorJson); + const staleRowsForSettings = staleRows.filter((row) => + row.settingsHash === input.activeSettingsHash + && canonicalizeLocatorJsonString(row.locatorJson) === activeCanonicalLocator, + ); const staleIds = staleRowsForSettings.map((row) => row.segmentId); if (staleIds.length === 0) return; @@ -164,9 +183,9 @@ async function cleanupStaleErroredVariants(input: { inArray(ttsSegments.segmentId, staleIds), )); - const staleAudioKeys = staleRowsForSettings + const staleAudioKeys = Array.from(new Set(staleRowsForSettings .map((row) => row.audioKey) - .filter((key): key is string => Boolean(key)); + .filter((key): key is string => Boolean(key)))); if (staleAudioKeys.length > 0) { try { await deleteTtsSegmentAudioObjects(staleAudioKeys); @@ -263,11 +282,12 @@ export async function POST(request: NextRequest) { const existing = existingById.get(segment.segmentId); if (existing?.status === 'completed' && existing.audioKey) { - await cleanupStaleErroredVariants({ + await cleanupStaleCanonicalVariants({ userId: scope.storageUserId, documentId: parsed.documentId, documentVersion: scope.documentVersion, segmentIndex: segment.original.segmentIndex, + activeLocatorJson: existing.locatorJson, activeSegmentId: segment.segmentId, activeSettingsHash: settingsHash, }); @@ -333,6 +353,7 @@ export async function POST(request: NextRequest) { segmentId: segment.segmentId, }); + const segmentLocatorJson = canonicalLocatorJson(segment.locator); await db .insert(ttsSegments) .values({ @@ -342,7 +363,7 @@ export async function POST(request: NextRequest) { readerType: scope.readerType, documentVersion: scope.documentVersion, segmentIndex: segment.original.segmentIndex, - locatorJson: segment.locator ? JSON.stringify(segment.locator) : null, + locatorJson: segmentLocatorJson, settingsHash, settingsJson, textHash: segment.textHash, @@ -360,7 +381,7 @@ export async function POST(request: NextRequest) { readerType: scope.readerType, documentVersion: scope.documentVersion, segmentIndex: segment.original.segmentIndex, - locatorJson: segment.locator ? JSON.stringify(segment.locator) : null, + locatorJson: segmentLocatorJson, settingsHash, settingsJson, textHash: segment.textHash, @@ -472,11 +493,12 @@ export async function POST(request: NextRequest) { }) .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); - await cleanupStaleErroredVariants({ + await cleanupStaleCanonicalVariants({ userId: scope.storageUserId, documentId: parsed.documentId, documentVersion: scope.documentVersion, segmentIndex: segment.original.segmentIndex, + activeLocatorJson: canonicalLocatorJson(segment.locator), activeSegmentId: segment.segmentId, activeSettingsHash: settingsHash, }); @@ -492,11 +514,12 @@ export async function POST(request: NextRequest) { }); } catch (error) { const message = error instanceof Error ? error.message : 'Failed to generate segment'; + const aborted = isAbortLikeError(error); await db .update(ttsSegments) .set({ - status: 'error', - error: message, + status: aborted ? 'pending' : 'error', + error: aborted ? null : message, updatedAt: Date.now(), }) .where(and(eq(ttsSegments.segmentId, segment.segmentId), eq(ttsSegments.userId, scope.storageUserId))); @@ -509,7 +532,7 @@ export async function POST(request: NextRequest) { durationMs: 0, alignment: null, locator: segment.locator, - status: 'error', + status: aborted ? 'pending' : 'error', }); } } diff --git a/src/app/api/tts/segments/manifest/route.ts b/src/app/api/tts/segments/manifest/route.ts index c123e6d..fcaf0ca 100644 --- a/src/app/api/tts/segments/manifest/route.ts +++ b/src/app/api/tts/segments/manifest/route.ts @@ -3,6 +3,14 @@ import { and, asc, eq } from 'drizzle-orm'; import { db } from '@/db'; import { ttsSegments } from '@/db/schema'; import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; +import { + compareManifestSegments, + decodeManifestCursor, + dedupeManifestVariants, + encodeManifestCursor, + locatorGroupKey, + parseManifestPageSize, +} from '@/lib/server/tts/segments-manifest'; import type { TTSSegmentLocator, TTSSegmentRow, @@ -65,45 +73,17 @@ function buildSegmentAudioUrls(documentId: string, segmentId: string): { }; } -function statusRank(status: TTSSegmentVariant['status']): number { - if (status === 'completed') return 3; - if (status === 'pending') return 2; - return 1; -} - -function dedupeVariants(variants: Array<{ dedupeKey: string; variant: TTSSegmentVariant }>): TTSSegmentVariant[] { - const byKey = new Map(); - for (const { dedupeKey, variant } of variants) { - const existing = byKey.get(dedupeKey); - if (!existing) { - byKey.set(dedupeKey, variant); - continue; - } - const existingRank = statusRank(existing.status); - const nextRank = statusRank(variant.status); - const existingUpdatedAt = existing.updatedAt ?? 0; - const nextUpdatedAt = variant.updatedAt ?? 0; - if (nextRank > existingRank || (nextRank === existingRank && nextUpdatedAt >= existingUpdatedAt)) { - byKey.set(dedupeKey, variant); - } - } - return Array.from(byKey.values()); -} - -function locatorGroupKey(locator: TTSSegmentLocator | null): string { - if (!locator) return 'none'; - const page = typeof locator.page === 'number' && Number.isFinite(locator.page) - ? String(Math.floor(locator.page)) - : ''; - const location = typeof locator.location === 'string' ? locator.location : ''; - const readerType = locator.readerType || ''; - return `p:${page}|l:${location}|r:${readerType}`; +function isAbortLikeMessage(message: string | null | undefined): boolean { + if (!message) return false; + return /abort/i.test(message); } export async function GET(request: NextRequest) { try { const documentIdRaw = request.nextUrl.searchParams.get('documentId'); const documentId = documentIdRaw?.trim().toLowerCase(); + const limit = parseManifestPageSize(request.nextUrl.searchParams.get('limit')); + const cursor = decodeManifestCursor(request.nextUrl.searchParams.get('cursor')); if (!documentId) { return NextResponse.json({ error: 'Missing documentId' }, { status: 400 }); } @@ -171,9 +151,11 @@ export async function GET(request: NextRequest) { } } - const status: TTSSegmentVariant['status'] = row.status === 'completed' || row.status === 'error' - ? row.status - : 'pending'; + const status: TTSSegmentVariant['status'] = row.status === 'completed' + ? 'completed' + : row.status === 'error' && !isAbortLikeMessage(row.error) + ? 'error' + : 'pending'; const audioUrls = row.status === 'completed' && row.audioKey ? buildSegmentAudioUrls(documentId, row.segmentId) @@ -196,22 +178,40 @@ export async function GET(request: NextRequest) { }); } - const segments = Array.from(grouped.values()) - .map((segment) => ({ + const segments = Array.from(grouped.entries()) + .map(([groupKey, segment]) => ({ + groupKey, segmentIndex: segment.segmentIndex, locator: segment.locator, - variants: dedupeVariants(segment.variants), + variants: dedupeManifestVariants(segment.variants), })) - .sort((a, b) => { - if (a.segmentIndex !== b.segmentIndex) return a.segmentIndex - b.segmentIndex; - const aPage = typeof a.locator?.page === 'number' ? a.locator.page : Number.MAX_SAFE_INTEGER; - const bPage = typeof b.locator?.page === 'number' ? b.locator.page : Number.MAX_SAFE_INTEGER; - if (aPage !== bPage) return aPage - bPage; - const aLoc = a.locator?.location || ''; - const bLoc = b.locator?.location || ''; - return aLoc.localeCompare(bLoc); - }); - const response: TTSSegmentsManifestResponse = { documentId, segments }; + .sort(compareManifestSegments); + + let startIndex = 0; + if (cursor) { + const cursorIndex = segments.findIndex((segment) => segment.groupKey === cursor); + if (cursorIndex < 0) { + return NextResponse.json({ error: 'Invalid cursor' }, { status: 400 }); + } + if (cursorIndex >= 0) startIndex = cursorIndex + 1; + } + + const page = segments.slice(startIndex, startIndex + limit); + const hasMore = startIndex + page.length < segments.length; + const nextCursor = hasMore && page.length > 0 + ? encodeManifestCursor(page[page.length - 1].groupKey) + : null; + + const response: TTSSegmentsManifestResponse = { + documentId, + segments: page.map((segment) => ({ + segmentIndex: segment.segmentIndex, + locator: segment.locator, + variants: segment.variants, + })), + nextCursor, + hasMore, + }; return NextResponse.json(response); } catch (error) { console.error('Error listing TTS segments manifest:', error); diff --git a/src/app/api/user/state/preferences/route.ts b/src/app/api/user/state/preferences/route.ts index 6841e82..5e23286 100644 --- a/src/app/api/user/state/preferences/route.ts +++ b/src/app/api/user/state/preferences/route.ts @@ -62,6 +62,9 @@ function sanitizePreferencesPatch(input: unknown): SyncedPreferencesPatch { break; case 'voiceSpeed': case 'audioPlayerSpeed': + case 'segmentPreloadDepthPages': + case 'segmentPreloadSentenceLookahead': + case 'ttsSegmentMaxBlockLength': case 'headerMargin': case 'footerMargin': case 'leftMargin': diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index 7060a47..ab874fe 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -464,6 +464,7 @@ export function AudiobookExportModal({ onClose={() => setIsOpen(false)} ariaLabel="Export audiobook" title="Export Audiobook" + subtitle="Only leaving the document cancels generation." > {isLoadingExisting ? (
@@ -882,9 +883,7 @@ export function AudiobookExportModal({ {chapters.length === 0 && !isGenerating && !isLoadingExisting && (

- Audiobook settings are fixed after generation. Chapters will appear here as they are ready. -

- You can close this dialog while the audiobook is being generated. But returning to the home screen will cancel the generation. + Audiobook settings are fixed after generation. Chapters will appear here as they are ready.

)} diff --git a/src/components/documents/DocumentSettings.tsx b/src/components/documents/DocumentSettings.tsx index ed721b2..faf6cb1 100644 --- a/src/components/documents/DocumentSettings.tsx +++ b/src/components/documents/DocumentSettings.tsx @@ -1,10 +1,20 @@ 'use client'; -import { Fragment, useState, useEffect } from 'react'; -import { Transition, Listbox, ListboxButton, ListboxOptions, ListboxOption } from '@headlessui/react'; +import { useState, useEffect, type ChangeEvent } from 'react'; import { useConfig, ViewType } from '@/contexts/ConfigContext'; -import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; +import { + SEGMENT_PRELOAD_DEPTH_MIN, + SEGMENT_PRELOAD_DEPTH_MAX, + SEGMENT_PRELOAD_SENTENCE_LOOKAHEAD_MIN, + SEGMENT_PRELOAD_SENTENCE_LOOKAHEAD_MAX, + TTS_SEGMENT_MAX_BLOCK_LENGTH_MIN, + TTS_SEGMENT_MAX_BLOCK_LENGTH_MAX, + TTS_SEGMENT_MAX_BLOCK_LENGTH_STEP, + clampSegmentPreloadDepth, + clampSegmentPreloadSentenceLookahead, + clampTtsSegmentMaxBlockLength, +} from '@/types/config'; const canWordHighlight = process.env.NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT?.toLowerCase() !== 'false'; @@ -14,6 +24,81 @@ const viewTypeTextMapping = [ { id: 'scroll', name: 'Continuous Scroll' }, ]; +const rangeInputClassName = 'w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent'; + +type MarginKey = 'header' | 'footer' | 'left' | 'right'; + +type RangeSettingProps = { + label: string; + value: number; + min: number; + max: number; + step: number; + description: string; + valueWidth?: string; + formatter?: (value: number) => string; + onChange: (value: number) => void; +}; + +function RangeSetting({ + label, + value, + min, + max, + step, + description, + valueWidth = 'w-10', + formatter = (next) => String(next), + onChange, +}: RangeSettingProps) { + return ( +
+ +
+ onChange(Number(event.target.value))} + className={`flex-1 ${rangeInputClassName}`} + /> + {formatter(value)} +
+

{description}

+
+ ); +} + +type ToggleRowProps = { + label: string; + description: string; + checked: boolean; + onChange: (checked: boolean) => void; + disabled?: boolean; +}; + +function ToggleRow({ label, description, checked, onChange, disabled = false }: ToggleRowProps) { + return ( +
+ +
+ ); +} + export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { isOpen: boolean, setIsOpen: (isOpen: boolean) => void, @@ -25,6 +110,9 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { skipBlank, epubTheme, smartSentenceSplitting, + segmentPreloadDepthPages, + segmentPreloadSentenceLookahead, + ttsSegmentMaxBlockLength, headerMargin, footerMargin, leftMargin, @@ -42,6 +130,15 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { right: rightMargin }); const selectedView = viewTypeTextMapping.find(v => v.id === viewType) || viewTypeTextMapping[0]; + const [localPreloadDepth, setLocalPreloadDepth] = useState(segmentPreloadDepthPages); + const [localSentenceLookahead, setLocalSentenceLookahead] = useState(segmentPreloadSentenceLookahead); + const [localMaxBlockLength, setLocalMaxBlockLength] = useState(ttsSegmentMaxBlockLength); + const marginValues: Record = { + header: headerMargin, + footer: footerMargin, + left: leftMargin, + right: rightMargin, + }; useEffect(() => { setLocalMargins({ @@ -52,297 +149,235 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { }); }, [headerMargin, footerMargin, leftMargin, rightMargin]); - // Handler for slider change (updates local state only) - const handleMarginChange = (margin: keyof typeof localMargins) => (event: React.ChangeEvent) => { - setLocalMargins(prev => ({ - ...prev, - [margin]: Number(event.target.value) - })); - }; + useEffect(() => { + setLocalPreloadDepth(segmentPreloadDepthPages); + }, [segmentPreloadDepthPages]); + + useEffect(() => { + setLocalSentenceLookahead(segmentPreloadSentenceLookahead); + }, [segmentPreloadSentenceLookahead]); + + useEffect(() => { + setLocalMaxBlockLength(ttsSegmentMaxBlockLength); + }, [ttsSegmentMaxBlockLength]); // Handler for slider release - const handleMarginChangeComplete = (margin: keyof typeof localMargins) => () => { + const handleMarginChangeComplete = (margin: MarginKey) => () => { const value = localMargins[margin]; const configKey = `${margin}Margin`; - if (value !== (useConfig)[configKey as keyof typeof useConfig]) { + if (value !== marginValues[margin]) { updateConfigKey(configKey as 'headerMargin' | 'footerMargin' | 'leftMargin' | 'rightMargin', value); } }; + const handleMarginSliderChange = (margin: MarginKey) => (event: ChangeEvent) => { + setLocalMargins((previous) => ({ + ...previous, + [margin]: Number(event.target.value), + })); + }; + return ( setIsOpen(false)} ariaLabel="Document settings" - title="Settings" + title="Reader Settings" + subtitle="Configure layout, preloading, and playback behavior for this document." + bodyClassName="flex-1 overflow-y-auto px-4 py-4 bg-[radial-gradient(circle_at_top_right,rgba(239,68,68,0.08),transparent_35%)]" + panelClassName="w-full sm:w-[30rem]" >
- {!epub && !html &&
-
- -
- {/* Header Margin */} -
-
- Header - {Math.round(localMargins.header * 100)}% -
- -
+ {!html && ( +
+
+

Playback Flow

+

Control segment generation and lookahead while audio is active.

+
- {/* Footer Margin */} -
-
- Footer - {Math.round(localMargins.footer * 100)}% -
- -
+ updateConfigKey('skipBlank', checked)} + /> - {/* Left Margin */} -
-
- Left - {Math.round(localMargins.left * 100)}% -
- -
+ updateConfigKey('smartSentenceSplitting', checked)} + /> - {/* Right Margin */} -
-
- Right - {Math.round(localMargins.right * 100)}% -
- -
-
-

- Adjust margins to exclude content from edges of the page during text extraction (experimental) -

-
- updateConfigKey('viewType', newView.id as ViewType)} - > -
- - - {selectedView.name} - - - - - - - {viewTypeTextMapping.map((view) => ( - - `relative cursor-pointer select-none py-1.5 pl-10 pr-4 ${active ? 'bg-offbase text-accent' : 'text-foreground' - }` - } - value={view} - > - {({ selected }) => ( - <> - - {view.name} - - {selected ? ( - - - - ) : null} - - )} - - ))} - - - {selectedView.id === 'scroll' && ( -

- Note: Continuous scroll may perform poorly for larger documents. -

- )} -
- +
+ String(value)} + onChange={(value) => { + const next = clampSegmentPreloadDepth(value); + setLocalPreloadDepth(next); + void updateConfigKey('segmentPreloadDepthPages', next); + }} + /> -
} + String(value)} + onChange={(value) => { + const next = clampSegmentPreloadSentenceLookahead(value); + setLocalSentenceLookahead(next); + void updateConfigKey('segmentPreloadSentenceLookahead', next); + }} + /> - {!html &&
- -

- Automatically skip pages with no text content -

-
} - {!html && ( -
- -

- Merge sentences across page or section breaks -

-
- )} - {!epub && !html && ( -
-
- -

- Visual text playback highlighting in the PDF viewer -

-
-
- -

- Highlight individual words using audio timestamps generated by whisper.cpp {!canWordHighlight && '(disabled by configuration)'} -

-
-
- )} - {epub && ( -
-
- -

- Visual text playback highlighting in the EPUB viewer -

-
-
- -

- Highlight individual words using audio timestamps generated by whisper.cpp {!canWordHighlight && '(disabled by configuration)'} -

-
-
- )} - {epub && ( -
- -

- Apply the current app theme to the EPUB viewer background and text colors -

-
- )} + String(value)} + onChange={(value) => { + const next = clampTtsSegmentMaxBlockLength(value); + setLocalMaxBlockLength(next); + void updateConfigKey('ttsSegmentMaxBlockLength', next); + }} + /> +
+ + )} + + {!epub && !html && ( +
+
+

PDF Layout & Extraction

+

Set viewer mode and trim page edges before extraction.

+
+ +
+ +
+ {viewTypeTextMapping.map((view) => { + const active = selectedView.id === view.id; + return ( + + ); + })} +
+ {selectedView.id === 'scroll' ? ( +

Continuous scroll may perform poorly for very large PDFs.

+ ) : null} +
+ +
+

Text extraction margins

+

+ Exclude content near edges before sentence extraction. +

+
+ {(['header', 'footer', 'left', 'right'] as MarginKey[]).map((margin) => ( +
+
+ {margin} + {Math.round(localMargins[margin] * 100)}% +
+ +
+ ))} +
+
+
+ )} + + {!epub && !html && ( +
+
+

PDF Highlighting

+

Control playback highlighting behavior in PDF mode.

+
+ updateConfigKey('pdfHighlightEnabled', checked)} + /> + updateConfigKey('pdfWordHighlightEnabled', checked)} + /> +
+ )} + + {epub && ( +
+
+

EPUB Appearance

+

Apply app styling and playback highlighting in EPUB mode.

+
+ updateConfigKey('epubTheme', checked)} + /> + updateConfigKey('epubHighlightEnabled', checked)} + /> + updateConfigKey('epubWordHighlightEnabled', checked)} + /> +
+ )}
); diff --git a/src/components/reader/ReaderSidebarShell.tsx b/src/components/reader/ReaderSidebarShell.tsx index dbc9837..17ea545 100644 --- a/src/components/reader/ReaderSidebarShell.tsx +++ b/src/components/reader/ReaderSidebarShell.tsx @@ -10,6 +10,7 @@ interface ReaderSidebarShellProps { onClose: () => void; ariaLabel: string; title: string; + subtitle?: string; children: ReactNode; headerActions?: ReactNode; footer?: ReactNode; @@ -22,6 +23,7 @@ export function ReaderSidebarShell({ onClose, ariaLabel, title, + subtitle, children, headerActions, footer, @@ -70,6 +72,7 @@ export function ReaderSidebarShell({

{title}

+ {subtitle ?

{subtitle}

: null}
{headerActions} diff --git a/src/components/reader/SegmentsSidebar.tsx b/src/components/reader/SegmentsSidebar.tsx index 8049087..328b21c 100644 --- a/src/components/reader/SegmentsSidebar.tsx +++ b/src/components/reader/SegmentsSidebar.tsx @@ -2,10 +2,12 @@ import { Fragment, type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Popover, PopoverButton, PopoverPanel, Transition } from '@headlessui/react'; +import toast from 'react-hot-toast'; import { useTTS } from '@/contexts/TTSContext'; import { useConfig } from '@/contexts/ConfigContext'; import { RefreshIcon, InfoIcon } from '@/components/icons/Icons'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; +import { locatorGroupKey, normalizeEpubLocationToken } from '@/lib/shared/tts-locator'; import type { TTSSegmentLocator, TTSSegmentRow, @@ -23,9 +25,18 @@ interface SegmentsSidebarProps { type FetchState = | { kind: 'idle' } | { kind: 'loading' } - | { kind: 'ready'; data: TTSSegmentRow[]; fetchedAt: number } + | { + kind: 'ready'; + data: TTSSegmentRow[]; + fetchedAt: number; + nextCursor: string | null; + hasMore: boolean; + loadingMore: boolean; + } | { kind: 'error'; message: string }; +const MANIFEST_PAGE_SIZE = 150; + function formatDuration(ms: number | null | undefined): string { if (!ms || !Number.isFinite(ms) || ms <= 0) return '—'; const sec = ms / 1000; @@ -83,6 +94,10 @@ function locatorMatchesCurrent( ): boolean { if (!locator) return false; if (typeof locator.location === 'string' && locator.location.length > 0) { + if (locator.readerType === 'epub') { + if (typeof currentLocation !== 'string') return false; + return normalizeEpubLocationToken(locator.location) === normalizeEpubLocationToken(currentLocation); + } return String(locator.location) === String(currentLocation); } if (typeof locator.page === 'number' && Number.isFinite(locator.page)) { @@ -91,11 +106,71 @@ function locatorMatchesCurrent( return false; } -function latestUpdatedAt(row: TTSSegmentRow): number { - return row.variants.reduce((max, variant) => { - const updated = typeof variant.updatedAt === 'number' ? variant.updatedAt : 0; - return Math.max(max, updated); - }, 0); +function formatLocatorGroupLabel(locator: TTSSegmentLocator | null): string { + if (!locator) return 'Unknown location'; + const parts: string[] = []; + if (typeof locator.page === 'number' && Number.isFinite(locator.page)) { + parts.push(`Page ${Math.floor(locator.page)}`); + } + if (typeof locator.location === 'string' && locator.location) { + parts.push(locator.location); + } + if (locator.readerType) { + parts.push(locator.readerType.toUpperCase()); + } + return parts.join(' · ') || 'Unknown location'; +} + +function compareRows(a: TTSSegmentRow, b: TTSSegmentRow): number { + const aPage = typeof a.locator?.page === 'number' ? a.locator.page : Number.MAX_SAFE_INTEGER; + const bPage = typeof b.locator?.page === 'number' ? b.locator.page : Number.MAX_SAFE_INTEGER; + if (aPage !== bPage) return aPage - bPage; + const aLoc = a.locator?.location || ''; + const bLoc = b.locator?.location || ''; + const byLocation = aLoc.localeCompare(bLoc); + if (byLocation !== 0) return byLocation; + if (a.segmentIndex !== b.segmentIndex) return a.segmentIndex - b.segmentIndex; + return locatorGroupKey(a.locator).localeCompare(locatorGroupKey(b.locator)); +} + +function mergeRows(existing: TTSSegmentRow[], incoming: TTSSegmentRow[]): TTSSegmentRow[] { + const map = new Map(); + const upsert = (row: TTSSegmentRow) => { + const key = `${row.segmentIndex}|${locatorGroupKey(row.locator)}`; + const prev = map.get(key); + if (!prev) { + map.set(key, row); + return; + } + const bySegmentId = new Map(); + for (const variant of prev.variants) bySegmentId.set(variant.segmentId, variant); + for (const variant of row.variants) bySegmentId.set(variant.segmentId, variant); + map.set(key, { + ...row, + variants: Array.from(bySegmentId.values()), + }); + }; + existing.forEach(upsert); + incoming.forEach(upsert); + return Array.from(map.values()).sort(compareRows); +} + +function findScrollableAncestor(node: HTMLElement, fallback: HTMLElement | null): HTMLElement | null { + let current: HTMLElement | null = node.parentElement; + while (current) { + const style = window.getComputedStyle(current); + const overflowY = style.overflowY.toLowerCase(); + const canScroll = (overflowY === 'auto' || overflowY === 'scroll') && current.scrollHeight > current.clientHeight + 1; + if (canScroll) return current; + current = current.parentElement; + } + return fallback; +} + +function isElementFullyVisibleWithinContainer(element: HTMLElement, container: HTMLElement): boolean { + const elRect = element.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + return elRect.top >= containerRect.top && elRect.bottom <= containerRect.bottom; } export function SegmentsSidebar({ isOpen, setIsOpen, documentId }: SegmentsSidebarProps) { @@ -105,14 +180,15 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId }: SegmentsSideb currDocPage, currDocPageNumber, isPlaying, - stopAndPlayFromIndex, + playFromSegment, } = useTTS(); const { ttsProvider, ttsModel, voice, voiceSpeed, ttsInstructions, updateConfigKey } = useConfig(); const [state, setState] = useState({ kind: 'idle' }); const [isClearing, setIsClearing] = useState(false); - const [clearError, setClearError] = useState(null); const abortRef = useRef(null); + const listRef = useRef(null); + const didAutoScrollOnOpenRef = useRef(false); const activeSettings = useMemo(() => ({ ttsProvider, @@ -122,15 +198,33 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId }: SegmentsSideb ttsInstructions: ttsInstructions || '', }), [ttsProvider, ttsModel, voice, voiceSpeed, ttsInstructions]); - const loadManifest = useCallback(async () => { + const loadManifest = useCallback(async ( + mode: 'reset' | 'append' = 'reset', + cursorOverride: string | null = null, + ) => { if (!documentId) return; + const cursor = mode === 'append' ? cursorOverride : null; + if (mode === 'append' && !cursor) return; + abortRef.current?.abort(); const controller = new AbortController(); abortRef.current = controller; - setState({ kind: 'loading' }); + if (mode === 'reset') { + setState({ kind: 'loading' }); + } else { + setState((prev) => { + if (prev.kind !== 'ready') return prev; + return { ...prev, loadingMore: true }; + }); + } try { + const params = new URLSearchParams({ + documentId, + limit: String(MANIFEST_PAGE_SIZE), + }); + if (cursor) params.set('cursor', cursor); const res = await fetch( - `/api/tts/segments/manifest?documentId=${encodeURIComponent(documentId)}`, + `/api/tts/segments/manifest?${params.toString()}`, { signal: controller.signal, cache: 'no-store' }, ); if (!res.ok) { @@ -138,33 +232,71 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId }: SegmentsSideb throw new Error(body || `Request failed (${res.status})`); } const data = (await res.json()) as TTSSegmentsManifestResponse; - setState({ kind: 'ready', data: data.segments, fetchedAt: Date.now() }); + setState((prev) => { + const merged = mode === 'append' && prev.kind === 'ready' + ? mergeRows(prev.data, data.segments) + : mergeRows([], data.segments); + return { + kind: 'ready', + data: merged, + fetchedAt: Date.now(), + nextCursor: data.nextCursor, + hasMore: data.hasMore, + loadingMore: false, + }; + }); } catch (err) { if (controller.signal.aborted) return; + if (mode === 'append' && err instanceof Error && err.message.includes('Invalid cursor')) { + setState((prev) => { + if (prev.kind !== 'ready') return prev; + return { + ...prev, + loadingMore: false, + hasMore: false, + nextCursor: null, + }; + }); + return; + } setState({ kind: 'error', message: err instanceof Error ? err.message : 'Failed to load' }); } }, [documentId]); const handleClearCache = useCallback(async () => { if (!documentId || isClearing) return; - const confirmed = window.confirm('Clear cached segments for this document? This removes generated audio and metadata for listed segments.'); + const confirmed = window.confirm('Clear cached segments for this document version? This removes stored segment metadata and audio objects.'); if (!confirmed) return; setIsClearing(true); - setClearError(null); try { const res = await fetch('/api/tts/segments/clear', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ documentId }), }); + const payload = (await res.json().catch(() => null)) as { + error?: string; + deletedSegments?: number; + requestedAudioObjects?: number; + deletedAudioObjects?: number; + warning?: string; + } | null; if (!res.ok) { - const body = await res.text().catch(() => ''); - throw new Error(body || `Request failed (${res.status})`); + throw new Error(payload?.error || `Request failed (${res.status})`); } - await loadManifest(); + + if (payload?.warning) { + toast.error(`Segments cleared, but audio cleanup was partial: ${payload.warning}`); + } else if (payload) { + const deletedSegments = Number(payload.deletedSegments ?? 0); + const deletedAudioObjects = Number(payload.deletedAudioObjects ?? 0); + const requestedAudioObjects = Number(payload.requestedAudioObjects ?? deletedAudioObjects); + toast.success(`Cleared ${deletedSegments} segments and ${deletedAudioObjects}/${requestedAudioObjects} audio objects.`); + } + await loadManifest('reset'); } catch (error) { - setClearError(error instanceof Error ? error.message : 'Failed to clear segments cache'); + toast.error(error instanceof Error ? error.message : 'Failed to clear segments cache'); } finally { setIsClearing(false); } @@ -172,12 +304,31 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId }: SegmentsSideb useEffect(() => { if (!isOpen) return; - void loadManifest(); + void loadManifest('reset'); return () => { abortRef.current?.abort(); }; }, [isOpen, loadManifest]); + useEffect(() => { + if (!isOpen) return; + const node = listRef.current; + if (!node) return; + + const onScroll = () => { + setState((prev) => { + if (prev.kind !== 'ready' || !prev.hasMore || prev.loadingMore) return prev; + const distance = node.scrollHeight - node.scrollTop - node.clientHeight; + if (distance > 280) return prev; + void loadManifest('append', prev.nextCursor); + return { ...prev, loadingMore: true }; + }); + }; + + node.addEventListener('scroll', onScroll); + return () => node.removeEventListener('scroll', onScroll); + }, [isOpen, loadManifest]); + const handleSelectVariant = useCallback(async (settings: TTSSegmentSettings | null) => { if (!settings) return; await Promise.all([ @@ -189,61 +340,132 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId }: SegmentsSideb ]); }, [updateConfigKey]); - const handleJump = useCallback((index: number) => { - stopAndPlayFromIndex(index); - }, [stopAndPlayFromIndex]); + const handleRefresh = useCallback(() => { + didAutoScrollOnOpenRef.current = false; + void loadManifest('reset'); + }, [loadManifest]); - const segmentsByIndex = useMemo(() => { - if (state.kind !== 'ready') return new Map(); - const map = new Map(); - for (const row of state.data) { - const isCurrent = locatorMatchesCurrent(row.locator, currDocPage, currDocPageNumber); - const score = isCurrent ? 2 : row.locator ? 0 : 1; - if (score === 0) continue; + const handleJump = useCallback((index: number, locator: TTSSegmentLocator | null) => { + playFromSegment(index, locator); + }, [playFromSegment]); - const candidateUpdatedAt = latestUpdatedAt(row); - const existing = map.get(row.segmentIndex); - if (!existing) { - map.set(row.segmentIndex, { row, score, updatedAt: candidateUpdatedAt }); - continue; + const rowsToRender = useMemo(() => { + if (state.kind !== 'ready') return [] as Array<{ + segmentIndex: number; + sentenceText: string; + row: TTSSegmentRow; + isCurrentLocation: boolean; + groupKey: string; + groupLabel: string; + }>; + const currentRowsFromManifest = state.data.filter((row) => + locatorMatchesCurrent(row.locator, currDocPage, currDocPageNumber), + ); + const nonCurrentRows = state.data.filter((row) => + !locatorMatchesCurrent(row.locator, currDocPage, currDocPageNumber), + ); + + const inferredCurrentLocator = (() => { + const first = currentRowsFromManifest[0]?.locator; + if (first) return first; + if (typeof currDocPage === 'string' && currDocPage.length > 0) { + return { + location: currDocPage, + readerType: 'epub' as const, + }; } + if (typeof currDocPageNumber === 'number' && Number.isFinite(currDocPageNumber)) { + return { + page: Math.floor(currDocPageNumber), + readerType: 'pdf' as const, + }; + } + return null; + })(); - if (score > existing.score || (score === existing.score && candidateUpdatedAt >= existing.updatedAt)) { - map.set(row.segmentIndex, { row, score, updatedAt: candidateUpdatedAt }); + const variantsByIndex = new Map(); + for (const row of currentRowsFromManifest) { + if (!variantsByIndex.has(row.segmentIndex)) { + variantsByIndex.set(row.segmentIndex, []); + } + const merged = variantsByIndex.get(row.segmentIndex)!; + const seenIds = new Set(merged.map((variant) => variant.segmentId)); + for (const variant of row.variants ?? []) { + if (seenIds.has(variant.segmentId)) continue; + seenIds.add(variant.segmentId); + merged.push(variant); } } - const selected = new Map(); - for (const [idx, entry] of map) selected.set(idx, entry.row); - return selected; - }, [state, currDocPage, currDocPageNumber]); + const currentRows: TTSSegmentRow[] = sentences.map((_, segmentIndex) => ({ + segmentIndex, + locator: inferredCurrentLocator, + variants: variantsByIndex.get(segmentIndex) ?? [], + })); - const indicesToRender = useMemo(() => { - const indices = new Set(); - for (let i = 0; i < sentences.length; i += 1) indices.add(i); - if (state.kind === 'ready') { - for (const row of state.data) { - if (Number.isInteger(row.segmentIndex) && row.segmentIndex >= 0) { - indices.add(row.segmentIndex); - } - } - } - return Array.from(indices).sort((a, b) => a - b); - }, [sentences, state]); - - const rowsToRender: Array<{ segmentIndex: number; sentenceText: string; row: TTSSegmentRow | null }> = []; - for (const i of indicesToRender) { - rowsToRender.push({ - segmentIndex: i, - sentenceText: sentences[i] ?? '', - row: segmentsByIndex.get(i) ?? null, + const mergedRows = [...currentRows, ...nonCurrentRows].sort(compareRows); + return mergedRows.map((row) => { + const isCurrentLocation = locatorMatchesCurrent(row.locator, currDocPage, currDocPageNumber); + return { + segmentIndex: row.segmentIndex, + sentenceText: isCurrentLocation ? (sentences[row.segmentIndex] ?? '') : '', + row, + isCurrentLocation, + groupKey: locatorGroupKey(row.locator), + groupLabel: formatLocatorGroupLabel(row.locator), + }; }); - } + }, [state, currDocPage, currDocPageNumber, sentences]); const totalVariants = state.kind === 'ready' - ? state.data.reduce((sum, r) => sum + r.variants.length, 0) + ? rowsToRender.reduce((sum, r) => sum + r.row.variants.length, 0) : 0; + useEffect(() => { + if (!isOpen) { + didAutoScrollOnOpenRef.current = false; + return; + } + if (didAutoScrollOnOpenRef.current) return; + if (state.kind !== 'ready' || rowsToRender.length === 0) return; + + const container = listRef.current; + if (!container) return; + const activeRow = container.querySelector('[data-active-segment="true"]'); + if (!activeRow) return; + + requestAnimationFrame(() => { + activeRow.scrollIntoView({ block: 'center', behavior: 'auto' }); + didAutoScrollOnOpenRef.current = true; + }); + }, [isOpen, state.kind, rowsToRender.length, currentSentenceIndex]); + + useEffect(() => { + if (!isOpen || !isPlaying) return; + if (state.kind !== 'ready' || rowsToRender.length === 0) return; + + const root = listRef.current; + if (!root) return; + const activeRow = root.querySelector('[data-active-segment="true"]'); + if (!activeRow) return; + + const scrollContainer = findScrollableAncestor(activeRow, root); + if (!scrollContainer) return; + if (isElementFullyVisibleWithinContainer(activeRow, scrollContainer)) return; + + requestAnimationFrame(() => { + activeRow.scrollIntoView({ block: 'center', behavior: 'auto' }); + }); + }, [ + isOpen, + isPlaying, + state.kind, + rowsToRender.length, + currentSentenceIndex, + currDocPage, + currDocPageNumber, + ]); + const headerActions = ( <>
- {row && ( - - )} +
); })} )} + {state.kind === 'ready' && state.loadingMore && ( +
Loading more segments…
+ )} ); diff --git a/src/components/views/EPUBViewer.tsx b/src/components/views/EPUBViewer.tsx index 615ec0b..43f4798 100644 --- a/src/components/views/EPUBViewer.tsx +++ b/src/components/views/EPUBViewer.tsx @@ -36,10 +36,12 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { highlightPattern, clearHighlights, highlightWordIndex, - clearWordHighlights + clearWordHighlights, + walkUpcomingRenderedLocations, } = useEPUB(); const { registerLocationChangeHandler, + registerEpubLocationWalker, pause, currentSentence, currentSentenceAlignment, @@ -71,7 +73,12 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { // Register the location change handler useEffect(() => { registerLocationChangeHandler(handleLocationChanged); - }, [registerLocationChangeHandler, handleLocationChanged]); + registerEpubLocationWalker(walkUpcomingRenderedLocations); + return () => { + registerLocationChangeHandler(null); + registerEpubLocationWalker(null); + }; + }, [registerLocationChangeHandler, registerEpubLocationWalker, handleLocationChanged, walkUpcomingRenderedLocations]); // Handle highlighting useEffect(() => { @@ -132,7 +139,7 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {