From 733b4180eb72d0234f8cff16fff11a51db1e11de Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Fri, 21 Nov 2025 19:22:54 -0700 Subject: [PATCH 1/9] feat(tts): add word-by-word highlighting with whisper.cpp - Introduce `/api/whisper` endpoint which uses `whisper.cpp` (via a `WHISPER_CPP_BIN` executable) and `ffmpeg` to generate word-level audio alignments from provided audio and text. - Integrate word-level alignments into `TTSContext`, tracking the currently spoken word based on audio seek position and provided timestamps. Alignments are cached in-memory and fetched asynchronously. - Add new configuration options (`pdfWordHighlightEnabled`, `epubWordHighlightEnabled`) to `ConfigContext` and `Dexie` for enabling/disabling the feature. - Implement visual word highlighting in both `PDFViewer` and `EPUBViewer` by mapping TTS-aligned words to rendered text elements. - Enhance `EPUBContext` and `PDFContext` with new `highlightWordIndex` and `clearWordHighlights` functions, utilizing fuzzy string matching (`cmpstr`) to robustly align spoken words with displayed text for accurate highlighting. - Update `DocumentSettings` to include user-facing toggles for the new highlighting modes. --- src/app/api/whisper/route.ts | 451 ++++++++++++++++++++++++++++ src/components/DocumentSettings.tsx | 98 ++++-- src/components/EPUBViewer.tsx | 48 ++- src/components/PDFViewer.tsx | 46 ++- src/contexts/ConfigContext.tsx | 8 +- src/contexts/EPUBContext.tsx | 302 ++++++++++++++++++- src/contexts/PDFContext.tsx | 26 +- src/contexts/TTSContext.tsx | 160 +++++++++- src/lib/dexie.ts | 6 + src/lib/pdf.ts | 406 +++++++++++++------------ src/types/config.ts | 4 + src/types/tts.ts | 18 +- 12 files changed, 1333 insertions(+), 240 deletions(-) create mode 100644 src/app/api/whisper/route.ts diff --git a/src/app/api/whisper/route.ts b/src/app/api/whisper/route.ts new file mode 100644 index 0000000..146e452 --- /dev/null +++ b/src/app/api/whisper/route.ts @@ -0,0 +1,451 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { createHash, randomUUID } from 'crypto'; +import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { spawn } from 'child_process'; +import type { TTSSentenceAlignment } from '@/types/tts'; +import { preprocessSentenceForAudio } from '@/lib/nlp'; + +export const runtime = 'nodejs'; + +interface WhisperRequestBody { + text: string; + audio: number[]; // raw bytes from Uint8Array + lang?: string; +} + +interface WhisperAlignmentOptions { + engine?: 'whisper.cpp'; + lang?: string; +} + +interface WhisperWord { + start: number; + end: number; + word: string; +} + +// Simple in-memory cache keyed by a hash of text+lang+audio length +const alignmentCache = new Map(); + +// Model management using a fixed tiny.en GGML model stored under docstore/model +const MODEL_NAME = 'ggml-tiny.en.bin'; +const MODEL_URL = + 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin'; +const DOCSTORE_DIR = join(process.cwd(), 'docstore'); +const MODEL_DIR = join(DOCSTORE_DIR, 'model'); +const MODEL_PATH = join(MODEL_DIR, MODEL_NAME); +const modelReadyPromises = new Map>(); + +async function ensureModelAvailable(): Promise { + // Fast path: model already present + try { + await access(MODEL_PATH); + return; + } catch { + // fall through to download + } + + const existing = modelReadyPromises.get(MODEL_PATH); + if (existing) return existing; + + const promise = (async () => { + try { + await access(MODEL_PATH); + return; + } catch { + // still missing + } + + await mkdir(MODEL_DIR, { recursive: true }); + + const res = await fetch(MODEL_URL); + if (!res.ok) { + throw new Error( + `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}` + ); + } + + const arrayBuffer = await res.arrayBuffer(); + await writeFile(MODEL_PATH, Buffer.from(arrayBuffer)); + })(); + + modelReadyPromises.set(MODEL_PATH, promise); + return promise; +} + +async function runWhisperCpp( + wavPath: string, + opts: WhisperAlignmentOptions +): Promise { + const binary = process.env.WHISPER_CPP_BIN; + if (!binary) { + throw new Error( + 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.' + ); + } + + await ensureModelAvailable(); + + return new Promise((resolve, reject) => { + const jsonBase = `${wavPath}.json_out`; + const jsonPath = `${jsonBase}.json`; + // Request full JSON output (including token-level details when available) + // and ask whisper-cli not to print anything to stdout. + const args = [ + '-m', + MODEL_PATH, + '-f', + wavPath, + '-of', + jsonBase, + '-ojf', + '-np', + ]; + + if (opts.lang) { + args.push('-l', opts.lang); + } + + const child = spawn(binary, args); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('error', (err) => { + reject(err); + }); + + child.on('close', (code) => { + if (code !== 0) { + console.error('whisper-cli exited with error', { + code, + wavPath, + stdout: stdout.slice(0, 500), + stderr: stderr.slice(0, 500), + }); + return reject( + new Error( + `whisper.cpp exited with code ${code}: ${stderr || stdout}` + ) + ); + } + + readFile(jsonPath, 'utf-8') + .then((content: string) => { + const words: WhisperWord[] = []; + const parsed = JSON.parse(content) as { + transcription?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + tokens?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + }>; + }>; + }; + + const transcription = parsed.transcription; + + const parseTimecode = (value?: string): number | null => { + if (!value) return null; + const m = value.match(/(\d+):(\d+):(\d+),(\d+)/); + if (!m) return null; + const h = Number(m[1]); + const min = Number(m[2]); + const s = Number(m[3]); + const ms = Number(m[4]); + if ( + Number.isNaN(h) || + Number.isNaN(min) || + Number.isNaN(s) || + Number.isNaN(ms) + ) { + return null; + } + return h * 3600 + min * 60 + s + ms / 1000; + }; + + if (Array.isArray(transcription)) { + for (const seg of transcription) { + const segText = (seg.text || '').trim(); + const segStartSecFromTs = parseTimecode( + seg.timestamps?.from + ); + const segEndSecFromTs = parseTimecode(seg.timestamps?.to); + const segStartSecFromMs = + typeof seg.offsets?.from === 'number' + ? seg.offsets.from / 1000 + : null; + const segEndSecFromMs = + typeof seg.offsets?.to === 'number' + ? seg.offsets.to / 1000 + : null; + + const segStartSec = + segStartSecFromTs ?? + segStartSecFromMs ?? + 0; + const segEndSec = + segEndSecFromTs ?? + segEndSecFromMs ?? + segStartSec; + + const tokens = Array.isArray(seg.tokens) + ? seg.tokens + : []; + + if (tokens.length > 0) { + for (const token of tokens) { + const rawText = token.text || ''; + const tokenText = rawText.trim(); + // Skip special markers like [_BEG_] + if (!tokenText || /^\[.*\]$/.test(tokenText)) continue; + + const tokStartSecFromTs = parseTimecode( + token.timestamps?.from + ); + const tokEndSecFromTs = parseTimecode( + token.timestamps?.to + ); + const tokStartSecFromMs = + typeof token.offsets?.from === 'number' + ? token.offsets.from / 1000 + : null; + const tokEndSecFromMs = + typeof token.offsets?.to === 'number' + ? token.offsets.to / 1000 + : null; + + const startSec = + tokStartSecFromTs ?? + tokStartSecFromMs ?? + segStartSec; + const endSec = + tokEndSecFromTs ?? + tokEndSecFromMs ?? + segEndSec; + + words.push({ + word: tokenText, + start: startSec, + end: endSec, + }); + } + } else if (segText) { + // Fallback: no token list, approximate per-word timing within the segment + const segTokens = segText.split(/\s+/).filter(Boolean); + if (segTokens.length) { + const totalDur = Math.max(segEndSec - segStartSec, 0); + const step = + segTokens.length > 0 + ? totalDur / segTokens.length + : 0; + segTokens.forEach((token, index) => { + const wStart = + step > 0 + ? segStartSec + step * index + : segStartSec; + const wEnd = + step > 0 + ? index === segTokens.length - 1 + ? segEndSec + : segStartSec + step * (index + 1) + : segEndSec; + words.push({ + word: token, + start: wStart, + end: wEnd, + }); + }); + } + } + } + } + + resolve(words); + }) + .catch((err: unknown) => { + reject(err); + }); + }); + }); +} + +function mapWordsToSentenceOffsets( + sentence: string, + words: WhisperWord[] +): TTSSentenceAlignment { + const normalizedSentence = preprocessSentenceForAudio(sentence); + let cursor = 0; + + const alignedWords = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = normalizedSentence + .toLowerCase() + .indexOf(token.toLowerCase(), cursor); + + const start = + idx !== -1 + ? idx + : cursor; + const end = start + token.length; + + cursor = end; + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords.filter((w) => w.text.length > 0), + }; +} + +async function alignAudioWithText( + audioBuffer: ArrayBuffer, + text: string, + cacheKey?: string, + opts: WhisperAlignmentOptions = {} +): Promise { + if (!text.trim()) { + return []; + } + + if (cacheKey && alignmentCache.has(cacheKey)) { + return alignmentCache.get(cacheKey)!; + } + + const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); + const inputPath = join(tmpBase, `${randomUUID()}-input.bin`); + const wavPath = join(tmpBase, `${randomUUID()}-input.wav`); + + try { + await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); + + await new Promise((resolve, reject) => { + const ffmpeg = spawn('ffmpeg', [ + '-y', + '-i', + inputPath, + '-ar', + '16000', + '-ac', + '1', + wavPath, + ]); + + let stderr = ''; + ffmpeg.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('error', (err) => { + reject(err); + }); + + ffmpeg.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject( + new Error(`ffmpeg failed with code ${code}: ${stderr}`) + ); + } + }); + }); + + const words = await runWhisperCpp(wavPath, opts); + const alignment = mapWordsToSentenceOffsets(text, words); + const result: TTSSentenceAlignment[] = [alignment]; + + if (cacheKey) { + alignmentCache.set(cacheKey, result); + } + + return result; + } finally { + try { + await rm(tmpBase, { recursive: true, force: true }); + } catch { + // ignore + } + } +} + +function makeCacheKey(input: WhisperRequestBody) { + const hash = createHash('sha256') + .update( + JSON.stringify({ + text: input.text, + lang: input.lang || '', + audioLen: input.audio?.length || 0, + }) + ) + .digest('hex'); + return hash; +} + +export async function POST(req: NextRequest) { + try { + const body = (await req.json()) as WhisperRequestBody; + const { text, audio, lang } = body; + + if (!text || !audio || !Array.isArray(audio)) { + return NextResponse.json( + { error: 'Missing text or audio in request body' }, + { status: 400 } + ); + } + + const cacheKey = makeCacheKey(body); + const audioBuffer = new Uint8Array(audio).buffer; + + const alignments: TTSSentenceAlignment[] = await alignAudioWithText( + audioBuffer, + text, + cacheKey, + { engine: 'whisper.cpp', lang } + ); + + return NextResponse.json({ alignments }, { status: 200 }); + } catch (error) { + console.error('Error in whisper route:', error); + return NextResponse.json( + { + error: 'WHISPER_ALIGNMENT_FAILED', + message: 'Failed to compute word-level alignment', + }, + { status: 500 } + ); + } +} + diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index b216730..76a3e7f 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -35,6 +35,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { updateConfigKey, pdfHighlightEnabled, epubHighlightEnabled, + pdfWordHighlightEnabled, + epubWordHighlightEnabled, } = useConfig(); const { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter } = useEPUB(); const { createFullAudioBook: createPDFAudioBook, regenerateChapter: regeneratePDFChapter } = usePDF(); @@ -324,40 +326,82 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {

- Merge sentences across page or section breaks for smoother TTS. + Merge sentences across page or section breaks

)} {!epub && !html && ( -
- -

- Show visual highlighting in the PDF viewer while TTS is reading. -

+
+
+ +

+ Visual text playback highlighting in the PDF viewer +

+
+
+ +

+ Highlight individual words using audio timestamps generated by whisper.cpp +

+
)} {epub && ( -
- -

- Show visual highlighting in the EPUB viewer while TTS is reading. -

+
+
+ +

+ Visual text playback highlighting in the EPUB viewer +

+
+
+ +

+ Highlight individual words using audio timestamps generated by whisper.cpp +

+
)} {epub && ( diff --git a/src/components/EPUBViewer.tsx b/src/components/EPUBViewer.tsx index 03e101a..7f0e5c3 100644 --- a/src/components/EPUBViewer.tsx +++ b/src/components/EPUBViewer.tsx @@ -30,10 +30,18 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { setRendition, extractPageText, highlightPattern, - clearHighlights + clearHighlights, + highlightWordIndex, + clearWordHighlights } = useEPUB(); - const { registerLocationChangeHandler, pause, currentSentence } = useTTS(); - const { epubTheme } = useConfig(); + const { + registerLocationChangeHandler, + pause, + currentSentence, + currentSentenceAlignment, + currentWordIndex + } = useTTS(); + const { epubTheme, epubHighlightEnabled, epubWordHighlightEnabled } = useConfig(); const { updateTheme } = useEPUBTheme(epubTheme, renditionRef.current); const containerRef = useRef(null); const { isResizing, setIsResizing, dimensions } = useEPUBResize(containerRef); @@ -70,6 +78,38 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { } }, [currentSentence, highlightPattern, clearHighlights]); + // Word-level highlight layered on top of the block highlight + useEffect(() => { + if (!epubHighlightEnabled || !epubWordHighlightEnabled) { + clearWordHighlights(); + return; + } + + if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { + clearWordHighlights(); + return; + } + + if (!currentSentenceAlignment) { + clearWordHighlights(); + return; + } + + highlightWordIndex( + currentSentenceAlignment, + currentWordIndex, + currentSentence || '' + ); + }, [ + currentWordIndex, + currentSentence, + currentSentenceAlignment, + epubHighlightEnabled, + epubWordHighlightEnabled, + clearWordHighlights, + highlightWordIndex + ]); + if (!currDocData) { return ; } @@ -95,4 +135,4 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
); -} \ No newline at end of file +} diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index f3fb414..de88b1d 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -26,11 +26,13 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { const { containerWidth } = usePDFResize(containerRef); // Config context - const { viewType, pdfHighlightEnabled } = useConfig(); + const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig(); // TTS context const { currentSentence, + currentWordIndex, + currentSentenceAlignment, skipToLocation, } = useTTS(); @@ -38,6 +40,8 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { const { highlightPattern, clearHighlights, + clearWordHighlights, + highlightWordIndex, onDocumentLoadSuccess, currDocData, currDocPages, @@ -74,6 +78,46 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { }; }, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]); + // Word-level highlight layered on top of the block highlight + useEffect(() => { + if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) { + clearWordHighlights(); + return; + } + + if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { + clearWordHighlights(); + return; + } + + const wordEntry = + currentSentenceAlignment && + currentWordIndex < currentSentenceAlignment.words.length + ? currentSentenceAlignment.words[currentWordIndex] + : undefined; + const wordText = wordEntry?.text || null; + + if (!wordText) { + clearWordHighlights(); + return; + } + + highlightWordIndex( + currentSentenceAlignment, + currentWordIndex, + currentSentence || '', + containerRef as RefObject + ); + }, [ + currentWordIndex, + currentSentence, + currentSentenceAlignment, + pdfHighlightEnabled, + pdfWordHighlightEnabled, + clearWordHighlights, + highlightWordIndex + ]); + // Add page dimensions state const [pageWidth, setPageWidth] = useState(595); // default A4 width const [pageHeight, setPageHeight] = useState(842); // default A4 height diff --git a/src/contexts/ConfigContext.tsx b/src/contexts/ConfigContext.tsx index f560b93..5089775 100644 --- a/src/contexts/ConfigContext.tsx +++ b/src/contexts/ConfigContext.tsx @@ -32,7 +32,9 @@ interface ConfigContextType { isLoading: boolean; isDBReady: boolean; pdfHighlightEnabled: boolean; + pdfWordHighlightEnabled: boolean; epubHighlightEnabled: boolean; + epubWordHighlightEnabled: boolean; } const ConfigContext = createContext(undefined); @@ -103,7 +105,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) { savedVoices, smartSentenceSplitting, pdfHighlightEnabled, + pdfWordHighlightEnabled, epubHighlightEnabled, + epubWordHighlightEnabled, } = config || APP_CONFIG_DEFAULTS; /** @@ -201,7 +205,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) { isLoading, isDBReady, pdfHighlightEnabled, - epubHighlightEnabled + pdfWordHighlightEnabled, + epubHighlightEnabled, + epubWordHighlightEnabled }}> {children} diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index 46ab2bf..1922d2b 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -22,7 +22,13 @@ import { createRangeCfi } from '@/lib/epub'; import { useParams } from 'next/navigation'; import { useConfig } from './ConfigContext'; import { withRetry } from '@/utils/audio'; -import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts'; +import { CmpStr } from 'cmpstr'; +import type { + TTSRequestHeaders, + TTSRequestPayload, + TTSRetryOptions, + TTSSentenceAlignment +} from '@/types/tts'; interface EPUBContextType { currDocData: ArrayBuffer | undefined; @@ -44,12 +50,26 @@ interface EPUBContextType { isAudioCombining: boolean; highlightPattern: (text: string) => void; clearHighlights: () => void; + highlightWordIndex: ( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined + ) => void; + clearWordHighlights: () => void; } const EPUBContext = createContext(undefined); const EPUB_CONTINUATION_CHARS = 600; +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); + +const normalizeWordForMatch = (text: string): string => + text + .trim() + .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '') + .toLowerCase(); + const stepToNextNode = (node: Node | null, root: Node): Node | null => { if (!node) return null; if (node.firstChild) { @@ -160,6 +180,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const shouldPauseRef = useRef(true); // Track current highlight CFI for removal const currentHighlightCfi = useRef(null); + const currentWordHighlightCfi = useRef(null); /** * Clears all current document state and stops any active TTS @@ -711,15 +732,22 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } }, [id, skipToLocation, extractPageText, setIsEPUB]); - const clearHighlights = useCallback(() => { - if (renditionRef.current) { - if (currentHighlightCfi.current) { - renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight'); - currentHighlightCfi.current = null; - } + const clearWordHighlights = useCallback(() => { + if (!renditionRef.current) return; + if (currentWordHighlightCfi.current) { + renditionRef.current.annotations.remove(currentWordHighlightCfi.current, 'highlight'); + currentWordHighlightCfi.current = null; } }, []); + const clearHighlights = useCallback(() => { + if (renditionRef.current && currentHighlightCfi.current) { + renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight'); + currentHighlightCfi.current = null; + } + clearWordHighlights(); + }, [clearWordHighlights]); + const highlightPattern = useCallback(async (text: string) => { if (!renditionRef.current) return; @@ -772,6 +800,262 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } }, [epubHighlightEnabled, clearHighlights]); + const highlightWordIndex = useCallback(( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined + ) => { + clearWordHighlights(); + + if (!epubHighlightEnabled) return; + if (!alignment) return; + if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return; + + const words = alignment.words || []; + if (!words.length || wordIndex >= words.length) return; + + if (!renditionRef.current) return; + if (!currentHighlightCfi.current) return; + + const cleanSentence = + sentence && sentence.trim() + ? sentence.trim().replace(/\s+/g, ' ') + : null; + if (!cleanSentence) return; + + const alignmentSentenceClean = alignment.sentence + ? alignment.sentence.trim().replace(/\s+/g, ' ') + : null; + if (!alignmentSentenceClean || alignmentSentenceClean !== cleanSentence) { + return; + } + + const contents = renditionRef.current.getContents(); + const contentsArray = Array.isArray(contents) ? contents : [contents]; + + for (const content of contentsArray) { + let range: Range | null = null; + try { + range = content.range(currentHighlightCfi.current as string); + } catch { + range = null; + } + if (!range) continue; + + const root = range.commonAncestorContainer; + if (!root) continue; + + const domTokens: Array<{ + node: Text; + startOffset: number; + endOffset: number; + norm: string; + }> = []; + + const addTokensFromNode = (textNode: Text, start: number, end: number) => { + const full = textNode.textContent || ''; + const safeStart = Math.max(0, Math.min(start, full.length)); + const safeEnd = Math.max(safeStart, Math.min(end, full.length)); + if (safeEnd <= safeStart) return; + + const slice = full.slice(safeStart, safeEnd); + const wordRegex = /\S+/g; + let match: RegExpExecArray | null; + while ((match = wordRegex.exec(slice)) !== null) { + const raw = match[0]; + const norm = normalizeWordForMatch(raw); + if (!norm) continue; + const tokenStart = safeStart + match.index; + const tokenEnd = tokenStart + raw.length; + domTokens.push({ + node: textNode, + startOffset: tokenStart, + endOffset: tokenEnd, + norm, + }); + } + }; + + const nextTextNode = (node: Node | null): Text | null => { + let next = getNextTextNode(node, root); + while (next) { + if (next.nodeType === Node.TEXT_NODE) { + return next as Text; + } + next = getNextTextNode(next, root); + } + return null; + }; + + // Collect tokens within the sentence range + if (range.startContainer === range.endContainer && range.startContainer.nodeType === Node.TEXT_NODE) { + addTokensFromNode(range.startContainer as Text, range.startOffset, range.endOffset); + } else { + let current: Text | null = null; + + if (range.startContainer.nodeType === Node.TEXT_NODE) { + const startText = range.startContainer as Text; + const isEnd = range.endContainer === startText && range.endContainer.nodeType === Node.TEXT_NODE; + const endOffset = isEnd ? range.endOffset : (startText.textContent || '').length; + addTokensFromNode(startText, range.startOffset, endOffset); + if (isEnd) { + current = null; + } else { + current = nextTextNode(startText); + } + } else { + current = nextTextNode(range.startContainer); + } + + while (current) { + if (range.endContainer.nodeType === Node.TEXT_NODE && current === range.endContainer) { + addTokensFromNode(current, 0, range.endOffset); + break; + } else { + addTokensFromNode(current, 0, (current.textContent || '').length); + } + current = nextTextNode(current); + } + } + + if (!domTokens.length) { + return; + } + + const domFiltered: Array<{ tokenIndex: number; norm: string }> = []; + for (let i = 0; i < domTokens.length; i++) { + const norm = domTokens[i].norm; + if (!norm) continue; + domFiltered.push({ tokenIndex: i, norm }); + } + + const ttsFiltered: Array<{ wordIndex: number; norm: string }> = []; + for (let i = 0; i < words.length; i++) { + const norm = normalizeWordForMatch(words[i].text); + if (!norm) continue; + ttsFiltered.push({ wordIndex: i, norm }); + } + + const wordToToken = new Array(words.length).fill(-1); + const m = domFiltered.length; + const n = ttsFiltered.length; + + if (m && n) { + const dp: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(Number.POSITIVE_INFINITY) + ); + const bt: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(0) + ); // 0=diag,1=up,2=left + + dp[0][0] = 0; + const GAP_COST = 0.7; + + for (let i = 0; i <= m; i++) { + for (let j = 0; j <= n; j++) { + if (i > 0 && j > 0) { + const a = domFiltered[i - 1].norm; + const b = ttsFiltered[j - 1].norm; + const sim = cmp.compare(a, b); + const subCost = 1 - sim; + const cand = dp[i - 1][j - 1] + subCost; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 0; + } + } + if (i > 0) { + const cand = dp[i - 1][j] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 1; + } + } + if (j > 0) { + const cand = dp[i][j - 1] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 2; + } + } + } + } + + let i = m; + let j = n; + while (i > 0 || j > 0) { + const move = bt[i][j]; + if (i > 0 && j > 0 && move === 0) { + const domIdx = domFiltered[i - 1].tokenIndex; + const ttsIdx = ttsFiltered[j - 1].wordIndex; + if (wordToToken[ttsIdx] === -1) { + wordToToken[ttsIdx] = domIdx; + } + i -= 1; + j -= 1; + } else if (i > 0 && (move === 1 || j === 0)) { + i -= 1; + } else if (j > 0 && (move === 2 || i === 0)) { + j -= 1; + } else { + break; + } + } + + // Propagate nearest known mapping to fill gaps + let lastSeen = -1; + for (let k = 0; k < wordToToken.length; k++) { + if (wordToToken[k] !== -1) { + lastSeen = wordToToken[k]; + } else if (lastSeen !== -1) { + wordToToken[k] = lastSeen; + } + } + let nextSeen = -1; + for (let k = wordToToken.length - 1; k >= 0; k--) { + if (wordToToken[k] !== -1) { + nextSeen = wordToToken[k]; + } else if (nextSeen !== -1) { + wordToToken[k] = nextSeen; + } + } + } + + const mappedIndex = + wordIndex < wordToToken.length ? wordToToken[wordIndex] : -1; + if (mappedIndex === -1) { + return; + } + + const token = domTokens[mappedIndex]; + const doc = token.node.ownerDocument || (range.commonAncestorContainer as Document); + const wordRange = doc.createRange(); + wordRange.setStart(token.node, token.startOffset); + wordRange.setEnd(token.node, token.endOffset); + + try { + const wordCfi = content.cfiFromRange(wordRange); + currentWordHighlightCfi.current = wordCfi; + renditionRef.current.annotations.add( + 'highlight', + wordCfi, + {}, + () => {}, + '', + { + fill: 'var(--accent)', + 'fill-opacity': '0.4', + 'mix-blend-mode': 'multiply', + } + ); + } catch (error) { + console.error('Error highlighting EPUB word:', error); + } + + break; + } + }, [epubHighlightEnabled, clearWordHighlights]); + // Context value memoization @@ -796,6 +1080,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) { isAudioCombining, highlightPattern, clearHighlights, + highlightWordIndex, + clearWordHighlights, }), [ setCurrentDocument, @@ -813,6 +1099,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) { isAudioCombining, highlightPattern, clearHighlights, + highlightWordIndex, + clearWordHighlights, ] ); diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index d555e9b..dfdb6b1 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -36,10 +36,16 @@ import { extractTextFromPDF, highlightPattern, clearHighlights, - handleTextClick, + clearWordHighlights, + highlightWordIndex, } from '@/lib/pdf'; -import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts'; +import type { + TTSRequestHeaders, + TTSRequestPayload, + TTSRetryOptions, + TTSSentenceAlignment +} from '@/types/tts'; /** * Interface defining all available methods and properties in the PDF context @@ -59,13 +65,12 @@ interface PDFContextType { onDocumentLoadSuccess: (pdf: PDFDocumentProxy) => void; highlightPattern: (text: string, pattern: string, containerRef: RefObject) => void; clearHighlights: () => void; - handleTextClick: ( - event: MouseEvent, - pdfText: string, - containerRef: RefObject, - stopAndPlayFromIndex: (index: number) => void, - isProcessing: boolean, - enableHighlight?: boolean + clearWordHighlights: () => void; + highlightWordIndex: ( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined, + containerRef: RefObject ) => void; createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise; regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>; @@ -655,7 +660,8 @@ export function PDFProvider({ children }: { children: ReactNode }) { clearCurrDoc, highlightPattern, clearHighlights, - handleTextClick, + clearWordHighlights, + highlightWordIndex, pdfDocument, createFullAudioBook, regenerateChapter, diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 9c52ed3..4124f13 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -47,6 +47,7 @@ import type { TTSRequestPayload, TTSRequestHeaders, TTSRetryOptions, + TTSSentenceAlignment, } from '@/types/tts'; // Media globals @@ -63,6 +64,10 @@ interface TTSContextType extends TTSPlaybackState { // Voice settings availableVoices: string[]; + // Alignment metadata for the current sentence + currentSentenceAlignment?: TTSSentenceAlignment; + currentWordIndex?: number | null; + // Control functions togglePlay: () => void; skipForward: () => void; @@ -264,6 +269,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement updateConfigKey, skipBlank, smartSentenceSplitting, + pdfHighlightEnabled, + pdfWordHighlightEnabled, + epubHighlightEnabled, + epubWordHighlightEnabled, } = useConfig(); // Audio and voice management hooks @@ -331,6 +340,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const epubContinuationRef = useRef(null); const pageTurnEstimateRef = useRef(null); const pageTurnTimeoutRef = useRef | null>(null); + const sentenceAlignmentCacheRef = useRef>(new Map()); + const [currentSentenceAlignment, setCurrentSentenceAlignment] = useState(); + const [currentWordIndex, setCurrentWordIndex] = useState(null); + const sentencesRef = useRef([]); + const currentIndexRef = useRef(0); + + useEffect(() => { + sentencesRef.current = sentences; + }, [sentences]); + + useEffect(() => { + currentIndexRef.current = currentIndex; + }, [currentIndex]); /** * Processes text into sentences using the shared NLP utility @@ -370,6 +392,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement clearTimeout(pageTurnTimeoutRef.current); pageTurnTimeoutRef.current = null; } + setCurrentWordIndex(null); }, [activeHowl]); /** @@ -558,6 +581,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setCurrentIndex(0); } + // Reset alignment state whenever the text block changes + sentenceAlignmentCacheRef.current.clear(); + setCurrentSentenceAlignment(undefined); + setCurrentWordIndex(null); + // Compute auto page-turn estimate for PDFs when we have a continuation if (smartSentenceSplitting && !isEPUB && continuationCarried && normalizedOptions.nextLocation !== undefined) { const continuationNormalized = preprocessSentenceForAudio(continuationCarried); @@ -706,10 +734,61 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @returns {Promise} The generated audio buffer */ const getAudio = useCallback(async (sentence: string): Promise => { + const alignmentEnabledForCurrentDoc = + (!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) || + (isEPUB && epubHighlightEnabled && epubWordHighlightEnabled); + // Helper to ensure we have an alignment for a given + // sentence/audio pair, even when the audio itself is + // served from the local cache. + const ensureAlignment = (arrayBuffer: ArrayBuffer) => { + if (!alignmentEnabledForCurrentDoc) return; + if (sentenceAlignmentCacheRef.current.has(sentence)) return; + + try { + const audioBytes = Array.from(new Uint8Array(arrayBuffer)); + const alignmentBody = { + text: sentence, + audio: audioBytes, + }; + + void fetch('/api/whisper', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(alignmentBody), + }) + .then(async (res) => { + if (!res.ok) return; + const data = await res.json().catch(() => null); + if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) { + return; + } + const alignment = data.alignments[0] as TTSSentenceAlignment; + sentenceAlignmentCacheRef.current.set(sentence, alignment); + + const currentSentence = sentencesRef.current[currentIndexRef.current]; + if (currentSentence === sentence) { + setCurrentSentenceAlignment(alignment); + setCurrentWordIndex(null); + } + }) + .catch((err) => { + console.warn('Alignment request failed:', err); + }); + } catch (err) { + console.warn('Failed to start alignment request:', err); + } + }; + // Check if the audio is already cached const cachedAudio = audioCache.get(sentence); if (cachedAudio) { console.log('Using cached audio for sentence:', sentence.substring(0, 20)); + // If we have audio but no alignment (e.g. after a + // navigation or TTS reset), kick off a fresh alignment + // request using the cached audio buffer. + ensureAlignment(cachedAudio); return cachedAudio; } @@ -769,6 +848,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Cache the array buffer audioCache.set(sentence, arrayBuffer); + // Fire-and-forget alignment request; do not block audio playback + ensureAlignment(arrayBuffer); + return arrayBuffer; } catch (error) { // Check if this was an abort error @@ -788,7 +870,21 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement }); throw error; } - }, [voice, speed, ttsModel, ttsInstructions, audioCache, openApiKey, openApiBaseUrl, configTTSProvider]); + }, [ + voice, + speed, + ttsModel, + ttsInstructions, + audioCache, + openApiKey, + openApiBaseUrl, + configTTSProvider, + isEPUB, + pdfHighlightEnabled, + pdfWordHighlightEnabled, + epubHighlightEnabled, + epubWordHighlightEnabled + ]); /** * Processes and plays the current sentence @@ -1004,7 +1100,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement }, [isPlaying, advance, activeHowl, processSentence, audioSpeed]); const playAudio = useCallback(async () => { - const howl = await playSentenceWithHowl(sentences[currentIndex], currentIndex); + const sentence = sentences[currentIndex]; + const cachedAlignment = sentenceAlignmentCacheRef.current.get(sentence); + if (cachedAlignment) { + setCurrentSentenceAlignment(cachedAlignment); + setCurrentWordIndex(null); + } else { + setCurrentSentenceAlignment(undefined); + setCurrentWordIndex(null); + } + + const howl = await playSentenceWithHowl(sentence, currentIndex); if (howl) { howl.play(); } @@ -1017,6 +1123,47 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement playAudio, }); + // Track the current word index during playback using Howler's seek position + useEffect(() => { + if (!activeHowl || !isPlaying || !currentSentenceAlignment || !currentSentenceAlignment.words.length) { + setCurrentWordIndex(null); + return; + } + + let frameId: number; + + const tick = () => { + try { + const pos = activeHowl.seek() as number; + if (typeof pos === 'number' && Number.isFinite(pos)) { + const words = currentSentenceAlignment.words; + let idx = -1; + for (let i = 0; i < words.length; i++) { + const w = words[i]; + if (pos >= w.startSec && pos < w.endSec) { + idx = i; + break; + } + } + if (idx !== -1) { + setCurrentWordIndex((prev) => (prev === idx ? prev : idx)); + } + } + } catch { + // ignore seek errors + } + frameId = requestAnimationFrame(tick); + }; + + frameId = requestAnimationFrame(tick); + + return () => { + if (frameId) { + cancelAnimationFrame(frameId); + } + }; + }, [activeHowl, isPlaying, currentSentenceAlignment]); + /** * Preloads the next sentence's audio */ @@ -1085,6 +1232,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setCurrDocPages(undefined); setIsProcessing(false); setIsEPUB(false); + sentenceAlignmentCacheRef.current.clear(); + setCurrentSentenceAlignment(undefined); + setCurrentWordIndex(null); }, [abortAudio]); /** @@ -1205,6 +1355,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement isProcessing, isBackgrounded, currentSentence: sentences[currentIndex] || '', + currentSentenceAlignment, + currentWordIndex, currDocPage, currDocPageNumber, currDocPages, @@ -1248,7 +1400,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement skipToLocation, registerLocationChangeHandler, registerVisualPageChangeHandler, - setIsEPUB + setIsEPUB, + currentSentenceAlignment, + currentWordIndex ]); // Use media session hook diff --git a/src/lib/dexie.ts b/src/lib/dexie.ts index ee62a29..3fe1d72 100644 --- a/src/lib/dexie.ts +++ b/src/lib/dexie.ts @@ -136,6 +136,12 @@ function buildAppConfigFromRaw(raw: RawConfigMap): AppConfigRow { savedVoices, pdfHighlightEnabled: raw.pdfHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfHighlightEnabled, + pdfWordHighlightEnabled: + raw.pdfWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfWordHighlightEnabled, + epubHighlightEnabled: + raw.epubHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubHighlightEnabled, + epubWordHighlightEnabled: + raw.epubWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubWordHighlightEnabled, firstVisit: raw.firstVisit === 'true', documentListState, }; diff --git a/src/lib/pdf.ts b/src/lib/pdf.ts index cfcb945..a8e938b 100644 --- a/src/lib/pdf.ts +++ b/src/lib/pdf.ts @@ -3,9 +3,10 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist'; import "core-js/proposals/promise-with-resolvers"; import { processTextToSentences } from '@/lib/nlp'; +import type { TTSSentenceAlignment } from '@/types/tts'; import { CmpStr } from 'cmpstr'; -const cmp = CmpStr.create().setMetric( 'dice' ).setFlags( 'itw' ); +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); // Worker coordination for offloading highlight token matching interface HighlightTokenMatchRequest { @@ -141,12 +142,25 @@ try { console.error('Error patching TextLayer.render:', e); } -interface TextMatch { - elements: HTMLElement[]; - rating: number; +type PDFToken = { + spanIndex: number; + textNode: Text; text: string; - lengthDiff: number; -} + startOffset: number; + endOffset: number; +}; + +let lastSpanNodes: HTMLElement[] = []; +let lastTokens: PDFToken[] = []; +let lastSentenceTokenWindow: { start: number; end: number } | null = null; +let lastSentencePattern: string | null = null; +let lastSentenceWordToTokenMap: number[] | null = null; + +const normalizeWordForMatch = (text: string): string => + text + .trim() + .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '') + .toLowerCase(); // Text Processing functions export async function extractTextFromPDF( @@ -279,50 +293,23 @@ export function clearHighlights() { element.parentElement.removeChild(element); } }); + const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay'); + wordOverlays.forEach((node) => { + const element = node as HTMLElement; + if (element.parentElement) { + element.parentElement.removeChild(element); + } + }); } -export function findBestTextMatch( - elements: Array<{ element: HTMLElement; text: string }>, - targetText: string, - maxCombinedLength: number -): TextMatch { - let bestMatch = { - elements: [] as HTMLElement[], - rating: 0, - text: '', - lengthDiff: Infinity, - }; - - const SPAN_SEARCH_LIMIT = 10; - - for (let i = 0; i < elements.length; i++) { - let combinedText = ''; - const currentElements = []; - for (let j = i; j < Math.min(i + SPAN_SEARCH_LIMIT, elements.length); j++) { - const node = elements[j]; - const newText = combinedText ? `${combinedText} ${node.text}` : node.text; - if (newText.length > maxCombinedLength) break; - - combinedText = newText; - currentElements.push(node.element); - - const similarity = cmp.compare(combinedText, targetText); - const lengthDiff = Math.abs(combinedText.length - targetText.length); - const lengthPenalty = lengthDiff / targetText.length; - const adjustedRating = similarity * (1 - lengthPenalty * 0.5); - - if (adjustedRating > bestMatch.rating) { - bestMatch = { - elements: [...currentElements], - rating: adjustedRating, - text: combinedText, - lengthDiff, - }; - } +export function clearWordHighlights() { + const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay'); + wordOverlays.forEach((node) => { + const element = node as HTMLElement; + if (element.parentElement) { + element.parentElement.removeChild(element); } - } - - return bestMatch; + }); } export function highlightPattern( @@ -338,22 +325,18 @@ export function highlightPattern( const cleanPattern = pattern.trim().replace(/\s+/g, ' '); if (!cleanPattern) return; + lastSentencePattern = cleanPattern; + lastSentenceWordToTokenMap = null; + lastSentenceTokenWindow = null; const spanNodes = Array.from( container.querySelectorAll('.react-pdf__Page__textContent span') ) as HTMLElement[]; if (!spanNodes.length) return; + lastSpanNodes = spanNodes; - type Token = { - spanIndex: number; - textNode: Text; - text: string; - startOffset: number; - endOffset: number; - }; - - const tokens: Token[] = []; + const tokens: PDFToken[] = []; spanNodes.forEach((span, spanIndex) => { const node = span.firstChild; @@ -377,6 +360,7 @@ export function highlightPattern( }); if (!tokens.length) return; + lastTokens = tokens; const patternLen = cleanPattern.length; @@ -415,6 +399,11 @@ export function highlightPattern( bestLengthDiff < patternLen * 0.3 ? 0.3 : 0.5; if (hasTokenMatch && bestRating >= similarityThreshold) { + lastSentenceTokenWindow = { + start: bestStart, + end: bestEnd, + }; + const rangesBySpan = new Map< number, { startOffset: number; endOffset: number } @@ -454,65 +443,6 @@ export function highlightPattern( }); } - // Fallback: if token-level matching failed, use span-based fuzzy matching - if (!highlightRanges.length) { - const spanEntries = spanNodes - .map((node) => ({ - element: node as HTMLElement, - text: (node.textContent || '').trim(), - })) - .filter((entry) => entry.text.length > 0); - - const containerRect = container.getBoundingClientRect(); - const visibleTop = container.scrollTop; - const visibleBottom = visibleTop + containerRect.height; - const bufferSize = containerRect.height; - - const visibleNodes = spanEntries.filter(({ element }) => { - const rect = element.getBoundingClientRect(); - const elementTop = - rect.top - containerRect.top + container.scrollTop; - return ( - elementTop >= visibleTop - bufferSize && - elementTop <= visibleBottom + bufferSize - ); - }); - - let bestMatch = findBestTextMatch( - visibleNodes, - cleanPattern, - cleanPattern.length * 2 - ); - - if (bestMatch.rating < 0.3) { - bestMatch = findBestTextMatch( - spanEntries, - cleanPattern, - cleanPattern.length * 2 - ); - } - - const spanSimilarityThreshold = - bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5; - - if (bestMatch.rating >= spanSimilarityThreshold) { - bestMatch.elements.forEach((element) => { - const node = element.firstChild; - if (!node || node.nodeType !== Node.TEXT_NODE) return; - const textNode = node as Text; - const content = textNode.textContent || ''; - if (!content) return; - - highlightRanges.push({ - textNode, - startOffset: 0, - endOffset: content.length, - span: element, - }); - }); - } - } - if (!highlightRanges.length) return; // Create overlay rectangles for each range, relative to its page text layer @@ -577,7 +507,7 @@ export function highlightPattern( runHighlightTokenMatch(cleanPattern, tokenTexts) .then((result) => { if (!result || result.bestStart === -1) { - // No worker result or no good match; rely on span-level fallback + // No worker result or no good match; nothing to highlight applyHighlightFromTokens(null); } else { applyHighlightFromTokens({ @@ -590,95 +520,199 @@ export function highlightPattern( }) .catch((error) => { console.error( - 'Error in PDF highlight worker, falling back to span-based matching:', + 'Error in PDF highlight worker; no highlights applied:', error ); applyHighlightFromTokens(null); }); } -// Text Click Handler -export function handleTextClick( - event: MouseEvent, - pdfText: string, - containerRef: React.RefObject, - stopAndPlayFromIndex: (index: number) => void, - isProcessing: boolean, - enableHighlight = true +export function highlightWordIndex( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined, + containerRef: React.RefObject ) { - if (isProcessing) return; + clearWordHighlights(); - const target = event.target as HTMLElement; - if (!target.matches('.react-pdf__Page__textContent span')) return; - - const parentElement = target.closest('.react-pdf__Page__textContent'); - if (!parentElement) return; - - const spans = Array.from(parentElement.querySelectorAll('span')); - const clickedIndex = spans.indexOf(target); - const contextWindow = 3; - const startIndex = Math.max(0, clickedIndex - contextWindow); - const endIndex = Math.min(spans.length - 1, clickedIndex + contextWindow); - const contextText = spans - .slice(startIndex, endIndex + 1) - .map((span) => span.textContent) - .join(' ') - .trim(); - - if (!contextText?.trim()) return; - - const cleanContext = contextText.trim().replace(/\s+/g, ' '); - - // Fast path when highlight overlays are disabled: - // avoid expensive span-level fuzzy matching and just map - // the clicked context to a sentence using cheap string checks. - if (!enableHighlight) { - const sentences = processTextToSentences(pdfText); - const idx = sentences.findIndex((sentence) => { - const cleanSentence = sentence.trim().replace(/\s+/g, ' '); - return ( - cleanSentence.includes(cleanContext) || - cleanContext.includes(cleanSentence) - ); - }); - - if (idx !== -1) { - stopAndPlayFromIndex(idx); - } + if (!alignment) return; + if (wordIndex === null || wordIndex === undefined || wordIndex < 0) { return; } - const allText = Array.from(parentElement.querySelectorAll('span')).map((node) => ({ - element: node as HTMLElement, - text: (node.textContent || '').trim(), - })).filter((node) => node.text.length > 0); + const words = alignment.words || []; + if (!words.length || wordIndex >= words.length) return; - const bestMatch = findBestTextMatch(allText, cleanContext, cleanContext.length * 2); - const similarityThreshold = bestMatch.lengthDiff < cleanContext.length * 0.3 ? 0.3 : 0.5; + const container = containerRef.current; + if (!container) return; + if (!lastSentenceTokenWindow) return; + if (!lastTokens.length || !lastSpanNodes.length) return; - if (bestMatch.rating >= similarityThreshold) { - const matchText = bestMatch.text; - // Use the same sentence processing logic as TTSContext for consistency - const sentences = processTextToSentences(pdfText); - console.log("sentences inside handleTextClick: %d", sentences.length) - let bestSentenceMatch = { sentence: '', rating: 0 }; + const cleanSentence = + sentence && sentence.trim() + ? sentence.trim().replace(/\s+/g, ' ') + : null; + if (!cleanSentence || !lastSentencePattern) return; + if (cleanSentence !== lastSentencePattern) return; - for (const sentence of sentences) { - const rating = cmp.compare(matchText, sentence); - if (rating > bestSentenceMatch.rating) { - bestSentenceMatch = { sentence, rating }; - } + const start = lastSentenceTokenWindow.start; + const end = lastSentenceTokenWindow.end; + if (end < start) return; + + // Lazily build or refresh the mapping from alignment word + // indices to PDF token indices for this sentence window. + if ( + !lastSentenceWordToTokenMap || + lastSentenceWordToTokenMap.length !== words.length + ) { + const pdfFiltered: { tokenIndex: number; norm: string }[] = []; + for (let i = start; i <= end; i++) { + const norm = normalizeWordForMatch(lastTokens[i].text); + if (!norm) continue; + pdfFiltered.push({ tokenIndex: i, norm }); } - if (bestSentenceMatch.rating >= 0.5) { - const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence); - if (sentenceIndex !== -1) { - stopAndPlayFromIndex(sentenceIndex); - if (enableHighlight) { - highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef); + const ttsFiltered: { wordIndex: number; norm: string }[] = []; + for (let i = 0; i < words.length; i++) { + const norm = normalizeWordForMatch(words[i].text); + if (!norm) continue; + ttsFiltered.push({ wordIndex: i, norm }); + } + + const wordToToken = new Array(words.length).fill(-1); + + const m = pdfFiltered.length; + const n = ttsFiltered.length; + + if (m && n) { + const dp: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(Number.POSITIVE_INFINITY) + ); + const bt: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(0) + ); // 0=diag,1=up,2=left + + dp[0][0] = 0; + const GAP_COST = 0.7; + + for (let i = 0; i <= m; i++) { + for (let j = 0; j <= n; j++) { + if (i > 0 && j > 0) { + const a = pdfFiltered[i - 1].norm; + const b = ttsFiltered[j - 1].norm; + const sim = cmp.compare(a, b); + const subCost = 1 - sim; + const cand = dp[i - 1][j - 1] + subCost; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 0; + } + } + if (i > 0) { + const cand = dp[i - 1][j] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 1; + } + } + if (j > 0) { + const cand = dp[i][j - 1] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 2; + } + } + } + } + + let i = m; + let j = n; + while (i > 0 || j > 0) { + const move = bt[i][j]; + if (i > 0 && j > 0 && move === 0) { + const pdfIdx = pdfFiltered[i - 1].tokenIndex; + const ttsIdx = ttsFiltered[j - 1].wordIndex; + if (wordToToken[ttsIdx] === -1) { + wordToToken[ttsIdx] = pdfIdx; + } + i -= 1; + j -= 1; + } else if (i > 0 && (move === 1 || j === 0)) { + i -= 1; + } else if (j > 0 && (move === 2 || i === 0)) { + j -= 1; + } else { + break; + } + } + + // Propagate nearest known mapping to fill gaps + let lastSeen = -1; + for (let k = 0; k < wordToToken.length; k++) { + if (wordToToken[k] !== -1) { + lastSeen = wordToToken[k]; + } else if (lastSeen !== -1) { + wordToToken[k] = lastSeen; + } + } + let nextSeen = -1; + for (let k = wordToToken.length - 1; k >= 0; k--) { + if (wordToToken[k] !== -1) { + nextSeen = wordToToken[k]; + } else if (nextSeen !== -1) { + wordToToken[k] = nextSeen; } } } + + lastSentenceWordToTokenMap = wordToToken; + } + + const mappedIndex = + lastSentenceWordToTokenMap && wordIndex < lastSentenceWordToTokenMap.length + ? lastSentenceWordToTokenMap[wordIndex] + : -1; + + if (mappedIndex === -1) return; + + const chosenTokenIndex = mappedIndex; + + const token = lastTokens[chosenTokenIndex]; + const span = lastSpanNodes[token.spanIndex]; + if (!span) return; + + const node = token.textNode; + if (!node || node.nodeType !== Node.TEXT_NODE) return; + + try { + const range = document.createRange(); + range.setStart(node, token.startOffset); + range.setEnd(node, token.endOffset); + + const pageLayer = span.closest( + '.react-pdf__Page__textContent' + ) as HTMLElement | null; + if (!pageLayer) return; + + const pageRect = pageLayer.getBoundingClientRect(); + const rects = Array.from(range.getClientRects()); + + rects.forEach((rect) => { + const highlight = document.createElement('div'); + highlight.className = 'pdf-word-highlight-overlay'; + highlight.style.position = 'absolute'; + highlight.style.backgroundColor = 'var(--accent)'; + highlight.style.opacity = '0.4'; + highlight.style.pointerEvents = 'none'; + highlight.style.left = `${rect.left - pageRect.left}px`; + highlight.style.top = `${rect.top - pageRect.top}px`; + highlight.style.width = `${rect.width}px`; + highlight.style.height = `${rect.height}px`; + highlight.style.zIndex = '2'; + pageLayer.appendChild(highlight); + }); + } catch { + // Ignore range errors } } diff --git a/src/types/config.ts b/src/types/config.ts index 30401c3..950aaa3 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -23,7 +23,9 @@ export interface AppConfigValues { savedVoices: SavedVoices; smartSentenceSplitting: boolean; pdfHighlightEnabled: boolean; + pdfWordHighlightEnabled: boolean; epubHighlightEnabled: boolean; + epubWordHighlightEnabled: boolean; firstVisit: boolean; documentListState: DocumentListState; } @@ -47,7 +49,9 @@ export const APP_CONFIG_DEFAULTS: AppConfigValues = { savedVoices: {}, smartSentenceSplitting: true, pdfHighlightEnabled: true, + pdfWordHighlightEnabled: true, epubHighlightEnabled: true, + epubWordHighlightEnabled: true, firstVisit: false, documentListState: { sortBy: 'name', diff --git a/src/types/tts.ts b/src/types/tts.ts index 6cbec2c..a2744b2 100644 --- a/src/types/tts.ts +++ b/src/types/tts.ts @@ -63,6 +63,22 @@ export interface TTSPageTurnEstimate { fraction: number; } +// Word-level alignment within a single spoken sentence/block +export interface TTSSentenceWord { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; +} + +// Alignment metadata for a single TTS sentence/block +export interface TTSSentenceAlignment { + sentence: string; + sentenceIndex: number; + words: TTSSentenceWord[]; +} + // Metadata for an audiobook chapter export interface TTSAudiobookChapter { index: number; @@ -71,4 +87,4 @@ export interface TTSAudiobookChapter { status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b'; -} \ No newline at end of file +} From 7a29f73d078d6299924ecf9f733740c624971ecc Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Fri, 21 Nov 2025 19:29:49 -0700 Subject: [PATCH 2/9] refactor(api): move audiobook endpoints to new path - Relocated all audiobook-related API routes from `/api/audio/convert/*` to `/api/audiobook/*`. - This change affects endpoints for chapter conversion, retrieval, deletion, and overall audiobook status. - Updated client-side calls in `AudiobookExportModal.tsx`, `EPUBContext.tsx`, and `PDFContext.tsx` to reflect the new API paths. - Modified API tests (`api.spec.ts`, `export.spec.ts`) to target the restructured endpoints. - The new API structure provides better organization and a clearer, more consistent interface for audiobook functionality. --- .../convert => audiobook}/chapter/route.ts | 0 .../api/{audio/convert => audiobook}/route.ts | 29 ++++++++++++++++++- .../chapters => audiobook/status}/route.ts | 28 +----------------- src/components/AudiobookExportModal.tsx | 10 +++---- src/contexts/EPUBContext.tsx | 6 ++-- src/contexts/PDFContext.tsx | 6 ++-- tests/api.spec.ts | 4 +-- tests/export.spec.ts | 4 +-- 8 files changed, 44 insertions(+), 43 deletions(-) rename src/app/api/{audio/convert => audiobook}/chapter/route.ts (100%) rename src/app/api/{audio/convert => audiobook}/route.ts (91%) rename src/app/api/{audio/convert/chapters => audiobook/status}/route.ts (67%) diff --git a/src/app/api/audio/convert/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts similarity index 100% rename from src/app/api/audio/convert/chapter/route.ts rename to src/app/api/audiobook/chapter/route.ts diff --git a/src/app/api/audio/convert/route.ts b/src/app/api/audiobook/route.ts similarity index 91% rename from src/app/api/audio/convert/route.ts rename to src/app/api/audiobook/route.ts index 4c4a3f3..b37dfae 100644 --- a/src/app/api/audio/convert/route.ts +++ b/src/app/api/audiobook/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { spawn } from 'child_process'; -import { writeFile, readFile, mkdir, unlink, readdir } from 'fs/promises'; +import { writeFile, readFile, mkdir, unlink, readdir, rm } from 'fs/promises'; import { existsSync, createReadStream } from 'fs'; import { join } from 'path'; import { randomUUID } from 'crypto'; @@ -378,4 +378,31 @@ function streamFile(filePath: string, format: string) { 'Cache-Control': 'no-cache', }, }); +} +export async function DELETE(request: NextRequest) { + try { + const bookId = request.nextUrl.searchParams.get('bookId'); + if (!bookId) { + return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 }); + } + + const docstoreDir = join(process.cwd(), 'docstore'); + const intermediateDir = join(docstoreDir, `${bookId}-audiobook`); + + // If directory doesn't exist, consider it already reset + if (!existsSync(intermediateDir)) { + return NextResponse.json({ success: true, existed: false }); + } + + // Recursively delete the entire audiobook directory + await rm(intermediateDir, { recursive: true, force: true }); + + return NextResponse.json({ success: true, existed: true }); + } catch (error) { + console.error('Error resetting audiobook:', error); + return NextResponse.json( + { error: 'Failed to reset audiobook' }, + { status: 500 } + ); + } } \ No newline at end of file diff --git a/src/app/api/audio/convert/chapters/route.ts b/src/app/api/audiobook/status/route.ts similarity index 67% rename from src/app/api/audio/convert/chapters/route.ts rename to src/app/api/audiobook/status/route.ts index 76efd83..d682a77 100644 --- a/src/app/api/audio/convert/chapters/route.ts +++ b/src/app/api/audiobook/status/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; -import { readdir, readFile, rm } from 'fs/promises'; +import { readdir, readFile } from 'fs/promises'; import { existsSync } from 'fs'; import { join } from 'path'; @@ -69,30 +69,4 @@ export async function GET(request: NextRequest) { } } -export async function DELETE(request: NextRequest) { - try { - const bookId = request.nextUrl.searchParams.get('bookId'); - if (!bookId) { - return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 }); - } - const docstoreDir = join(process.cwd(), 'docstore'); - const intermediateDir = join(docstoreDir, `${bookId}-audiobook`); - - // If directory doesn't exist, consider it already reset - if (!existsSync(intermediateDir)) { - return NextResponse.json({ success: true, existed: false }); - } - - // Recursively delete the entire audiobook directory - await rm(intermediateDir, { recursive: true, force: true }); - - return NextResponse.json({ success: true, existed: true }); - } catch (error) { - console.error('Error resetting audiobook:', error); - return NextResponse.json( - { error: 'Failed to reset audiobook' }, - { status: 500 } - ); - } -} diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index 51fc052..c9a022c 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -61,7 +61,7 @@ export function AudiobookExportModal({ setIsLoadingExisting(true); } try { - const response = await fetch(`/api/audio/convert/chapters?bookId=${documentId}`); + const response = await fetch(`/api/audiobook/status?bookId=${documentId}`); if (response.ok) { const data = await response.json(); if (data.exists && data.chapters.length > 0) { @@ -241,7 +241,7 @@ export function AudiobookExportModal({ const performDeleteChapter = useCallback(async () => { if (!bookId || !pendingDeleteChapter) return; try { - const response = await fetch(`/api/audio/convert/chapter?bookId=${bookId}&chapterIndex=${pendingDeleteChapter.index}`, { + const response = await fetch(`/api/audiobook/chapter?bookId=${bookId}&chapterIndex=${pendingDeleteChapter.index}`, { method: 'DELETE' }); if (!response.ok) { @@ -261,7 +261,7 @@ export function AudiobookExportModal({ const targetBookId = bookId || documentId; if (!targetBookId) return; try { - const resp = await fetch(`/api/audio/convert/chapters?bookId=${targetBookId}`, { method: 'DELETE' }); + const resp = await fetch(`/api/audiobook?bookId=${targetBookId}`, { method: 'DELETE' }); if (!resp.ok) { throw new Error('Reset failed'); } @@ -281,7 +281,7 @@ export function AudiobookExportModal({ if (!chapter.bookId) return; try { - const response = await fetch(`/api/audio/convert/chapter?bookId=${chapter.bookId}&chapterIndex=${chapter.index}`); + const response = await fetch(`/api/audiobook/chapter?bookId=${chapter.bookId}&chapterIndex=${chapter.index}`); if (!response.ok) throw new Error('Download failed'); const blob = await response.blob(); @@ -306,7 +306,7 @@ export function AudiobookExportModal({ setIsCombining(true); try { - const response = await fetch(`/api/audio/convert?bookId=${bookId}&format=${format}`); + const response = await fetch(`/api/audiobook?bookId=${bookId}&format=${format}`); if (!response.ok) throw new Error('Download failed'); const reader = response.body?.getReader(); diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index 1922d2b..b555ff5 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -339,7 +339,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const existingIndices = new Set(); if (bookId) { try { - const existingResponse = await fetch(`/api/audio/convert/chapters?bookId=${bookId}`); + const existingResponse = await fetch(`/api/audiobook/status?bookId=${bookId}`); if (existingResponse.ok) { const existingData = await existingResponse.json(); if (existingData.chapters && existingData.chapters.length > 0) { @@ -469,7 +469,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch('/api/audio/convert', { + const convertResponse = await fetch(`/api/audiobook`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -645,7 +645,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch('/api/audio/convert', { + const convertResponse = await fetch('/api/audiobook', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index dfdb6b1..fb03eb8 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -299,7 +299,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { const existingIndices = new Set(); if (bookId) { try { - const existingResponse = await fetch(`/api/audio/convert/chapters?bookId=${bookId}`); + const existingResponse = await fetch(`/api/audiobook/status?bookId=${bookId}`); if (existingResponse.ok) { const existingData = await existingResponse.json(); if (existingData.chapters && existingData.chapters.length > 0) { @@ -399,7 +399,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch('/api/audio/convert', { + const convertResponse = await fetch(`/api/audiobook`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -587,7 +587,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch('/api/audio/convert', { + const convertResponse = await fetch('/api/audiobook', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/tests/api.spec.ts b/tests/api.spec.ts index ddfd974..dde7756 100644 --- a/tests/api.spec.ts +++ b/tests/api.spec.ts @@ -9,9 +9,9 @@ test.describe('API health checks', () => { expect(json.voices.length).toBeGreaterThan(0); }); - test('GET /api/audio/convert/chapters returns 200 with exists flag and chapters array', async ({ request }) => { + test('GET /api/audiobook/status returns 200 with exists flag and chapters array', async ({ request }) => { const bookId = `healthcheck-${Date.now()}`; - const res = await request.get(`/api/audio/convert/chapters?bookId=${bookId}`); + const res = await request.get(`/api/audiobook/status?bookId=${bookId}`); expect(res.ok()).toBeTruthy(); const json = await res.json(); expect(json).toHaveProperty('exists'); diff --git a/tests/export.spec.ts b/tests/export.spec.ts index a25bfe4..77324a7 100644 --- a/tests/export.spec.ts +++ b/tests/export.spec.ts @@ -67,7 +67,7 @@ async function getAudioDurationSeconds(filePath: string) { } async function expectChaptersBackendState(page: Page, bookId: string) { - const res = await page.request.get(`/api/audio/convert/chapters?bookId=${bookId}`); + const res = await page.request.get(`/api/audiobook/status?bookId=${bookId}`); expect(res.ok()).toBeTruthy(); const json = await res.json(); return json; @@ -271,7 +271,7 @@ test.describe('Audiobook export', () => { ).toBeVisible({ timeout: 60_000 }); // Backend should report no existing chapters for this bookId - const res = await page.request.get(`/api/audio/convert/chapters?bookId=${bookId}`); + const res = await page.request.get(`/api/audiobook/status?bookId=${bookId}`); expect(res.ok()).toBeTruthy(); const json = await res.json(); expect(json.exists).toBe(false); From b5769105230a326e9193738357ba47f7fcf8d44c Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Fri, 21 Nov 2025 23:33:41 -0700 Subject: [PATCH 3/9] refactor(client): centralize client-side API calls and refine types Abstracted direct fetch calls across components and contexts into new functions within `src/lib/client.ts`. This provides a consistent and centralized interface for interacting with backend APIs. - Introduced `src/lib/client.ts` to encapsulate API request logic. - Standardized audio buffer types (`TTSAudioBuffer`, `TTSAudioBytes`) in `src/types/tts.ts`. - Moved client-specific request types (`TTSRequestPayload`, `TTSRequestHeaders`, `TTSRetryOptions`) to `src/types/client.ts`. - Updated API routes and consumer components/contexts to leverage the new client library functions and type definitions. - Removed `src/utils/audio.ts` as its utility functions are now part of `src/lib/client.ts`. --- src/app/api/audiobook/route.ts | 16 +- src/app/api/audiobook/status/route.ts | 5 +- src/app/api/tts/route.ts | 17 +- src/app/api/whisper/route.ts | 7 +- src/app/epub/[id]/page.tsx | 9 +- src/app/pdf/[id]/page.tsx | 7 +- src/components/AudiobookExportModal.tsx | 70 ++++---- src/components/DocumentSettings.tsx | 7 +- src/components/DocumentUploader.tsx | 15 +- src/components/SettingsModal.tsx | 8 +- src/contexts/EPUBContext.tsx | 157 +++++------------- src/contexts/PDFContext.tsx | 158 +++++------------- src/contexts/TTSContext.tsx | 41 ++--- src/hooks/audio/useAudioCache.ts | 5 +- src/hooks/audio/useVoiceManagement.ts | 17 +- src/lib/client.ts | 205 ++++++++++++++++++++++++ src/lib/pdf.ts | 1 - src/types/client.ts | 67 ++++++++ src/types/tts.ts | 33 +--- src/utils/audio.ts | 49 ------ 20 files changed, 455 insertions(+), 439 deletions(-) create mode 100644 src/lib/client.ts create mode 100644 src/types/client.ts delete mode 100644 src/utils/audio.ts diff --git a/src/app/api/audiobook/route.ts b/src/app/api/audiobook/route.ts index b37dfae..17bf12a 100644 --- a/src/app/api/audiobook/route.ts +++ b/src/app/api/audiobook/route.ts @@ -4,12 +4,13 @@ import { writeFile, readFile, mkdir, unlink, readdir, rm } from 'fs/promises'; import { existsSync, createReadStream } from 'fs'; import { join } from 'path'; import { randomUUID } from 'crypto'; +import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts'; interface ConversionRequest { chapterTitle: string; - buffer: number[]; + buffer: TTSAudioBytes; bookId?: string; - format?: 'mp3' | 'm4b'; + format?: TTSAudiobookFormat; chapterIndex?: number; } @@ -206,9 +207,12 @@ export async function POST(request: NextRequest) { await unlink(inputPath).catch(console.error); return NextResponse.json({ + index: chapterIndex, + title: data.chapterTitle, + duration, + status: 'completed' as const, bookId, - chapterIndex, - duration + format }); } catch (error) { @@ -229,7 +233,7 @@ export async function POST(request: NextRequest) { export async function GET(request: NextRequest) { try { const bookId = request.nextUrl.searchParams.get('bookId'); - const requestedFormat = request.nextUrl.searchParams.get('format') as 'mp3' | 'm4b' | null; + const requestedFormat = request.nextUrl.searchParams.get('format') as TTSAudiobookFormat | null; if (!bookId) { return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 }); } @@ -405,4 +409,4 @@ export async function DELETE(request: NextRequest) { { status: 500 } ); } -} \ No newline at end of file +} diff --git a/src/app/api/audiobook/status/route.ts b/src/app/api/audiobook/status/route.ts index d682a77..44e8fd6 100644 --- a/src/app/api/audiobook/status/route.ts +++ b/src/app/api/audiobook/status/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { readdir, readFile } from 'fs/promises'; import { existsSync } from 'fs'; import { join } from 'path'; +import type { TTSAudiobookFormat } from '@/types/tts'; export async function GET(request: NextRequest) { try { @@ -26,7 +27,7 @@ export async function GET(request: NextRequest) { duration?: number; status: 'completed' | 'error'; bookId: string; - format?: 'mp3' | 'm4b'; + format?: TTSAudiobookFormat; }> = []; for (const metaFile of metaFiles) { @@ -68,5 +69,3 @@ export async function GET(request: NextRequest) { ); } } - - diff --git a/src/app/api/tts/route.ts b/src/app/api/tts/route.ts index a075d41..ff3680b 100644 --- a/src/app/api/tts/route.ts +++ b/src/app/api/tts/route.ts @@ -4,7 +4,8 @@ import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs'; import { isKokoroModel } from '@/utils/voice'; import { LRUCache } from 'lru-cache'; import { createHash } from 'crypto'; -import type { TTSRequestPayload, TTSError } from '@/types/tts'; +import type { TTSRequestPayload } from '@/types/client'; +import type { TTSError, TTSAudioBuffer } from '@/types/tts'; export const runtime = 'nodejs'; @@ -13,7 +14,7 @@ type ExtendedSpeechParams = Omit & { voice: SpeechCreateParams['voice'] | CustomVoice; instructions?: string; }; -type AudioBufferValue = ArrayBuffer; +type AudioBufferValue = TTSAudioBuffer; const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes @@ -25,7 +26,7 @@ const ttsAudioCache = new LRUCache({ }); type InflightEntry = { - promise: Promise; + promise: Promise; controller: AbortController; consumers: number; }; @@ -40,7 +41,7 @@ async function fetchTTSBufferWithRetry( openai: OpenAI, createParams: ExtendedSpeechParams, signal: AbortSignal -): Promise { +): Promise { let attempt = 0; const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2); let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250); @@ -135,7 +136,7 @@ export async function POST(req: NextRequest) { voice: normalizedVoice, input: text, speed: speed, - response_format: format === 'aac' ? 'aac' : 'mp3', + response_format: format, }; // Only add instructions if model is gpt-4o-mini-tts and instructions are provided if ((model as string) === 'gpt-4o-mini-tts' && instructions) { @@ -143,7 +144,7 @@ export async function POST(req: NextRequest) { } // Compute cache key and check LRU before making provider call - const contentType = format === 'aac' ? 'audio/aac' : 'audio/mpeg'; + const contentType = 'audio/mpeg'; // Preserve voice string as-is for cache key (no weight stripping) const voiceForKey = typeof createParams.voice === 'string' @@ -245,7 +246,7 @@ export async function POST(req: NextRequest) { }; req.signal.addEventListener('abort', onAbort, { once: true }); - let buffer: ArrayBuffer; + let buffer: TTSAudioBuffer; try { buffer = await entry.promise; } finally { @@ -280,4 +281,4 @@ export async function POST(req: NextRequest) { { status: 500 } ); } -} \ No newline at end of file +} diff --git a/src/app/api/whisper/route.ts b/src/app/api/whisper/route.ts index 146e452..a0d0da7 100644 --- a/src/app/api/whisper/route.ts +++ b/src/app/api/whisper/route.ts @@ -4,14 +4,14 @@ import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; import { spawn } from 'child_process'; -import type { TTSSentenceAlignment } from '@/types/tts'; +import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts'; import { preprocessSentenceForAudio } from '@/lib/nlp'; export const runtime = 'nodejs'; interface WhisperRequestBody { text: string; - audio: number[]; // raw bytes from Uint8Array + audio: TTSAudioBytes; // raw bytes from Uint8Array lang?: string; } @@ -331,7 +331,7 @@ function mapWordsToSentenceOffsets( } async function alignAudioWithText( - audioBuffer: ArrayBuffer, + audioBuffer: TTSAudioBuffer, text: string, cacheKey?: string, opts: WhisperAlignmentOptions = {} @@ -448,4 +448,3 @@ export async function POST(req: NextRequest) { ); } } - diff --git a/src/app/epub/[id]/page.tsx b/src/app/epub/[id]/page.tsx index 96f8b37..325b96c 100644 --- a/src/app/epub/[id]/page.tsx +++ b/src/app/epub/[id]/page.tsx @@ -14,6 +14,7 @@ import TTSPlayer from '@/components/player/TTSPlayer'; import { ZoomControl } from '@/components/ZoomControl'; import { AudiobookExportModal } from '@/components/AudiobookExportModal'; import { DownloadIcon } from '@/components/icons/Icons'; +import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -86,8 +87,8 @@ export default function EPUBPage() { const handleGenerateAudiobook = useCallback(async ( onProgress: (progress: number) => void, signal: AbortSignal, - onChapterComplete: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, - format: 'mp3' | 'm4b' + onChapterComplete: (chapter: TTSAudiobookChapter) => void, + format: TTSAudiobookFormat ) => { return createEPUBAudioBook(onProgress, signal, onChapterComplete, id as string, format); }, [createEPUBAudioBook, id]); @@ -95,7 +96,7 @@ export default function EPUBPage() { const handleRegenerateChapter = useCallback(async ( chapterIndex: number, bookId: string, - format: 'mp3' | 'm4b', + format: TTSAudiobookFormat, signal: AbortSignal ) => { return regenerateEPUBChapter(chapterIndex, bookId, format, signal); @@ -190,4 +191,4 @@ export default function EPUBPage() { ); -} \ No newline at end of file +} diff --git a/src/app/pdf/[id]/page.tsx b/src/app/pdf/[id]/page.tsx index 7cd6957..276dc52 100644 --- a/src/app/pdf/[id]/page.tsx +++ b/src/app/pdf/[id]/page.tsx @@ -12,6 +12,7 @@ import { SettingsIcon, DownloadIcon } from '@/components/icons/Icons'; import { Header } from '@/components/Header'; import { ZoomControl } from '@/components/ZoomControl'; import { AudiobookExportModal } from '@/components/AudiobookExportModal'; +import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; import TTSPlayer from '@/components/player/TTSPlayer'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -80,8 +81,8 @@ export default function PDFViewerPage() { const handleGenerateAudiobook = useCallback(async ( onProgress: (progress: number) => void, signal: AbortSignal, - onChapterComplete: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, - format: 'mp3' | 'm4b' + onChapterComplete: (chapter: TTSAudiobookChapter) => void, + format: TTSAudiobookFormat ) => { return createPDFAudioBook(onProgress, signal, onChapterComplete, id as string, format); }, [createPDFAudioBook, id]); @@ -89,7 +90,7 @@ export default function PDFViewerPage() { const handleRegenerateChapter = useCallback(async ( chapterIndex: number, bookId: string, - format: 'mp3' | 'm4b', + format: TTSAudiobookFormat, signal: AbortSignal ) => { return regeneratePDFChapter(chapterIndex, bookId, format, signal); diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index c9a022c..d9712bd 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -9,7 +9,14 @@ import { DownloadIcon, CheckCircleIcon, XCircleIcon, ClockIcon, ChevronUpDownIco import { ConfirmDialog } from '@/components/ConfirmDialog'; import { LoadingSpinner } from '@/components/Spinner'; import { useConfig } from '@/contexts/ConfigContext'; -import type { TTSAudiobookChapter } from '@/types/tts'; +import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; +import { + getAudiobookStatus, + deleteAudiobookChapter, + deleteAudiobook, + downloadAudiobookChapter, + downloadAudiobook +} from '@/lib/client'; interface AudiobookExportModalProps { isOpen: boolean; setIsOpen: (isOpen: boolean) => void; @@ -19,12 +26,12 @@ interface AudiobookExportModalProps { onProgress: (progress: number) => void, signal: AbortSignal, onChapterComplete: (chapter: TTSAudiobookChapter) => void, - format: 'mp3' | 'm4b' + format: TTSAudiobookFormat ) => Promise; // Returns bookId onRegenerateChapter?: ( chapterIndex: number, bookId: string, - format: 'mp3' | 'm4b', + format: TTSAudiobookFormat, signal: AbortSignal ) => Promise; } @@ -46,7 +53,7 @@ export function AudiobookExportModal({ const [isLoadingExisting, setIsLoadingExisting] = useState(false); const [isRefreshingChapters, setIsRefreshingChapters] = useState(false); const [currentChapter, setCurrentChapter] = useState(''); - const [format, setFormat] = useState<'mp3' | 'm4b'>('m4b'); + const [format, setFormat] = useState('m4b'); const [regeneratingChapter, setRegeneratingChapter] = useState(null); const abortControllerRef = useRef(null); const [pendingDeleteChapter, setPendingDeleteChapter] = useState(null); @@ -61,26 +68,23 @@ export function AudiobookExportModal({ setIsLoadingExisting(true); } try { - const response = await fetch(`/api/audiobook/status?bookId=${documentId}`); - if (response.ok) { - const data = await response.json(); - if (data.exists && data.chapters.length > 0) { - setChapters(data.chapters); - setBookId(data.bookId); - // Set format from existing chapters - this ensures the format matches what was actually generated - if (data.chapters[0]?.format) { - const detectedFormat = data.chapters[0].format as 'mp3' | 'm4b'; - setFormat(detectedFormat); - } - // If we have a complete audiobook, we're done - if (data.hasComplete) { - setProgress(100); - } - } else { - // If nothing exists, clear chapters/bookId to reflect current state - setChapters([]); - setBookId(null); + const data = await getAudiobookStatus(documentId); + if (data.exists && data.chapters.length > 0) { + setChapters(data.chapters); + setBookId(data.bookId); + // Set format from existing chapters - this ensures the format matches what was actually generated + if (data.chapters[0]?.format) { + const detectedFormat = data.chapters[0].format as TTSAudiobookFormat; + setFormat(detectedFormat); } + // If we have a complete audiobook, we're done + if (data.hasComplete) { + setProgress(100); + } + } else { + // If nothing exists, clear chapters/bookId to reflect current state + setChapters([]); + setBookId(null); } } catch (error) { console.error('Error fetching existing chapters:', error); @@ -241,12 +245,7 @@ export function AudiobookExportModal({ const performDeleteChapter = useCallback(async () => { if (!bookId || !pendingDeleteChapter) return; try { - const response = await fetch(`/api/audiobook/chapter?bookId=${bookId}&chapterIndex=${pendingDeleteChapter.index}`, { - method: 'DELETE' - }); - if (!response.ok) { - throw new Error('Delete failed'); - } + await deleteAudiobookChapter(bookId, pendingDeleteChapter.index); setChapters(prev => prev.filter(c => c.index !== pendingDeleteChapter.index)); await fetchExistingChapters(true); } catch (error) { @@ -261,10 +260,7 @@ export function AudiobookExportModal({ const targetBookId = bookId || documentId; if (!targetBookId) return; try { - const resp = await fetch(`/api/audiobook?bookId=${targetBookId}`, { method: 'DELETE' }); - if (!resp.ok) { - throw new Error('Reset failed'); - } + await deleteAudiobook(targetBookId); setChapters([]); setBookId(null); setProgress(0); @@ -281,10 +277,7 @@ export function AudiobookExportModal({ if (!chapter.bookId) return; try { - const response = await fetch(`/api/audiobook/chapter?bookId=${chapter.bookId}&chapterIndex=${chapter.index}`); - if (!response.ok) throw new Error('Download failed'); - - const blob = await response.blob(); + const blob = await downloadAudiobookChapter(chapter.bookId, chapter.index); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; @@ -306,8 +299,7 @@ export function AudiobookExportModal({ setIsCombining(true); try { - const response = await fetch(`/api/audiobook?bookId=${bookId}&format=${format}`); - if (!response.ok) throw new Error('Download failed'); + const response = await downloadAudiobook(bookId, format); const reader = response.body?.getReader(); if (!reader) throw new Error('No response body'); diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index 76a3e7f..84abae2 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -8,6 +8,7 @@ import { useEPUB } from '@/contexts/EPUBContext'; import { usePDF } from '@/contexts/PDFContext'; import { AudiobookExportModal } from '@/components/AudiobookExportModal'; import { useParams } from 'next/navigation'; +import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -80,8 +81,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { const handleGenerateAudiobook = useCallback(async ( onProgress: (progress: number) => void, signal: AbortSignal, - onChapterComplete: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, - format: 'mp3' | 'm4b' + onChapterComplete: (chapter: TTSAudiobookChapter) => void, + format: TTSAudiobookFormat ) => { if (epub) { return createEPUBAudioBook(onProgress, signal, onChapterComplete, id as string, format); @@ -93,7 +94,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { const handleRegenerateChapter = useCallback(async ( chapterIndex: number, bookId: string, - format: 'mp3' | 'm4b', + format: TTSAudiobookFormat, signal: AbortSignal ) => { if (epub) { diff --git a/src/components/DocumentUploader.tsx b/src/components/DocumentUploader.tsx index 0f4d58a..cd35696 100644 --- a/src/components/DocumentUploader.tsx +++ b/src/components/DocumentUploader.tsx @@ -4,6 +4,7 @@ import { useState, useCallback } from 'react'; import { useDropzone } from 'react-dropzone'; import { UploadIcon } from '@/components/icons/Icons'; import { useDocuments } from '@/contexts/DocumentContext'; +import { convertDocxToPdf as convertDocxToPdfClient } from '@/lib/client'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -23,19 +24,7 @@ export function DocumentUploader({ className = '', variant = 'default' }: Docume const [error, setError] = useState(null); const convertDocxToPdf = async (file: File): Promise => { - const formData = new FormData(); - formData.append('file', file); - - const response = await fetch('/api/documents/docx-to-pdf', { - method: 'POST', - body: formData, - }); - - if (!response.ok) { - throw new Error('Failed to convert DOCX to PDF'); - } - - const pdfBlob = await response.blob(); + const pdfBlob = await convertDocxToPdfClient(file); return new File([pdfBlob], file.name.replace(/\.docx$/, '.pdf'), { type: 'application/pdf', }); diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index db2e421..22a0c05 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -28,6 +28,7 @@ import { ConfirmDialog } from '@/components/ConfirmDialog'; import { ProgressPopup } from '@/components/ProgressPopup'; import { useTimeEstimation } from '@/hooks/useTimeEstimation'; import { THEMES } from '@/contexts/ThemeContext'; +import { deleteServerDocuments } from '@/lib/client'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -222,12 +223,7 @@ export function SettingsModal() { const handleClearServer = async () => { try { - const response = await fetch('/api/documents', { - method: 'DELETE', - }); - if (!response.ok) { - throw new Error('Failed to delete server documents'); - } + await deleteServerDocuments(); } catch (error) { console.error('Delete failed:', error); } diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index b555ff5..365295f 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -21,14 +21,18 @@ import { useTTS } from '@/contexts/TTSContext'; import { createRangeCfi } from '@/lib/epub'; import { useParams } from 'next/navigation'; import { useConfig } from './ConfigContext'; -import { withRetry } from '@/utils/audio'; +import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client'; import { CmpStr } from 'cmpstr'; +import type { + TTSSentenceAlignment, + TTSAudiobookFormat, + TTSAudiobookChapter, +} from '@/types/tts'; import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions, - TTSSentenceAlignment -} from '@/types/tts'; +} from '@/types/client'; interface EPUBContextType { currDocData: ArrayBuffer | undefined; @@ -39,8 +43,8 @@ interface EPUBContextType { setCurrentDocument: (id: string) => Promise; clearCurrDoc: () => void; extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise; - createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise; - regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>; + createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: TTSAudiobookChapter) => void, bookId?: string, format?: TTSAudiobookFormat) => Promise; + regenerateChapter: (chapterIndex: number, bookId: string, format: TTSAudiobookFormat, signal: AbortSignal) => Promise; bookRef: RefObject; renditionRef: RefObject; tocRef: RefObject; @@ -319,9 +323,9 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const createFullAudioBook = useCallback(async ( onProgress: (progress: number) => void, signal?: AbortSignal, - onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, + onChapterComplete?: (chapter: TTSAudiobookChapter) => void, providedBookId?: string, - format: 'mp3' | 'm4b' = 'mp3' + format: TTSAudiobookFormat = 'mp3' ): Promise => { try { const sections = await extractBookText(); @@ -339,18 +343,15 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const existingIndices = new Set(); if (bookId) { try { - const existingResponse = await fetch(`/api/audiobook/status?bookId=${bookId}`); - if (existingResponse.ok) { - const existingData = await existingResponse.json(); - if (existingData.chapters && existingData.chapters.length > 0) { - for (const ch of existingData.chapters) { - existingIndices.add(ch.index); - } - // Log smallest missing index for visibility - let nextMissing = 0; - while (existingIndices.has(nextMissing)) nextMissing++; - console.log(`Resuming; next missing chapter index is ${nextMissing}`); + const existingData = await getAudiobookStatus(bookId); + if (existingData.chapters && existingData.chapters.length > 0) { + for (const ch of existingData.chapters) { + existingIndices.add(ch.index); } + // Log smallest missing index for visibility + let nextMissing = 0; + while (existingIndices.has(nextMissing)) nextMissing++; + console.log(`Resuming; next missing chapter index is ${nextMissing}`); } } catch (error) { console.error('Error checking existing chapters:', error); @@ -431,22 +432,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { throw new DOMException('Aborted', 'AbortError'); } - const ttsResponse = await fetch('/api/tts', { - method: 'POST', - headers: reqHeaders, - body: JSON.stringify(reqBody), - signal - }); - - if (!ttsResponse.ok) { - throw new Error(`TTS processing failed with status ${ttsResponse.status}`); - } - - const buffer = await ttsResponse.arrayBuffer(); - if (buffer.byteLength === 0) { - throw new Error('Received empty audio buffer from TTS'); - } - return buffer; + return await generateTTS(reqBody, reqHeaders, signal); }, retryOptions ); @@ -469,45 +455,21 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch(`/api/audiobook`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format, - chapterIndex: i - }), - signal - }); - - if (convertResponse.status === 499) { - throw new Error('cancelled'); - } - - if (!convertResponse.ok) { - throw new Error('Failed to convert audio chapter'); - } - - const { bookId: returnedBookId, chapterIndex, duration } = await convertResponse.json(); + const chapter = await createAudiobookChapter({ + chapterTitle, + buffer: Array.from(new Uint8Array(audioBuffer)), + bookId, + format, + chapterIndex: i + }, signal); if (!bookId) { - bookId = returnedBookId; + bookId = chapter.bookId!; } // Notify about completed chapter if (onChapterComplete) { - onChapterComplete({ - index: chapterIndex, - title: chapterTitle, - duration, - status: 'completed', - bookId, - format - }); + onChapterComplete(chapter); } processedLength += trimmedText.length; @@ -553,9 +515,9 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const regenerateChapter = useCallback(async ( chapterIndex: number, bookId: string, - format: 'mp3' | 'm4b', + format: TTSAudiobookFormat, signal: AbortSignal - ): Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }> => { + ): Promise => { try { const sections = await extractBookText(); if (chapterIndex >= sections.length) { @@ -620,22 +582,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { throw new DOMException('Aborted', 'AbortError'); } - const ttsResponse = await fetch('/api/tts', { - method: 'POST', - headers: reqHeaders, - body: JSON.stringify(reqBody), - signal - }); - - if (!ttsResponse.ok) { - throw new Error(`TTS processing failed with status ${ttsResponse.status}`); - } - - const buffer = await ttsResponse.arrayBuffer(); - if (buffer.byteLength === 0) { - throw new Error('Received empty audio buffer from TTS'); - } - return buffer; + return await generateTTS(reqBody, reqHeaders, signal); }, retryOptions ); @@ -645,39 +592,15 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch('/api/audiobook', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format, - chapterIndex - }), - signal - }); - - if (convertResponse.status === 499) { - throw new Error('cancelled'); - } - - if (!convertResponse.ok) { - throw new Error('Failed to convert audio chapter'); - } - - const { chapterIndex: returnedIndex, duration } = await convertResponse.json(); - - return { - index: returnedIndex, - title: chapterTitle, - duration, - status: 'completed', + const chapter = await createAudiobookChapter({ + chapterTitle, + buffer: Array.from(new Uint8Array(audioBuffer)), bookId, - format - }; + format, + chapterIndex + }, signal); + + return chapter; } catch (error) { if (error instanceof Error && (error.name === 'AbortError' || error.message.includes('cancelled'))) { diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index fb03eb8..72a2340 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -31,7 +31,7 @@ import { getPdfDocument } from '@/lib/dexie'; import { useTTS } from '@/contexts/TTSContext'; import { useConfig } from '@/contexts/ConfigContext'; import { processTextToSentences } from '@/lib/nlp'; -import { withRetry } from '@/utils/audio'; +import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client'; import { extractTextFromPDF, highlightPattern, @@ -40,12 +40,17 @@ import { highlightWordIndex, } from '@/lib/pdf'; +import type { + TTSSentenceAlignment, + TTSAudioBuffer, + TTSAudiobookFormat, + TTSAudiobookChapter, +} from '@/types/tts'; import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions, - TTSSentenceAlignment -} from '@/types/tts'; +} from '@/types/client'; /** * Interface defining all available methods and properties in the PDF context @@ -72,8 +77,8 @@ interface PDFContextType { sentence: string | null | undefined, containerRef: RefObject ) => void; - createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise; - regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>; + createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: TTSAudiobookChapter) => void, bookId?: string, format?: TTSAudiobookFormat) => Promise; + regenerateChapter: (chapterIndex: number, bookId: string, format: TTSAudiobookFormat, signal: AbortSignal) => Promise; isAudioCombining: boolean; } @@ -257,9 +262,9 @@ export function PDFProvider({ children }: { children: ReactNode }) { const createFullAudioBook = useCallback(async ( onProgress: (progress: number) => void, signal?: AbortSignal, - onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, + onChapterComplete?: (chapter: TTSAudiobookChapter) => void, providedBookId?: string, - format: 'mp3' | 'm4b' = 'mp3' + format: TTSAudiobookFormat = 'mp3' ): Promise => { try { if (!pdfDocument) { @@ -299,17 +304,14 @@ export function PDFProvider({ children }: { children: ReactNode }) { const existingIndices = new Set(); if (bookId) { try { - const existingResponse = await fetch(`/api/audiobook/status?bookId=${bookId}`); - if (existingResponse.ok) { - const existingData = await existingResponse.json(); - if (existingData.chapters && existingData.chapters.length > 0) { - for (const ch of existingData.chapters) { - existingIndices.add(ch.index); - } - let nextMissing = 0; - while (existingIndices.has(nextMissing)) nextMissing++; - console.log(`Resuming; next missing page index is ${nextMissing} (page ${nextMissing + 1})`); + const existingData = await getAudiobookStatus(bookId); + if (existingData.chapters && existingData.chapters.length > 0) { + for (const ch of existingData.chapters) { + existingIndices.add(ch.index); } + let nextMissing = 0; + while (existingIndices.has(nextMissing)) nextMissing++; + console.log(`Resuming; next missing page index is ${nextMissing} (page ${nextMissing + 1})`); } } catch (error) { console.error('Error checking existing chapters:', error); @@ -367,22 +369,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { throw new DOMException('Aborted', 'AbortError'); } - const ttsResponse = await fetch('/api/tts', { - method: 'POST', - headers: reqHeaders, - body: JSON.stringify(reqBody), - signal - }); - - if (!ttsResponse.ok) { - throw new Error(`TTS processing failed with status ${ttsResponse.status}`); - } - - const buffer = await ttsResponse.arrayBuffer(); - if (buffer.byteLength === 0) { - throw new Error('Received empty audio buffer from TTS'); - } - return buffer; + return await generateTTS(reqBody, reqHeaders, signal); }, retryOptions ); @@ -399,45 +386,21 @@ export function PDFProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch(`/api/audiobook`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format, - chapterIndex: i - }), - signal - }); - - if (convertResponse.status === 499) { - throw new Error('cancelled'); - } - - if (!convertResponse.ok) { - throw new Error('Failed to convert audio chapter'); - } - - const { bookId: returnedBookId, chapterIndex, duration } = await convertResponse.json(); + const chapter = await createAudiobookChapter({ + chapterTitle, + buffer: Array.from(new Uint8Array(audioBuffer)), + bookId, + format, + chapterIndex: i + }, signal); if (!bookId) { - bookId = returnedBookId; + bookId = chapter.bookId!; } // Notify about completed chapter if (onChapterComplete) { - onChapterComplete({ - index: chapterIndex, - title: chapterTitle, - duration, - status: 'completed', - bookId, - format - }); + onChapterComplete(chapter); } processedLength += text.length; @@ -483,9 +446,9 @@ export function PDFProvider({ children }: { children: ReactNode }) { const regenerateChapter = useCallback(async ( chapterIndex: number, bookId: string, - format: 'mp3' | 'm4b', + format: TTSAudiobookFormat, signal: AbortSignal - ): Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }> => { + ): Promise => { try { if (!pdfDocument) { throw new Error('No PDF document loaded'); @@ -556,28 +519,13 @@ export function PDFProvider({ children }: { children: ReactNode }) { backoffFactor: 2 }; - const audioBuffer = await withRetry( + const audioBuffer: TTSAudioBuffer = await withRetry( async () => { if (signal?.aborted) { throw new DOMException('Aborted', 'AbortError'); } - const ttsResponse = await fetch('/api/tts', { - method: 'POST', - headers: reqHeaders, - body: JSON.stringify(reqBody), - signal - }); - - if (!ttsResponse.ok) { - throw new Error(`TTS processing failed with status ${ttsResponse.status}`); - } - - const buffer = await ttsResponse.arrayBuffer(); - if (buffer.byteLength === 0) { - throw new Error('Received empty audio buffer from TTS'); - } - return buffer; + return await generateTTS(reqBody, reqHeaders, signal); }, retryOptions ); @@ -587,39 +535,15 @@ export function PDFProvider({ children }: { children: ReactNode }) { } // Send to server for conversion and storage - const convertResponse = await fetch('/api/audiobook', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chapterTitle, - buffer: Array.from(new Uint8Array(audioBuffer)), - bookId, - format, - chapterIndex - }), - signal - }); - - if (convertResponse.status === 499) { - throw new Error('cancelled'); - } - - if (!convertResponse.ok) { - throw new Error('Failed to convert audio chapter'); - } - - const { chapterIndex: returnedIndex, duration } = await convertResponse.json(); - - return { - index: returnedIndex, - title: chapterTitle, - duration, - status: 'completed', + const chapter = await createAudiobookChapter({ + chapterTitle, + buffer: Array.from(new Uint8Array(audioBuffer)), bookId, - format - }; + format, + chapterIndex + }, signal); + + return chapter; } catch (error) { if (error instanceof Error && (error.name === 'AbortError' || error.message.includes('cancelled'))) { diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 4124f13..afaa129 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -36,7 +36,7 @@ import { useMediaSession } from '@/hooks/audio/useMediaSession'; import { useAudioContext } from '@/hooks/audio/useAudioContext'; import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/dexie'; import { useBackgroundState } from '@/hooks/audio/useBackgroundState'; -import { withRetry } from '@/utils/audio'; +import { withRetry, generateTTS, alignAudio } from '@/lib/client'; import { preprocessSentenceForAudio, processTextToSentences } from '@/lib/nlp'; import { isKokoroModel } from '@/utils/voice'; import type { @@ -44,11 +44,14 @@ import type { TTSSmartMergeResult, TTSPageTurnEstimate, TTSPlaybackState, + TTSSentenceAlignment, + TTSAudioBuffer, +} from '@/types/tts'; +import type { TTSRequestPayload, TTSRequestHeaders, TTSRetryOptions, - TTSSentenceAlignment, -} from '@/types/tts'; +} from '@/types/client'; // Media globals declare global { @@ -731,16 +734,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * Generates and plays audio for the current sentence * * @param {string} sentence - The sentence to generate audio for - * @returns {Promise} The generated audio buffer + * @returns {Promise} The generated audio buffer */ - const getAudio = useCallback(async (sentence: string): Promise => { + const getAudio = useCallback(async (sentence: string): Promise => { const alignmentEnabledForCurrentDoc = (!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) || (isEPUB && epubHighlightEnabled && epubWordHighlightEnabled); // Helper to ensure we have an alignment for a given // sentence/audio pair, even when the audio itself is // served from the local cache. - const ensureAlignment = (arrayBuffer: ArrayBuffer) => { + const ensureAlignment = (arrayBuffer: TTSAudioBuffer) => { if (!alignmentEnabledForCurrentDoc) return; if (sentenceAlignmentCacheRef.current.has(sentence)) return; @@ -751,16 +754,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement audio: audioBytes, }; - void fetch('/api/whisper', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(alignmentBody), - }) - .then(async (res) => { - if (!res.ok) return; - const data = await res.json().catch(() => null); + void alignAudio(alignmentBody) + .then(async (data) => { if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) { return; } @@ -825,19 +820,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const arrayBuffer = await withRetry( async () => { - - const response = await fetch('/api/tts', { - method: 'POST', - headers: reqHeaders, - body: JSON.stringify(reqBody), - signal: controller.signal, - }); - - if (!response.ok) { - throw new Error('Failed to generate audio'); - } - - return response.arrayBuffer(); + return await generateTTS(reqBody, reqHeaders, controller.signal); }, retryOptions ); diff --git a/src/hooks/audio/useAudioCache.ts b/src/hooks/audio/useAudioCache.ts index 096176a..f71d0fa 100644 --- a/src/hooks/audio/useAudioCache.ts +++ b/src/hooks/audio/useAudioCache.ts @@ -2,6 +2,7 @@ import { useRef } from 'react'; import { LRUCache } from 'lru-cache'; +import type { TTSAudioBuffer } from '@/types/tts'; /** * Custom hook for managing audio cache using LRU strategy @@ -9,11 +10,11 @@ import { LRUCache } from 'lru-cache'; * @returns Object containing cache methods */ export function useAudioCache(maxSize = 50) { - const cacheRef = useRef(new LRUCache({ max: maxSize })); + const cacheRef = useRef(new LRUCache({ max: maxSize })); return { get: (key: string) => cacheRef.current.get(key), - set: (key: string, value: ArrayBuffer) => cacheRef.current.set(key, value), + set: (key: string, value: TTSAudioBuffer) => cacheRef.current.set(key, value), delete: (key: string) => cacheRef.current.delete(key), has: (key: string) => cacheRef.current.has(key), clear: () => cacheRef.current.clear(), diff --git a/src/hooks/audio/useVoiceManagement.ts b/src/hooks/audio/useVoiceManagement.ts index 6e5ed0b..67f0d6f 100644 --- a/src/hooks/audio/useVoiceManagement.ts +++ b/src/hooks/audio/useVoiceManagement.ts @@ -1,6 +1,7 @@ 'use client'; import { useState, useCallback } from 'react'; +import { getVoices } from '@/lib/client'; const DEFAULT_VOICES = ['alloy', 'ash', 'coral', 'echo', 'fable', 'onyx', 'nova', 'sage', 'shimmer']; @@ -23,18 +24,14 @@ export function useVoiceManagement( const fetchVoices = useCallback(async () => { try { console.log('Fetching voices...'); - const response = await fetch('/api/tts/voices', { - headers: { - 'x-openai-key': apiKey || '', - 'x-openai-base-url': baseUrl || '', - 'x-tts-provider': ttsProvider || 'openai', - 'x-tts-model': ttsModel || 'tts-1', - 'Content-Type': 'application/json', - }, + const data = await getVoices({ + 'x-openai-key': apiKey || '', + 'x-openai-base-url': baseUrl || '', + 'x-tts-provider': ttsProvider || 'openai', + 'x-tts-model': ttsModel || 'tts-1', + 'Content-Type': 'application/json', }); - if (!response.ok) throw new Error('Failed to fetch voices'); - const data = await response.json(); setAvailableVoices(data.voices || DEFAULT_VOICES); } catch (error) { console.error('Error fetching voices:', error); diff --git a/src/lib/client.ts b/src/lib/client.ts new file mode 100644 index 0000000..f6ca16d --- /dev/null +++ b/src/lib/client.ts @@ -0,0 +1,205 @@ +import type { + TTSRequestPayload, + TTSRequestHeaders, + TTSRetryOptions, + AudiobookStatusResponse, + CreateChapterPayload, + VoicesResponse, + AlignmentPayload, + AlignmentResponse +} from '@/types/client'; +import type { TTSAudiobookChapter, TTSAudioBuffer } from '@/types/tts'; + +/** + * Executes a function with exponential backoff retry logic + * @param operation Function to retry + * @param options Retry configuration options + * @returns Promise resolving to the operation result + */ +export const withRetry = async ( + operation: () => Promise, + options: TTSRetryOptions = {} +): Promise => { + const { + maxRetries = 3, + initialDelay = 1000, + maxDelay = 10000, + backoffFactor = 2 + } = options; + + let lastError: Error | null = null; + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await operation(); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // Do not retry on explicit cancellation/abort errors - surface them + // immediately so callers can stop work quickly when the user cancels. + if (lastError.name === 'AbortError' || lastError.message.includes('cancelled')) { + break; + } + + if (attempt === maxRetries - 1) { + break; + } + + const delay = Math.min( + initialDelay * Math.pow(backoffFactor, attempt), + maxDelay + ); + + console.log(`Retry attempt ${attempt + 1}/${maxRetries} failed. Retrying in ${delay}ms...`); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + throw lastError || new Error('Operation failed after retries'); +}; + +// --- Documents API --- + +export const convertDocxToPdf = async (file: File): Promise => { + const formData = new FormData(); + formData.append('file', file); + + const response = await fetch('/api/documents/docx-to-pdf', { + method: 'POST', + body: formData, + }); + + if (!response.ok) { + throw new Error('Failed to convert DOCX to PDF'); + } + + return await response.blob(); +}; + +export const deleteServerDocuments = async (): Promise => { + const response = await fetch('/api/documents', { + method: 'DELETE', + }); + if (!response.ok) { + throw new Error('Failed to delete server documents'); + } +}; + +// --- Audiobook API --- + + + +export const getAudiobookStatus = async (bookId: string): Promise => { + const response = await fetch(`/api/audiobook/status?bookId=${bookId}`); + if (!response.ok) { + throw new Error('Failed to fetch audiobook status'); + } + return await response.json(); +}; + + + +export const createAudiobookChapter = async ( + payload: CreateChapterPayload, + signal?: AbortSignal +): Promise => { + const response = await fetch(`/api/audiobook`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + signal + }); + + if (response.status === 499) { + throw new Error('cancelled'); + } + + if (!response.ok) { + throw new Error('Failed to convert audio chapter'); + } + + return await response.json(); +}; + +export const deleteAudiobook = async (bookId: string): Promise => { + const response = await fetch(`/api/audiobook?bookId=${bookId}`, { method: 'DELETE' }); + if (!response.ok) { + throw new Error('Reset failed'); + } +}; + +export const downloadAudiobook = async (bookId: string, format: string): Promise => { + const response = await fetch(`/api/audiobook?bookId=${bookId}&format=${format}`); + if (!response.ok) throw new Error('Download failed'); + return response; +}; + +export const deleteAudiobookChapter = async (bookId: string, chapterIndex: number): Promise => { + const response = await fetch(`/api/audiobook/chapter?bookId=${bookId}&chapterIndex=${chapterIndex}`, { + method: 'DELETE' + }); + if (!response.ok) { + throw new Error('Delete failed'); + } +}; + +export const downloadAudiobookChapter = async (bookId: string, chapterIndex: number): Promise => { + const response = await fetch(`/api/audiobook/chapter?bookId=${bookId}&chapterIndex=${chapterIndex}`); + if (!response.ok) throw new Error('Download failed'); + return await response.blob(); +}; + +// --- TTS API --- + + + +export const getVoices = async (headers: HeadersInit): Promise => { + const response = await fetch('/api/tts/voices', { + headers, + }); + + if (!response.ok) throw new Error('Failed to fetch voices'); + return await response.json(); +}; + +export const generateTTS = async ( + payload: TTSRequestPayload, + headers: TTSRequestHeaders, + signal?: AbortSignal +): Promise => { + const response = await fetch('/api/tts', { + method: 'POST', + headers: headers as HeadersInit, + body: JSON.stringify(payload), + signal + }); + + if (!response.ok) { + throw new Error(`TTS processing failed with status ${response.status}`); + } + + const buffer = await response.arrayBuffer(); + if (buffer.byteLength === 0) { + throw new Error('Received empty audio buffer from TTS'); + } + return buffer; +}; + +// --- Whisper API --- + + + +export const alignAudio = async (payload: AlignmentPayload): Promise => { + const response = await fetch('/api/whisper', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }); + + if (!response.ok) return null; + return await response.json(); +}; diff --git a/src/lib/pdf.ts b/src/lib/pdf.ts index a8e938b..fc4c120 100644 --- a/src/lib/pdf.ts +++ b/src/lib/pdf.ts @@ -2,7 +2,6 @@ import { pdfjs } from 'react-pdf'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist'; import "core-js/proposals/promise-with-resolvers"; -import { processTextToSentences } from '@/lib/nlp'; import type { TTSSentenceAlignment } from '@/types/tts'; import { CmpStr } from 'cmpstr'; diff --git a/src/types/client.ts b/src/types/client.ts new file mode 100644 index 0000000..113b6ad --- /dev/null +++ b/src/types/client.ts @@ -0,0 +1,67 @@ +import type { + TTSAudiobookChapter, + TTSSentenceAlignment, + TTSAudioBytes, + TTSAudiobookFormat, +} from '@/types/tts'; + +// --- TTS Client Request Types --- + +// Supported output formats for the TTS endpoint +export type TTSRequestFormat = 'mp3'; + +// JSON payload accepted by the /api/tts endpoint +export interface TTSRequestPayload { + text: string; + voice: string; + speed: number; + model?: string | null; + format?: TTSRequestFormat; + instructions?: string; +} + +// Headers used when calling the /api/tts endpoint from the client +export type TTSRequestHeaders = Record; + +// Options for retrying TTS requests on failure in withRetry +export interface TTSRetryOptions { + maxRetries?: number; + initialDelay?: number; + maxDelay?: number; + backoffFactor?: number; +} + +// --- Audiobook API Types --- + +export interface AudiobookStatusResponse { + exists: boolean; + chapters: TTSAudiobookChapter[]; + bookId: string | null; + hasComplete: boolean; +} + +export interface CreateChapterPayload { + chapterTitle: string; + buffer: TTSAudioBytes; // Array.from(new Uint8Array(audioBuffer)) + bookId: string; + format: TTSAudiobookFormat; + chapterIndex: number; +} + + +// --- TTS Voices API Types --- + +export interface VoicesResponse { + voices: string[]; +} + +// --- Whisper API Types --- + +export interface AlignmentPayload { + text: string; + audio: TTSAudioBytes; // Array.from(new Uint8Array(arrayBuffer)) +} + +export interface AlignmentResponse { + alignments: TTSSentenceAlignment[]; +} diff --git a/src/types/tts.ts b/src/types/tts.ts index a2744b2..5a944a4 100644 --- a/src/types/tts.ts +++ b/src/types/tts.ts @@ -1,5 +1,9 @@ export type TTSLocation = string | number; +// Core audio representations used across TTS and audiobook flows +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; // JSON-safe representation (Array.from(new Uint8Array(buffer))) + // Standardized error codes for the TTS API export type TTSErrorCode = | 'MISSING_PARAMETERS' @@ -15,22 +19,6 @@ export interface TTSError { details?: unknown; } -// Supported output formats for the TTS endpoint -export type TTSRequestFormat = 'mp3' | 'aac'; - -// JSON payload accepted by the /api/tts endpoint -export interface TTSRequestPayload { - text: string; - voice: string; - speed: number; - model?: string | null; - format?: TTSRequestFormat; - instructions?: string; -} - -// Headers used when calling the /api/tts endpoint from the client -export type TTSRequestHeaders = Record; - // Core playback state exposed by the TTS context export interface TTSPlaybackState { isPlaying: boolean; @@ -42,14 +30,6 @@ export interface TTSPlaybackState { currDocPages?: number; } -// Options for retrying TTS requests on failure in withRetry -export interface TTSRetryOptions { - maxRetries?: number; - initialDelay?: number; - maxDelay?: number; - backoffFactor?: number; -} - // Result of merging a continuation slice into the current text export interface TTSSmartMergeResult { text: string; @@ -79,6 +59,9 @@ export interface TTSSentenceAlignment { words: TTSSentenceWord[]; } +// Supported output formats for generated audiobooks +export type TTSAudiobookFormat = 'mp3' | 'm4b'; + // Metadata for an audiobook chapter export interface TTSAudiobookChapter { index: number; @@ -86,5 +69,5 @@ export interface TTSAudiobookChapter { duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; - format?: 'mp3' | 'm4b'; + format?: TTSAudiobookFormat; } diff --git a/src/utils/audio.ts b/src/utils/audio.ts deleted file mode 100644 index db46302..0000000 --- a/src/utils/audio.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { TTSRetryOptions } from '@/types/tts'; - -/** - * Executes a function with exponential backoff retry logic - * @param operation Function to retry - * @param options Retry configuration options - * @returns Promise resolving to the operation result - */ -export const withRetry = async ( - operation: () => Promise, - options: TTSRetryOptions = {} -): Promise => { - const { - maxRetries = 3, - initialDelay = 1000, - maxDelay = 10000, - backoffFactor = 2 - } = options; - - let lastError: Error | null = null; - - for (let attempt = 0; attempt < maxRetries; attempt++) { - try { - return await operation(); - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); - - // Do not retry on explicit cancellation/abort errors - surface them - // immediately so callers can stop work quickly when the user cancels. - if (lastError.name === 'AbortError' || lastError.message.includes('cancelled')) { - break; - } - - if (attempt === maxRetries - 1) { - break; - } - - const delay = Math.min( - initialDelay * Math.pow(backoffFactor, attempt), - maxDelay - ); - - console.log(`Retry attempt ${attempt + 1}/${maxRetries} failed. Retrying in ${delay}ms...`); - await new Promise(resolve => setTimeout(resolve, delay)); - } - } - - throw lastError || new Error('Operation failed after retries'); -}; \ No newline at end of file From 372c65f23e144362315897af5f2313cfbddc5603 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 22 Nov 2025 00:25:13 -0700 Subject: [PATCH 4/9] feat(whisper): integrate binary with build and docs The Dockerfile has been refactored to a multi-stage build, allowing the `whisper.cpp` CLI binary to be compiled and embedded within the application's runtime image. This enables word-by-word highlighting functionality when deployed via Docker. The `README.md` has been updated to include installation and configuration instructions for `whisper.cpp` when running locally. Additionally, the `WHISPER_CPP_BIN` environment variable has been added to `template.env` and the package version has been bumped to v1.1.0. --- Dockerfile | 46 +++++++++++++++++++++++++++++++++---- README.md | 64 ++++++++++++++++------------------------------------ package.json | 2 +- template.env | 5 +++- 4 files changed, 65 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52a26f9..3c36a2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,18 @@ -# Use Node.js slim image -FROM node:current-alpine +# Stage 1: build whisper.cpp (no model download – the app handles that) +FROM alpine:3.20 AS whisper-builder -# Add ffmpeg and libreoffice using Alpine package manager -RUN apk add --no-cache ffmpeg libreoffice-writer +RUN apk add --no-cache git cmake build-base + +WORKDIR /opt + +RUN git clone --depth 1 https://github.com/ggml-org/whisper.cpp.git && \ + cd whisper.cpp && \ + cmake -B build && \ + cmake --build build -j --config Release + + +# Stage 2: build the Next.js app +FROM node:lts-alpine AS app-builder # Install pnpm globally RUN npm install -g pnpm @@ -23,8 +33,34 @@ COPY . . RUN pnpm exec next telemetry disable RUN pnpm build + +# Stage 3: minimal runtime image +FROM node:current-alpine AS runner + +# Add runtime OS dependencies: +# - ffmpeg: required for audiobook export and word-by-word alignment (/api/whisper) +# - libreoffice-writer: required for DOCX → PDF conversion +RUN apk add --no-cache ffmpeg libreoffice-writer + +# Install pnpm globally for running the app +RUN npm install -g pnpm + +# App runtime directory +WORKDIR /app + +# Copy built app and dependencies from the builder stage +COPY --from=app-builder /app ./ + +# Copy the compiled whisper.cpp build output into the runtime image +# (includes whisper-cli and its shared libraries, e.g. libwhisper.so, libggml.so) +COPY --from=whisper-builder /opt/whisper.cpp/build /opt/whisper.cpp/build + +# Point the app at the compiled whisper-cli binary and ensure its libs are discoverable +ENV WHISPER_CPP_BIN=/opt/whisper.cpp/build/bin/whisper-cli +ENV LD_LIBRARY_PATH=/opt/whisper.cpp/build + # Expose the port the app runs on EXPOSE 3003 # Start the application -CMD ["pnpm", "start"] \ No newline at end of file +CMD ["pnpm", "start"] diff --git a/README.md b/README.md index 691881a..02dc60d 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,6 @@ OpenReader WebUI is an open source text to speech document reader web app built using Next.js, offering a TTS read along experience with narration for **EPUB, PDF, TXT, MD, and DOCX documents**. It supports multiple TTS providers including OpenAI, Deepinfra, and custom OpenAI-compatible endpoints like [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) and [Orpheus-FastAPI](https://github.com/Lex-au/Orpheus-FastAPI) -- 🧠 *(New)* **Smart Sentence-Aware Narration** merges sentences across pages/chapters for smoother TTS -- 🎧 *(New)* **Reliable Audiobook Export** in **m4b/mp3**, with resumable, chapter-based export and regeneration - 🎯 *(New)* **Multi-Provider TTS Support** - [**Kokoro-FastAPI**](https://github.com/remsky/Kokoro-FastAPI): Supporting multi-voice combinations (like `af_heart+af_bella`) - [**Orpheus-FastAPI**](https://github.com/Lex-au/Orpheus-FastAPI) @@ -20,56 +18,18 @@ OpenReader WebUI is an open source text to speech document reader web app built - **Cloud TTS Providers (requiring API keys)** - [**Deepinfra**](https://deepinfra.com/models/text-to-speech): Kokoro-82M + models with support for cloned voices and more - [**OpenAI API ($$)**](https://platform.openai.com/docs/pricing#transcription-and-speech): tts-1, tts-1-hd, and gpt-4o-mini-tts w/ instructions -- 🚀 *(New)* **Optimized Next.js TTS Proxy** with audio caching and optimized repeat playback -- 💾 *(Updated)* **Local-First Architecture** stores documents and more in-browser with Dexie.js - 📖 *(Updated)* **Read Along Experience** providing real-time text highlighting during playback (PDF/EPUB) + - *(New)* **Word-by-word** highlighting uses word-by-word timestamps generated server-side with [*whisper.cpp*](https://github.com/ggml-org/whisper.cpp) (optional) +- 🧠 *(New)* **Smart Sentence-Aware Narration** merges sentences across pages/chapters for smoother TTS +- 🎧 *(New)* **Reliable Audiobook Export** in **m4b/mp3**, with resumable, chapter-based export and regeneration +- 🚀 *(New)* **Optimized Next.js TTS Proxy** with audio caching and optimized repeat playback +- 💾 **Local-First Architecture** stores documents and more in-browser with Dexie.js - 🛜 **Optional Server-side documents** using backend `/docstore` for all users - 🎨 **Customizable Experience** - 🎨 Multiple app theme options - ⚙️ Various TTS and document handling settings - And more ... -
- - -### 🆕 What's New in v1.0.0 - - - -- 🧠 **Smart sentence continuation** - - Improved NLP handling of complex structures and quoted dialogue provides more natural sentence boundaries and a smoother audio-text flow. - - EPUB and PDF playback now use smarter sentence splitting and continuation metadata so sentences that cross page/chapter boundaries are merged before hitting the TTS API. - - This yields more natural narration and fewer awkward pauses when a sentence spans multiple pages or EPUB spine items. -- 📄 **Modernized PDF text highlighting pipeline** - - Real-time PDF text highlighting is now offloaded to a dedicated Web Worker so scrolling and playback controls remain responsive during narration. - - A new overlay-based highlighting system draws independent highlight layers on top of the PDF, avoiding interference with the underlying text layer. - - Upgraded fuzzy matching with Dice-based similarity improves the accuracy of mapping spoken words to on-screen text. - - A new per-device setting lets you enable or disable real-time PDF highlighting during playback for a more tailored reading experience. -- 🎧 **Chapter/page-based audiobook export with resume & regeneration** - - Per-chapter/per-page generation to disk with persistent `bookId` - - Resumable generation (can cancel and continue later) - - Per-chapter regeneration & deletion - - Final combined **M4B** or **MP3** download with embedded chapter metadata. -- 💾 **Dexie-backed local storage & sync** - - All document types (PDF, EPUB, TXT/MD-as-HTML) and config are stored via a unified Dexie layer on top of IndexedDB. - - Document lists use live Dexie queries (no manual refresh needed), and server sync now correctly includes text/markdown documents as part of the library backup. -- 🗣️ **Kokoro multi-voice selection & utilities** - - Kokoro models now support multi-voice combination, with provider-aware limits and helpers (not supported on OpenAI or Deepinfra) -- ⚡ **Faster, more efficient TTS backend proxy** - - In-memory **LRU caching** for audio responses with configurable size/TTL - - **ETag** support (`304` on cache hits) + `X-Cache` headers (`HIT` / `MISS` / `INFLIGHT`) -- 📄 **More robust DOCX → PDF conversion** - - DOCX conversion now uses isolated per-job LibreOffice profiles and temp directories, polls for a stable output file size, and aggressively cleans up temp files. - - This reduces cross-job interference and flakiness when converting multiple DOCX files in parallel. -- ♿ **Accessibility & layout improvements** - - Dialogs and folder toggles expose proper roles and ARIA attributes. - - PDF/EPUB/HTML readers use a full-height app shell with a sticky bottom TTS bar, improved scrollbars, and refined focus styles. -- ✅ **End-to-end Playwright test suite with TTS mocks** - - Deterministic TTS responses in tests via a reusable Playwright route mock. - - Coverage for accessibility, upload, navigation, folder management, deletion flows, audiobook generation/export and playback across all document types. - -
- ## 🐳 Docker Quick Start ### Prerequisites @@ -194,6 +154,20 @@ Optionally required for different features: ```bash brew install libreoffice ``` +- [whisper.cpp](https://github.com/ggml-org/whisper.cpp) (optional, required for word-by-word highlighting) + ```bash + # clone and build whisper.cpp (no model download needed – OpenReader handles that) + git clone https://github.com/ggml-org/whisper.cpp.git + cd whisper.cpp + cmake -B build + cmake --build build -j --config Release + + # point OpenReader to the compiled whisper-cli binary + echo WHISPER_CPP_BIN=\"$(pwd)/build/bin/whisper-cli\" + ``` + + > **Note:** The `WHISPER_CPP_BIN` path should be set in your `.env` file for OpenReader to use word-by-word highlighting features. + ### Steps 1. Clone the repository: diff --git a/package.json b/package.json index e62e007..acc1eba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openreader-webui", - "version": "v1.0.1", + "version": "v1.1.0", "private": true, "scripts": { "dev": "next dev --turbopack -p 3003", diff --git a/template.env b/template.env index 5a24657..6d34cc4 100644 --- a/template.env +++ b/template.env @@ -5,4 +5,7 @@ API_KEY=api_key_here_if_needed # OpenAI API Base URL (default) # To use a local TTS model server, I suggest using https://github.com/remsky/Kokoro-FastAPI -API_BASE=https://api.openai.com/v1 \ No newline at end of file +API_BASE=https://api.openai.com/v1 + +# Path to your local whisper.cpp CLI binary +WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli \ No newline at end of file From 773778ee9073c02db579582f857ae2dda0c6b7b1 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 22 Nov 2025 01:33:53 -0700 Subject: [PATCH 5/9] feat(epub): add custom reader navigation and TOC - Implement custom previous/next page buttons for EPUB viewer. - Display current page number out of total pages. - Introduce an in-viewer, toggleable table of contents (TOC) for quick chapter navigation. - Hide default `react-reader` navigation arrows and title bar to prevent redundancy. - Adjust `react-reader` styles to optimize content area and ensure custom controls are visible. - Update description for EPUB theme setting in document settings. - Add `ChevronLeftIcon` and `ChevronRightIcon` to icon library. - Update Playwright tests to reflect new navigation button labels. --- src/components/DocumentSettings.tsx | 4 +- src/components/EPUBViewer.tsx | 67 +++++++++++++++++++++++++++-- src/components/icons/Icons.tsx | 44 +++++++++++++++++++ src/hooks/epub/useEPUBTheme.ts | 67 +++++++++++++++++++++++++---- tests/upload.spec.ts | 8 ++-- 5 files changed, 171 insertions(+), 19 deletions(-) diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index 84abae2..f78d482 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -414,10 +414,10 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { onChange={(e) => updateConfigKey('epubTheme', e.target.checked)} className="form-checkbox h-4 w-4 text-accent rounded border-muted" /> - Use theme (experimental) + Use theme

- Apply the current app theme to the EPUB viewer + Apply the current app theme to the EPUB viewer background and text colors

)} diff --git a/src/components/EPUBViewer.tsx b/src/components/EPUBViewer.tsx index 7f0e5c3..7289538 100644 --- a/src/components/EPUBViewer.tsx +++ b/src/components/EPUBViewer.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useEffect, useRef, useCallback } from 'react'; +import { useEffect, useRef, useCallback, useState } from 'react'; import dynamic from 'next/dynamic'; import { useEPUB } from '@/contexts/EPUBContext'; import { useTTS } from '@/contexts/TTSContext'; @@ -8,6 +8,7 @@ import { useConfig } from '@/contexts/ConfigContext'; import { DocumentSkeleton } from '@/components/DocumentSkeleton'; import { useEPUBTheme, getThemeStyles } from '@/hooks/epub/useEPUBTheme'; import { useEPUBResize } from '@/hooks/epub/useEPUBResize'; +import { DotsVerticalIcon, ChevronLeftIcon, ChevronRightIcon } from '@/components/icons/Icons'; const ReactReader = dynamic(() => import('react-reader').then(mod => mod.ReactReader), { ssr: false, @@ -19,9 +20,12 @@ interface EPUBViewerProps { } export function EPUBViewer({ className = '' }: EPUBViewerProps) { + const [isTocOpen, setIsTocOpen] = useState(false); const { currDocData, currDocName, + currDocPage, + currDocPages, locationRef, handleLocationChanged, bookRef, @@ -116,7 +120,62 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { return (
-
+
+
+ + +
+ {currDocPages !== undefined && typeof currDocPage === 'number' && ( + + {currDocPage} / {currDocPages} + + )} + +
+ {isTocOpen && tocRef.current && tocRef.current.length > 0 && ( +
+
Skip to chapters
+
+ {tocRef.current.map((item, index) => ( + + ))} +
+
+ )} +
} key={'epub-reader'} @@ -125,8 +184,8 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { url={currDocData} title={currDocName} tocChanged={(_toc) => (tocRef.current = _toc)} - showToc={true} - readerStyles={epubTheme && getThemeStyles() || undefined} + showToc={false} + readerStyles={getThemeStyles(epubTheme)} getRendition={(_rendition) => { setRendition(_rendition); updateTheme(); diff --git a/src/components/icons/Icons.tsx b/src/components/icons/Icons.tsx index ffbfead..292e937 100644 --- a/src/components/icons/Icons.tsx +++ b/src/components/icons/Icons.tsx @@ -373,6 +373,50 @@ export function DotsVerticalIcon(props: React.SVGProps) { ); } +export function ChevronLeftIcon(props: React.SVGProps) { + return ( + + + + ); +} + +export function ChevronRightIcon(props: React.SVGProps) { + return ( + + + + ); +} + export function SpeedometerIcon(props: React.SVGProps) { return ( { - const baseStyle = { - ...ReactReaderStyle, - readerArea: { - ...ReactReaderStyle.readerArea, - transition: undefined, - } - }; +// Returns ReactReader styles, with: +// - default look when epubTheme === false (except hiding built-in arrows) +// - themed colors + layout tweaks when epubTheme === true +export const getThemeStyles = (epubTheme: boolean): IReactReaderStyle => { + const baseStyle = ReactReaderStyle; + + // Always hide the built-in prev/next arrow buttons so we can + // provide our own navigation controls outside the reader. + if (!epubTheme) { + return { + ...baseStyle, + reader: { + ...baseStyle.reader, + // Always tighten the inset a bit for better use of space + top: 8, + left: 8, + right: 8, + bottom: 8, + }, + prev: { + ...baseStyle.prev, + display: 'none', + pointerEvents: 'none', + }, + next: { + ...baseStyle.next, + display: 'none', + pointerEvents: 'none', + }, + titleArea: { + ...baseStyle.titleArea, + display: 'none', + }, + }; + } const colors = { background: getComputedStyle(document.documentElement).getPropertyValue('--background'), @@ -21,6 +48,25 @@ export const getThemeStyles = (): IReactReaderStyle => { return { ...baseStyle, + reader: { + ...baseStyle.reader, + // Reduce the large default inset (50px 50px 20px) + // so the EPUB content can use more of the available area. + top: 8, + left: 8, + right: 8, + bottom: 8, + }, + prev: { + ...baseStyle.prev, + display: 'none', + pointerEvents: 'none', + }, + next: { + ...baseStyle.next, + display: 'none', + pointerEvents: 'none', + }, arrow: { ...baseStyle.arrow, color: colors.foreground, @@ -54,6 +100,9 @@ export const getThemeStyles = (): IReactReaderStyle => { tocButton: { ...baseStyle.tocButton, color: colors.muted, + // Ensure the TOC toggle sits above the swipe wrapper + // and text iframe, avoiding z-index conflicts. + zIndex: 300, }, tocAreaButton: { ...baseStyle.tocAreaButton, @@ -117,4 +166,4 @@ export const useEPUBTheme = (epubTheme: boolean, rendition: Rendition | undefine }, [epubTheme, rendition, updateTheme]); return { updateTheme }; -}; \ No newline at end of file +}; diff --git a/tests/upload.spec.ts b/tests/upload.spec.ts index fcf2051..dc56f38 100644 --- a/tests/upload.spec.ts +++ b/tests/upload.spec.ts @@ -43,9 +43,9 @@ test.describe('Document Upload Tests', () => { test('displays an EPUB document', async ({ page }) => { await uploadAndDisplay(page, 'sample.epub'); await expectViewerForFile(page, 'sample.epub'); - // Keep navigation button assertions - await expect(page.getByRole('button', { name: '‹' })).toBeVisible(); - await expect(page.getByRole('button', { name: '›' })).toBeVisible(); + // Navigation controls should be exposed via accessible labels + await expect(page.getByRole('button', { name: 'Previous section' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Next section' })).toBeVisible(); }); test('displays a DOCX document as PDF after conversion', async ({ page }) => { @@ -126,4 +126,4 @@ test.describe('Document Upload Tests', () => { // Also ensure no link with that filename exists await expect(page.getByRole('link', { name: /unsupported\.xyz/i })).toHaveCount(0); }); -}); \ No newline at end of file +}); From 7046666b3f8c7bebe7ced93c281c43d2dcedfb6c Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 22 Nov 2025 01:57:27 -0700 Subject: [PATCH 6/9] fix(tts): ensure robust caching for audio and alignments - New buildCacheKey function creates unique identifiers for TTS cache entries. - Cache keys now include sentence, voice, speed, provider, and model parameters. - Prevents serving cached audio or alignment data that mismatches current TTS parameters. - Removes redundant audioCache.clear() calls when voice or speed change, as entries are now distinct. --- src/contexts/TTSContext.tsx | 80 +++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index afaa129..293821f 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -246,6 +246,22 @@ const mergeContinuation = (text: string, nextText: string): TTSSmartMergeResult }; }; +const buildCacheKey = ( + sentence: string, + voice: string, + speed: number, + provider: string, + model: string, +) => { + return [ + `provider=${provider || ''}`, + `model=${model || ''}`, + `voice=${voice || ''}`, + `speed=${Number.isFinite(speed) ? speed : ''}`, + `text=${sentence}`, + ].join('|'); +}; + // Create the context const TTSContext = createContext(undefined); @@ -745,7 +761,14 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // served from the local cache. const ensureAlignment = (arrayBuffer: TTSAudioBuffer) => { if (!alignmentEnabledForCurrentDoc) return; - if (sentenceAlignmentCacheRef.current.has(sentence)) return; + const alignmentKey = buildCacheKey( + sentence, + voice, + speed, + configTTSProvider, + ttsModel, + ); + if (sentenceAlignmentCacheRef.current.has(alignmentKey)) return; try { const audioBytes = Array.from(new Uint8Array(arrayBuffer)); @@ -760,7 +783,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement return; } const alignment = data.alignments[0] as TTSSentenceAlignment; - sentenceAlignmentCacheRef.current.set(sentence, alignment); + sentenceAlignmentCacheRef.current.set(alignmentKey, alignment); const currentSentence = sentencesRef.current[currentIndexRef.current]; if (currentSentence === sentence) { @@ -776,8 +799,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } }; + const audioCacheKey = buildCacheKey( + sentence, + voice, + speed, + configTTSProvider, + ttsModel, + ); + // Check if the audio is already cached - const cachedAudio = audioCache.get(sentence); + const cachedAudio = audioCache.get(audioCacheKey); if (cachedAudio) { console.log('Using cached audio for sentence:', sentence.substring(0, 20)); // If we have audio but no alignment (e.g. after a @@ -829,7 +860,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement activeAbortControllers.current.delete(controller); // Cache the array buffer - audioCache.set(sentence, arrayBuffer); + audioCache.set(audioCacheKey, arrayBuffer); // Fire-and-forget alignment request; do not block audio playback ensureAlignment(arrayBuffer); @@ -1084,7 +1115,14 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const playAudio = useCallback(async () => { const sentence = sentences[currentIndex]; - const cachedAlignment = sentenceAlignmentCacheRef.current.get(sentence); + const alignmentKey = buildCacheKey( + sentence, + voice, + speed, + configTTSProvider, + ttsModel, + ); + const cachedAlignment = sentenceAlignmentCacheRef.current.get(alignmentKey); if (cachedAlignment) { setCurrentSentenceAlignment(cachedAlignment); setCurrentWordIndex(null); @@ -1097,7 +1135,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement if (howl) { howl.play(); } - }, [sentences, currentIndex, playSentenceWithHowl]); + }, [sentences, currentIndex, playSentenceWithHowl, voice, speed, configTTSProvider, ttsModel]); // Place useBackgroundState after playAudio is defined const isBackgrounded = useBackgroundState({ @@ -1153,16 +1191,26 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const preloadNextAudio = useCallback(async () => { try { const nextSentence = sentences[currentIndex + 1]; - if (nextSentence && !audioCache.has(nextSentence) && !preloadRequests.current.has(nextSentence)) { + if (nextSentence) { + const nextKey = buildCacheKey( + nextSentence, + voice, + speed, + configTTSProvider, + ttsModel, + ); + + if (!audioCache.has(nextKey) && !preloadRequests.current.has(nextSentence)) { // Start preloading but don't wait for it to complete - processSentence(nextSentence, true).catch(error => { - console.error('Error preloading next sentence:', error); - }); + processSentence(nextSentence, true).catch(error => { + console.error('Error preloading next sentence:', error); + }); + } } } catch (error) { console.error('Error initiating preload:', error); } - }, [currentIndex, sentences, audioCache, processSentence]); + }, [currentIndex, sentences, audioCache, processSentence, voice, speed, configTTSProvider, ttsModel]); /** * Main Playback Driver @@ -1251,9 +1299,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement abortAudio(true); // Clear pending requests since speed changed setActiveHowl(null); - // Update speed, clear cache, and config + // Update speed and config setSpeed(newSpeed); - audioCache.clear(); // Update config after state changes updateConfigKey('voiceSpeed', newSpeed).then(() => { @@ -1263,7 +1310,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setIsPlaying(true); } }); - }, [abortAudio, updateConfigKey, audioCache, isPlaying]); + }, [abortAudio, updateConfigKey, isPlaying]); /** * Sets the voice and restarts the playback @@ -1284,9 +1331,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement abortAudio(true); // Clear pending requests since voice changed setActiveHowl(null); - // Update voice, clear cache, and config + // Update voice and config setVoice(newVoice); - audioCache.clear(); // Update config after state changes updateConfigKey('voice', newVoice).then(() => { @@ -1296,7 +1342,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setIsPlaying(true); } }); - }, [abortAudio, updateConfigKey, audioCache, isPlaying]); + }, [abortAudio, updateConfigKey, isPlaying]); /** * Sets the audio player speed and restarts the playback From e39a5b8bcf077480a39b07d33103aab823dc14a4 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 22 Nov 2025 02:06:28 -0700 Subject: [PATCH 7/9] refactor(player): simplify popover input focus logic Removed `isPopoverOpen` state and associated `useEffect` hook. Auto-focus and select logic for the input are now directly handled within `handlePopoverOpen`, reducing state management complexity. --- src/components/player/Navigator.tsx | 86 ++++++++++++----------------- 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/src/components/player/Navigator.tsx b/src/components/player/Navigator.tsx index 20b1a98..dfbcceb 100644 --- a/src/components/player/Navigator.tsx +++ b/src/components/player/Navigator.tsx @@ -9,25 +9,12 @@ export const Navigator = ({ currentPage, numPages, skipToLocation }: { skipToLocation: (location: string | number, shouldPause?: boolean) => void; }) => { const [inputValue, setInputValue] = useState(''); - const [isPopoverOpen, setIsPopoverOpen] = useState(false); const inputRef = useRef(null); useEffect(() => { setInputValue(currentPage.toString()); }, [currentPage]); - // Auto-focus and select input when popover opens - useEffect(() => { - if (isPopoverOpen && inputRef.current) { - // Small delay to ensure the popover is fully rendered - const timer = setTimeout(() => { - inputRef.current?.focus(); - inputRef.current?.select(); - }, 50); - return () => clearTimeout(timer); - } - }, [isPopoverOpen]); - const handleInputChange = (e: React.ChangeEvent) => { // Only allow numbers const value = e.target.value.replace(/[^0-9]/g, ''); @@ -56,6 +43,11 @@ export const Navigator = ({ currentPage, numPages, skipToLocation }: { const handlePopoverOpen = () => { setInputValue(''); // Clear input when popup opens + // Auto-focus and select input shortly after opening + setTimeout(() => { + inputRef.current?.focus(); + inputRef.current?.select(); + }, 50); }; return ( @@ -73,44 +65,34 @@ export const Navigator = ({ currentPage, numPages, skipToLocation }: { {/* Page number popup */} - - {({ open }) => { - if (open !== isPopoverOpen) { - setIsPopoverOpen(open); - } - - return ( - <> - -

- {currentPage} / {numPages || 1} -

- - -
-
Go to page
- -
of {numPages || 1}
-
- - - ); - }} + + +

+ {currentPage} / {numPages || 1} +

+ + +
+
Go to page
+ +
of {numPages || 1}
+
+ {/* Page forward */} @@ -126,4 +108,4 @@ export const Navigator = ({ currentPage, numPages, skipToLocation }: {
); -} \ No newline at end of file +} From e86782bf383d79cea84a51d4934328914a0a0b46 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 22 Nov 2025 15:18:46 -0700 Subject: [PATCH 8/9] feat: improve document list and self-hosting clarity - Introduce a toggleable grid/list view for the document list, enhancing organization and display flexibility. This involved updates across `DocumentList`, `DocumentFolder`, `DocumentListItem`, and `SortControls`. - Add a new `CodeBlock` component and integrate detailed Docker-based self-hosting instructions into the footer. - Enhance privacy policy popover with clearer details on Deepinfra usage and strongly recommend self-hosting for secure experience. - Implement conditional rendering and feature gating based on the `isDev` environment variable, differentiating features between the production demo and self-hosted instances. Affected components include `page.tsx`, `DocumentSettings.tsx`, `SettingsModal.tsx`, and `config.ts`. - Clarify that advanced features like audiobook export and word-by-word highlighting via `whisper.cpp` are exclusive to self-hosted setups and are disabled in the demo. - Expand the list of supported document types on the homepage to include MD and TXT. - Integrate new `ListIcon`, `GridIcon`, and `CopyIcon` to support UI enhancements. - Add a custom `xs` breakpoint in `tailwind.config.ts` for improved responsive design. --- src/app/page.tsx | 8 +- src/components/CodeBlock.tsx | 32 ++++++++ src/components/DocumentSettings.tsx | 20 ++--- src/components/EPUBViewer.tsx | 13 ++-- src/components/Footer.tsx | 79 +++++++++++++++++--- src/components/SettingsModal.tsx | 2 +- src/components/doclist/DocumentFolder.tsx | 9 ++- src/components/doclist/DocumentList.tsx | 42 ++++++----- src/components/doclist/DocumentListItem.tsx | 8 +- src/components/doclist/SortControls.tsx | 83 ++++++++++++++------- src/components/icons/Icons.tsx | 52 +++++++++++++ src/types/config.ts | 11 ++- src/types/documents.ts | 1 + tailwind.config.ts | 3 + 14 files changed, 279 insertions(+), 84 deletions(-) create mode 100644 src/components/CodeBlock.tsx diff --git a/src/app/page.tsx b/src/app/page.tsx index 286c7e5..24a4558 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,7 +1,7 @@ import { HomeContent } from '@/components/HomeContent'; import { SettingsModal } from '@/components/SettingsModal'; -// Home page redesigned for fullscreen layout: hero + document area. +const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; export default function Home() { return ( @@ -10,9 +10,9 @@ export default function Home() {

OpenReader WebUI

-

- Bring your own text-to-speech API. - Read & listen to PDF, EPUB & HTML documents with high quality voices. +

+ Open source document reader web app {isDev ? 'self-hosted server' : 'demo'}. + Read & listen to PDF, EPUB, MD, and TXT documents with high quality text to speech voices.

diff --git a/src/components/CodeBlock.tsx b/src/components/CodeBlock.tsx new file mode 100644 index 0000000..a887c96 --- /dev/null +++ b/src/components/CodeBlock.tsx @@ -0,0 +1,32 @@ +'use client'; + +import { useState } from 'react'; +import { CopyIcon, CheckIcon } from '@/components/icons/Icons'; + +export function CodeBlock({ children }: { children: string }) { + const [copied, setCopied] = useState(false); + + const handleCopy = async () => { + await navigator.clipboard.writeText(children); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + return ( +
+ 
+        {children}
+
+ +
+ ); +} diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index f78d482..515d300 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -141,16 +141,18 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { leaveTo="opacity-0 scale-95" > - {isDev && !html &&
+ {!html &&
} @@ -352,18 +354,18 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { updateConfigKey('pdfWordHighlightEnabled', e.target.checked) } - className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50" + className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50 disabled:cursor-not-allowed" /> Word-by-word

- Highlight individual words using audio timestamps generated by whisper.cpp + Highlight individual words using audio timestamps generated by whisper.cpp {!isDev && '(requires self-hosted)'}

@@ -389,18 +391,18 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { updateConfigKey('epubWordHighlightEnabled', e.target.checked) } - className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50" + className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50 disabled:cursor-not-allowed" /> Word-by-word

- Highlight individual words using audio timestamps generated by whisper.cpp + Highlight individual words using audio timestamps generated by whisper.cpp {!isDev && '(requires self-hosted)'}

diff --git a/src/components/EPUBViewer.tsx b/src/components/EPUBViewer.tsx index 7289538..3c38034 100644 --- a/src/components/EPUBViewer.tsx +++ b/src/components/EPUBViewer.tsx @@ -156,18 +156,21 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { {isTocOpen && tocRef.current && tocRef.current.length > 0 && (
Skip to chapters
-
+
{tocRef.current.map((item, index) => ( diff --git a/src/components/Footer.tsx b/src/components/Footer.tsx index b81abaa..8b0182b 100644 --- a/src/components/Footer.tsx +++ b/src/components/Footer.tsx @@ -1,10 +1,14 @@ import { Popover, PopoverButton, PopoverPanel } from '@headlessui/react' import { GithubIcon } from '@/components/icons/Icons' +import { CodeBlock } from '@/components/CodeBlock' + +const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; + export function Footer() { return ( -