diff --git a/src/app/api/whisper/route.ts b/src/app/api/whisper/route.ts new file mode 100644 index 0000000..146e452 --- /dev/null +++ b/src/app/api/whisper/route.ts @@ -0,0 +1,451 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { createHash, randomUUID } from 'crypto'; +import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { spawn } from 'child_process'; +import type { TTSSentenceAlignment } from '@/types/tts'; +import { preprocessSentenceForAudio } from '@/lib/nlp'; + +export const runtime = 'nodejs'; + +interface WhisperRequestBody { + text: string; + audio: number[]; // raw bytes from Uint8Array + lang?: string; +} + +interface WhisperAlignmentOptions { + engine?: 'whisper.cpp'; + lang?: string; +} + +interface WhisperWord { + start: number; + end: number; + word: string; +} + +// Simple in-memory cache keyed by a hash of text+lang+audio length +const alignmentCache = new Map(); + +// Model management using a fixed tiny.en GGML model stored under docstore/model +const MODEL_NAME = 'ggml-tiny.en.bin'; +const MODEL_URL = + 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin'; +const DOCSTORE_DIR = join(process.cwd(), 'docstore'); +const MODEL_DIR = join(DOCSTORE_DIR, 'model'); +const MODEL_PATH = join(MODEL_DIR, MODEL_NAME); +const modelReadyPromises = new Map>(); + +async function ensureModelAvailable(): Promise { + // Fast path: model already present + try { + await access(MODEL_PATH); + return; + } catch { + // fall through to download + } + + const existing = modelReadyPromises.get(MODEL_PATH); + if (existing) return existing; + + const promise = (async () => { + try { + await access(MODEL_PATH); + return; + } catch { + // still missing + } + + await mkdir(MODEL_DIR, { recursive: true }); + + const res = await fetch(MODEL_URL); + if (!res.ok) { + throw new Error( + `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}` + ); + } + + const arrayBuffer = await res.arrayBuffer(); + await writeFile(MODEL_PATH, Buffer.from(arrayBuffer)); + })(); + + modelReadyPromises.set(MODEL_PATH, promise); + return promise; +} + +async function runWhisperCpp( + wavPath: string, + opts: WhisperAlignmentOptions +): Promise { + const binary = process.env.WHISPER_CPP_BIN; + if (!binary) { + throw new Error( + 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.' + ); + } + + await ensureModelAvailable(); + + return new Promise((resolve, reject) => { + const jsonBase = `${wavPath}.json_out`; + const jsonPath = `${jsonBase}.json`; + // Request full JSON output (including token-level details when available) + // and ask whisper-cli not to print anything to stdout. + const args = [ + '-m', + MODEL_PATH, + '-f', + wavPath, + '-of', + jsonBase, + '-ojf', + '-np', + ]; + + if (opts.lang) { + args.push('-l', opts.lang); + } + + const child = spawn(binary, args); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('error', (err) => { + reject(err); + }); + + child.on('close', (code) => { + if (code !== 0) { + console.error('whisper-cli exited with error', { + code, + wavPath, + stdout: stdout.slice(0, 500), + stderr: stderr.slice(0, 500), + }); + return reject( + new Error( + `whisper.cpp exited with code ${code}: ${stderr || stdout}` + ) + ); + } + + readFile(jsonPath, 'utf-8') + .then((content: string) => { + const words: WhisperWord[] = []; + const parsed = JSON.parse(content) as { + transcription?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + tokens?: Array<{ + text?: string; + timestamps?: { from?: string; to?: string }; + offsets?: { from?: number; to?: number }; + }>; + }>; + }; + + const transcription = parsed.transcription; + + const parseTimecode = (value?: string): number | null => { + if (!value) return null; + const m = value.match(/(\d+):(\d+):(\d+),(\d+)/); + if (!m) return null; + const h = Number(m[1]); + const min = Number(m[2]); + const s = Number(m[3]); + const ms = Number(m[4]); + if ( + Number.isNaN(h) || + Number.isNaN(min) || + Number.isNaN(s) || + Number.isNaN(ms) + ) { + return null; + } + return h * 3600 + min * 60 + s + ms / 1000; + }; + + if (Array.isArray(transcription)) { + for (const seg of transcription) { + const segText = (seg.text || '').trim(); + const segStartSecFromTs = parseTimecode( + seg.timestamps?.from + ); + const segEndSecFromTs = parseTimecode(seg.timestamps?.to); + const segStartSecFromMs = + typeof seg.offsets?.from === 'number' + ? seg.offsets.from / 1000 + : null; + const segEndSecFromMs = + typeof seg.offsets?.to === 'number' + ? seg.offsets.to / 1000 + : null; + + const segStartSec = + segStartSecFromTs ?? + segStartSecFromMs ?? + 0; + const segEndSec = + segEndSecFromTs ?? + segEndSecFromMs ?? + segStartSec; + + const tokens = Array.isArray(seg.tokens) + ? seg.tokens + : []; + + if (tokens.length > 0) { + for (const token of tokens) { + const rawText = token.text || ''; + const tokenText = rawText.trim(); + // Skip special markers like [_BEG_] + if (!tokenText || /^\[.*\]$/.test(tokenText)) continue; + + const tokStartSecFromTs = parseTimecode( + token.timestamps?.from + ); + const tokEndSecFromTs = parseTimecode( + token.timestamps?.to + ); + const tokStartSecFromMs = + typeof token.offsets?.from === 'number' + ? token.offsets.from / 1000 + : null; + const tokEndSecFromMs = + typeof token.offsets?.to === 'number' + ? token.offsets.to / 1000 + : null; + + const startSec = + tokStartSecFromTs ?? + tokStartSecFromMs ?? + segStartSec; + const endSec = + tokEndSecFromTs ?? + tokEndSecFromMs ?? + segEndSec; + + words.push({ + word: tokenText, + start: startSec, + end: endSec, + }); + } + } else if (segText) { + // Fallback: no token list, approximate per-word timing within the segment + const segTokens = segText.split(/\s+/).filter(Boolean); + if (segTokens.length) { + const totalDur = Math.max(segEndSec - segStartSec, 0); + const step = + segTokens.length > 0 + ? totalDur / segTokens.length + : 0; + segTokens.forEach((token, index) => { + const wStart = + step > 0 + ? segStartSec + step * index + : segStartSec; + const wEnd = + step > 0 + ? index === segTokens.length - 1 + ? segEndSec + : segStartSec + step * (index + 1) + : segEndSec; + words.push({ + word: token, + start: wStart, + end: wEnd, + }); + }); + } + } + } + } + + resolve(words); + }) + .catch((err: unknown) => { + reject(err); + }); + }); + }); +} + +function mapWordsToSentenceOffsets( + sentence: string, + words: WhisperWord[] +): TTSSentenceAlignment { + const normalizedSentence = preprocessSentenceForAudio(sentence); + let cursor = 0; + + const alignedWords = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = normalizedSentence + .toLowerCase() + .indexOf(token.toLowerCase(), cursor); + + const start = + idx !== -1 + ? idx + : cursor; + const end = start + token.length; + + cursor = end; + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords.filter((w) => w.text.length > 0), + }; +} + +async function alignAudioWithText( + audioBuffer: ArrayBuffer, + text: string, + cacheKey?: string, + opts: WhisperAlignmentOptions = {} +): Promise { + if (!text.trim()) { + return []; + } + + if (cacheKey && alignmentCache.has(cacheKey)) { + return alignmentCache.get(cacheKey)!; + } + + const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); + const inputPath = join(tmpBase, `${randomUUID()}-input.bin`); + const wavPath = join(tmpBase, `${randomUUID()}-input.wav`); + + try { + await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); + + await new Promise((resolve, reject) => { + const ffmpeg = spawn('ffmpeg', [ + '-y', + '-i', + inputPath, + '-ar', + '16000', + '-ac', + '1', + wavPath, + ]); + + let stderr = ''; + ffmpeg.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('error', (err) => { + reject(err); + }); + + ffmpeg.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject( + new Error(`ffmpeg failed with code ${code}: ${stderr}`) + ); + } + }); + }); + + const words = await runWhisperCpp(wavPath, opts); + const alignment = mapWordsToSentenceOffsets(text, words); + const result: TTSSentenceAlignment[] = [alignment]; + + if (cacheKey) { + alignmentCache.set(cacheKey, result); + } + + return result; + } finally { + try { + await rm(tmpBase, { recursive: true, force: true }); + } catch { + // ignore + } + } +} + +function makeCacheKey(input: WhisperRequestBody) { + const hash = createHash('sha256') + .update( + JSON.stringify({ + text: input.text, + lang: input.lang || '', + audioLen: input.audio?.length || 0, + }) + ) + .digest('hex'); + return hash; +} + +export async function POST(req: NextRequest) { + try { + const body = (await req.json()) as WhisperRequestBody; + const { text, audio, lang } = body; + + if (!text || !audio || !Array.isArray(audio)) { + return NextResponse.json( + { error: 'Missing text or audio in request body' }, + { status: 400 } + ); + } + + const cacheKey = makeCacheKey(body); + const audioBuffer = new Uint8Array(audio).buffer; + + const alignments: TTSSentenceAlignment[] = await alignAudioWithText( + audioBuffer, + text, + cacheKey, + { engine: 'whisper.cpp', lang } + ); + + return NextResponse.json({ alignments }, { status: 200 }); + } catch (error) { + console.error('Error in whisper route:', error); + return NextResponse.json( + { + error: 'WHISPER_ALIGNMENT_FAILED', + message: 'Failed to compute word-level alignment', + }, + { status: 500 } + ); + } +} + diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index b216730..76a3e7f 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -35,6 +35,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: { updateConfigKey, pdfHighlightEnabled, epubHighlightEnabled, + pdfWordHighlightEnabled, + epubWordHighlightEnabled, } = useConfig(); const { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter } = useEPUB(); const { createFullAudioBook: createPDFAudioBook, regenerateChapter: regeneratePDFChapter } = usePDF(); @@ -324,40 +326,82 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {

- Merge sentences across page or section breaks for smoother TTS. + Merge sentences across page or section breaks

)} {!epub && !html && ( -
- -

- Show visual highlighting in the PDF viewer while TTS is reading. -

+
+
+ +

+ Visual text playback highlighting in the PDF viewer +

+
+
+ +

+ Highlight individual words using audio timestamps generated by whisper.cpp +

+
)} {epub && ( -
- -

- Show visual highlighting in the EPUB viewer while TTS is reading. -

+
+
+ +

+ Visual text playback highlighting in the EPUB viewer +

+
+
+ +

+ Highlight individual words using audio timestamps generated by whisper.cpp +

+
)} {epub && ( diff --git a/src/components/EPUBViewer.tsx b/src/components/EPUBViewer.tsx index 03e101a..7f0e5c3 100644 --- a/src/components/EPUBViewer.tsx +++ b/src/components/EPUBViewer.tsx @@ -30,10 +30,18 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { setRendition, extractPageText, highlightPattern, - clearHighlights + clearHighlights, + highlightWordIndex, + clearWordHighlights } = useEPUB(); - const { registerLocationChangeHandler, pause, currentSentence } = useTTS(); - const { epubTheme } = useConfig(); + const { + registerLocationChangeHandler, + pause, + currentSentence, + currentSentenceAlignment, + currentWordIndex + } = useTTS(); + const { epubTheme, epubHighlightEnabled, epubWordHighlightEnabled } = useConfig(); const { updateTheme } = useEPUBTheme(epubTheme, renditionRef.current); const containerRef = useRef(null); const { isResizing, setIsResizing, dimensions } = useEPUBResize(containerRef); @@ -70,6 +78,38 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { } }, [currentSentence, highlightPattern, clearHighlights]); + // Word-level highlight layered on top of the block highlight + useEffect(() => { + if (!epubHighlightEnabled || !epubWordHighlightEnabled) { + clearWordHighlights(); + return; + } + + if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { + clearWordHighlights(); + return; + } + + if (!currentSentenceAlignment) { + clearWordHighlights(); + return; + } + + highlightWordIndex( + currentSentenceAlignment, + currentWordIndex, + currentSentence || '' + ); + }, [ + currentWordIndex, + currentSentence, + currentSentenceAlignment, + epubHighlightEnabled, + epubWordHighlightEnabled, + clearWordHighlights, + highlightWordIndex + ]); + if (!currDocData) { return ; } @@ -95,4 +135,4 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
); -} \ No newline at end of file +} diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index f3fb414..de88b1d 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -26,11 +26,13 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { const { containerWidth } = usePDFResize(containerRef); // Config context - const { viewType, pdfHighlightEnabled } = useConfig(); + const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig(); // TTS context const { currentSentence, + currentWordIndex, + currentSentenceAlignment, skipToLocation, } = useTTS(); @@ -38,6 +40,8 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { const { highlightPattern, clearHighlights, + clearWordHighlights, + highlightWordIndex, onDocumentLoadSuccess, currDocData, currDocPages, @@ -74,6 +78,46 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { }; }, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]); + // Word-level highlight layered on top of the block highlight + useEffect(() => { + if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) { + clearWordHighlights(); + return; + } + + if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { + clearWordHighlights(); + return; + } + + const wordEntry = + currentSentenceAlignment && + currentWordIndex < currentSentenceAlignment.words.length + ? currentSentenceAlignment.words[currentWordIndex] + : undefined; + const wordText = wordEntry?.text || null; + + if (!wordText) { + clearWordHighlights(); + return; + } + + highlightWordIndex( + currentSentenceAlignment, + currentWordIndex, + currentSentence || '', + containerRef as RefObject + ); + }, [ + currentWordIndex, + currentSentence, + currentSentenceAlignment, + pdfHighlightEnabled, + pdfWordHighlightEnabled, + clearWordHighlights, + highlightWordIndex + ]); + // Add page dimensions state const [pageWidth, setPageWidth] = useState(595); // default A4 width const [pageHeight, setPageHeight] = useState(842); // default A4 height diff --git a/src/contexts/ConfigContext.tsx b/src/contexts/ConfigContext.tsx index f560b93..5089775 100644 --- a/src/contexts/ConfigContext.tsx +++ b/src/contexts/ConfigContext.tsx @@ -32,7 +32,9 @@ interface ConfigContextType { isLoading: boolean; isDBReady: boolean; pdfHighlightEnabled: boolean; + pdfWordHighlightEnabled: boolean; epubHighlightEnabled: boolean; + epubWordHighlightEnabled: boolean; } const ConfigContext = createContext(undefined); @@ -103,7 +105,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) { savedVoices, smartSentenceSplitting, pdfHighlightEnabled, + pdfWordHighlightEnabled, epubHighlightEnabled, + epubWordHighlightEnabled, } = config || APP_CONFIG_DEFAULTS; /** @@ -201,7 +205,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) { isLoading, isDBReady, pdfHighlightEnabled, - epubHighlightEnabled + pdfWordHighlightEnabled, + epubHighlightEnabled, + epubWordHighlightEnabled }}> {children} diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index 46ab2bf..1922d2b 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -22,7 +22,13 @@ import { createRangeCfi } from '@/lib/epub'; import { useParams } from 'next/navigation'; import { useConfig } from './ConfigContext'; import { withRetry } from '@/utils/audio'; -import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts'; +import { CmpStr } from 'cmpstr'; +import type { + TTSRequestHeaders, + TTSRequestPayload, + TTSRetryOptions, + TTSSentenceAlignment +} from '@/types/tts'; interface EPUBContextType { currDocData: ArrayBuffer | undefined; @@ -44,12 +50,26 @@ interface EPUBContextType { isAudioCombining: boolean; highlightPattern: (text: string) => void; clearHighlights: () => void; + highlightWordIndex: ( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined + ) => void; + clearWordHighlights: () => void; } const EPUBContext = createContext(undefined); const EPUB_CONTINUATION_CHARS = 600; +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); + +const normalizeWordForMatch = (text: string): string => + text + .trim() + .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '') + .toLowerCase(); + const stepToNextNode = (node: Node | null, root: Node): Node | null => { if (!node) return null; if (node.firstChild) { @@ -160,6 +180,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { const shouldPauseRef = useRef(true); // Track current highlight CFI for removal const currentHighlightCfi = useRef(null); + const currentWordHighlightCfi = useRef(null); /** * Clears all current document state and stops any active TTS @@ -711,15 +732,22 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } }, [id, skipToLocation, extractPageText, setIsEPUB]); - const clearHighlights = useCallback(() => { - if (renditionRef.current) { - if (currentHighlightCfi.current) { - renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight'); - currentHighlightCfi.current = null; - } + const clearWordHighlights = useCallback(() => { + if (!renditionRef.current) return; + if (currentWordHighlightCfi.current) { + renditionRef.current.annotations.remove(currentWordHighlightCfi.current, 'highlight'); + currentWordHighlightCfi.current = null; } }, []); + const clearHighlights = useCallback(() => { + if (renditionRef.current && currentHighlightCfi.current) { + renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight'); + currentHighlightCfi.current = null; + } + clearWordHighlights(); + }, [clearWordHighlights]); + const highlightPattern = useCallback(async (text: string) => { if (!renditionRef.current) return; @@ -772,6 +800,262 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } }, [epubHighlightEnabled, clearHighlights]); + const highlightWordIndex = useCallback(( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined + ) => { + clearWordHighlights(); + + if (!epubHighlightEnabled) return; + if (!alignment) return; + if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return; + + const words = alignment.words || []; + if (!words.length || wordIndex >= words.length) return; + + if (!renditionRef.current) return; + if (!currentHighlightCfi.current) return; + + const cleanSentence = + sentence && sentence.trim() + ? sentence.trim().replace(/\s+/g, ' ') + : null; + if (!cleanSentence) return; + + const alignmentSentenceClean = alignment.sentence + ? alignment.sentence.trim().replace(/\s+/g, ' ') + : null; + if (!alignmentSentenceClean || alignmentSentenceClean !== cleanSentence) { + return; + } + + const contents = renditionRef.current.getContents(); + const contentsArray = Array.isArray(contents) ? contents : [contents]; + + for (const content of contentsArray) { + let range: Range | null = null; + try { + range = content.range(currentHighlightCfi.current as string); + } catch { + range = null; + } + if (!range) continue; + + const root = range.commonAncestorContainer; + if (!root) continue; + + const domTokens: Array<{ + node: Text; + startOffset: number; + endOffset: number; + norm: string; + }> = []; + + const addTokensFromNode = (textNode: Text, start: number, end: number) => { + const full = textNode.textContent || ''; + const safeStart = Math.max(0, Math.min(start, full.length)); + const safeEnd = Math.max(safeStart, Math.min(end, full.length)); + if (safeEnd <= safeStart) return; + + const slice = full.slice(safeStart, safeEnd); + const wordRegex = /\S+/g; + let match: RegExpExecArray | null; + while ((match = wordRegex.exec(slice)) !== null) { + const raw = match[0]; + const norm = normalizeWordForMatch(raw); + if (!norm) continue; + const tokenStart = safeStart + match.index; + const tokenEnd = tokenStart + raw.length; + domTokens.push({ + node: textNode, + startOffset: tokenStart, + endOffset: tokenEnd, + norm, + }); + } + }; + + const nextTextNode = (node: Node | null): Text | null => { + let next = getNextTextNode(node, root); + while (next) { + if (next.nodeType === Node.TEXT_NODE) { + return next as Text; + } + next = getNextTextNode(next, root); + } + return null; + }; + + // Collect tokens within the sentence range + if (range.startContainer === range.endContainer && range.startContainer.nodeType === Node.TEXT_NODE) { + addTokensFromNode(range.startContainer as Text, range.startOffset, range.endOffset); + } else { + let current: Text | null = null; + + if (range.startContainer.nodeType === Node.TEXT_NODE) { + const startText = range.startContainer as Text; + const isEnd = range.endContainer === startText && range.endContainer.nodeType === Node.TEXT_NODE; + const endOffset = isEnd ? range.endOffset : (startText.textContent || '').length; + addTokensFromNode(startText, range.startOffset, endOffset); + if (isEnd) { + current = null; + } else { + current = nextTextNode(startText); + } + } else { + current = nextTextNode(range.startContainer); + } + + while (current) { + if (range.endContainer.nodeType === Node.TEXT_NODE && current === range.endContainer) { + addTokensFromNode(current, 0, range.endOffset); + break; + } else { + addTokensFromNode(current, 0, (current.textContent || '').length); + } + current = nextTextNode(current); + } + } + + if (!domTokens.length) { + return; + } + + const domFiltered: Array<{ tokenIndex: number; norm: string }> = []; + for (let i = 0; i < domTokens.length; i++) { + const norm = domTokens[i].norm; + if (!norm) continue; + domFiltered.push({ tokenIndex: i, norm }); + } + + const ttsFiltered: Array<{ wordIndex: number; norm: string }> = []; + for (let i = 0; i < words.length; i++) { + const norm = normalizeWordForMatch(words[i].text); + if (!norm) continue; + ttsFiltered.push({ wordIndex: i, norm }); + } + + const wordToToken = new Array(words.length).fill(-1); + const m = domFiltered.length; + const n = ttsFiltered.length; + + if (m && n) { + const dp: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(Number.POSITIVE_INFINITY) + ); + const bt: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(0) + ); // 0=diag,1=up,2=left + + dp[0][0] = 0; + const GAP_COST = 0.7; + + for (let i = 0; i <= m; i++) { + for (let j = 0; j <= n; j++) { + if (i > 0 && j > 0) { + const a = domFiltered[i - 1].norm; + const b = ttsFiltered[j - 1].norm; + const sim = cmp.compare(a, b); + const subCost = 1 - sim; + const cand = dp[i - 1][j - 1] + subCost; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 0; + } + } + if (i > 0) { + const cand = dp[i - 1][j] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 1; + } + } + if (j > 0) { + const cand = dp[i][j - 1] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 2; + } + } + } + } + + let i = m; + let j = n; + while (i > 0 || j > 0) { + const move = bt[i][j]; + if (i > 0 && j > 0 && move === 0) { + const domIdx = domFiltered[i - 1].tokenIndex; + const ttsIdx = ttsFiltered[j - 1].wordIndex; + if (wordToToken[ttsIdx] === -1) { + wordToToken[ttsIdx] = domIdx; + } + i -= 1; + j -= 1; + } else if (i > 0 && (move === 1 || j === 0)) { + i -= 1; + } else if (j > 0 && (move === 2 || i === 0)) { + j -= 1; + } else { + break; + } + } + + // Propagate nearest known mapping to fill gaps + let lastSeen = -1; + for (let k = 0; k < wordToToken.length; k++) { + if (wordToToken[k] !== -1) { + lastSeen = wordToToken[k]; + } else if (lastSeen !== -1) { + wordToToken[k] = lastSeen; + } + } + let nextSeen = -1; + for (let k = wordToToken.length - 1; k >= 0; k--) { + if (wordToToken[k] !== -1) { + nextSeen = wordToToken[k]; + } else if (nextSeen !== -1) { + wordToToken[k] = nextSeen; + } + } + } + + const mappedIndex = + wordIndex < wordToToken.length ? wordToToken[wordIndex] : -1; + if (mappedIndex === -1) { + return; + } + + const token = domTokens[mappedIndex]; + const doc = token.node.ownerDocument || (range.commonAncestorContainer as Document); + const wordRange = doc.createRange(); + wordRange.setStart(token.node, token.startOffset); + wordRange.setEnd(token.node, token.endOffset); + + try { + const wordCfi = content.cfiFromRange(wordRange); + currentWordHighlightCfi.current = wordCfi; + renditionRef.current.annotations.add( + 'highlight', + wordCfi, + {}, + () => {}, + '', + { + fill: 'var(--accent)', + 'fill-opacity': '0.4', + 'mix-blend-mode': 'multiply', + } + ); + } catch (error) { + console.error('Error highlighting EPUB word:', error); + } + + break; + } + }, [epubHighlightEnabled, clearWordHighlights]); + // Context value memoization @@ -796,6 +1080,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) { isAudioCombining, highlightPattern, clearHighlights, + highlightWordIndex, + clearWordHighlights, }), [ setCurrentDocument, @@ -813,6 +1099,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) { isAudioCombining, highlightPattern, clearHighlights, + highlightWordIndex, + clearWordHighlights, ] ); diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index d555e9b..dfdb6b1 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -36,10 +36,16 @@ import { extractTextFromPDF, highlightPattern, clearHighlights, - handleTextClick, + clearWordHighlights, + highlightWordIndex, } from '@/lib/pdf'; -import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts'; +import type { + TTSRequestHeaders, + TTSRequestPayload, + TTSRetryOptions, + TTSSentenceAlignment +} from '@/types/tts'; /** * Interface defining all available methods and properties in the PDF context @@ -59,13 +65,12 @@ interface PDFContextType { onDocumentLoadSuccess: (pdf: PDFDocumentProxy) => void; highlightPattern: (text: string, pattern: string, containerRef: RefObject) => void; clearHighlights: () => void; - handleTextClick: ( - event: MouseEvent, - pdfText: string, - containerRef: RefObject, - stopAndPlayFromIndex: (index: number) => void, - isProcessing: boolean, - enableHighlight?: boolean + clearWordHighlights: () => void; + highlightWordIndex: ( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined, + containerRef: RefObject ) => void; createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise; regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>; @@ -655,7 +660,8 @@ export function PDFProvider({ children }: { children: ReactNode }) { clearCurrDoc, highlightPattern, clearHighlights, - handleTextClick, + clearWordHighlights, + highlightWordIndex, pdfDocument, createFullAudioBook, regenerateChapter, diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 9c52ed3..4124f13 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -47,6 +47,7 @@ import type { TTSRequestPayload, TTSRequestHeaders, TTSRetryOptions, + TTSSentenceAlignment, } from '@/types/tts'; // Media globals @@ -63,6 +64,10 @@ interface TTSContextType extends TTSPlaybackState { // Voice settings availableVoices: string[]; + // Alignment metadata for the current sentence + currentSentenceAlignment?: TTSSentenceAlignment; + currentWordIndex?: number | null; + // Control functions togglePlay: () => void; skipForward: () => void; @@ -264,6 +269,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement updateConfigKey, skipBlank, smartSentenceSplitting, + pdfHighlightEnabled, + pdfWordHighlightEnabled, + epubHighlightEnabled, + epubWordHighlightEnabled, } = useConfig(); // Audio and voice management hooks @@ -331,6 +340,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const epubContinuationRef = useRef(null); const pageTurnEstimateRef = useRef(null); const pageTurnTimeoutRef = useRef | null>(null); + const sentenceAlignmentCacheRef = useRef>(new Map()); + const [currentSentenceAlignment, setCurrentSentenceAlignment] = useState(); + const [currentWordIndex, setCurrentWordIndex] = useState(null); + const sentencesRef = useRef([]); + const currentIndexRef = useRef(0); + + useEffect(() => { + sentencesRef.current = sentences; + }, [sentences]); + + useEffect(() => { + currentIndexRef.current = currentIndex; + }, [currentIndex]); /** * Processes text into sentences using the shared NLP utility @@ -370,6 +392,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement clearTimeout(pageTurnTimeoutRef.current); pageTurnTimeoutRef.current = null; } + setCurrentWordIndex(null); }, [activeHowl]); /** @@ -558,6 +581,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setCurrentIndex(0); } + // Reset alignment state whenever the text block changes + sentenceAlignmentCacheRef.current.clear(); + setCurrentSentenceAlignment(undefined); + setCurrentWordIndex(null); + // Compute auto page-turn estimate for PDFs when we have a continuation if (smartSentenceSplitting && !isEPUB && continuationCarried && normalizedOptions.nextLocation !== undefined) { const continuationNormalized = preprocessSentenceForAudio(continuationCarried); @@ -706,10 +734,61 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @returns {Promise} The generated audio buffer */ const getAudio = useCallback(async (sentence: string): Promise => { + const alignmentEnabledForCurrentDoc = + (!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) || + (isEPUB && epubHighlightEnabled && epubWordHighlightEnabled); + // Helper to ensure we have an alignment for a given + // sentence/audio pair, even when the audio itself is + // served from the local cache. + const ensureAlignment = (arrayBuffer: ArrayBuffer) => { + if (!alignmentEnabledForCurrentDoc) return; + if (sentenceAlignmentCacheRef.current.has(sentence)) return; + + try { + const audioBytes = Array.from(new Uint8Array(arrayBuffer)); + const alignmentBody = { + text: sentence, + audio: audioBytes, + }; + + void fetch('/api/whisper', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(alignmentBody), + }) + .then(async (res) => { + if (!res.ok) return; + const data = await res.json().catch(() => null); + if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) { + return; + } + const alignment = data.alignments[0] as TTSSentenceAlignment; + sentenceAlignmentCacheRef.current.set(sentence, alignment); + + const currentSentence = sentencesRef.current[currentIndexRef.current]; + if (currentSentence === sentence) { + setCurrentSentenceAlignment(alignment); + setCurrentWordIndex(null); + } + }) + .catch((err) => { + console.warn('Alignment request failed:', err); + }); + } catch (err) { + console.warn('Failed to start alignment request:', err); + } + }; + // Check if the audio is already cached const cachedAudio = audioCache.get(sentence); if (cachedAudio) { console.log('Using cached audio for sentence:', sentence.substring(0, 20)); + // If we have audio but no alignment (e.g. after a + // navigation or TTS reset), kick off a fresh alignment + // request using the cached audio buffer. + ensureAlignment(cachedAudio); return cachedAudio; } @@ -769,6 +848,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Cache the array buffer audioCache.set(sentence, arrayBuffer); + // Fire-and-forget alignment request; do not block audio playback + ensureAlignment(arrayBuffer); + return arrayBuffer; } catch (error) { // Check if this was an abort error @@ -788,7 +870,21 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement }); throw error; } - }, [voice, speed, ttsModel, ttsInstructions, audioCache, openApiKey, openApiBaseUrl, configTTSProvider]); + }, [ + voice, + speed, + ttsModel, + ttsInstructions, + audioCache, + openApiKey, + openApiBaseUrl, + configTTSProvider, + isEPUB, + pdfHighlightEnabled, + pdfWordHighlightEnabled, + epubHighlightEnabled, + epubWordHighlightEnabled + ]); /** * Processes and plays the current sentence @@ -1004,7 +1100,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement }, [isPlaying, advance, activeHowl, processSentence, audioSpeed]); const playAudio = useCallback(async () => { - const howl = await playSentenceWithHowl(sentences[currentIndex], currentIndex); + const sentence = sentences[currentIndex]; + const cachedAlignment = sentenceAlignmentCacheRef.current.get(sentence); + if (cachedAlignment) { + setCurrentSentenceAlignment(cachedAlignment); + setCurrentWordIndex(null); + } else { + setCurrentSentenceAlignment(undefined); + setCurrentWordIndex(null); + } + + const howl = await playSentenceWithHowl(sentence, currentIndex); if (howl) { howl.play(); } @@ -1017,6 +1123,47 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement playAudio, }); + // Track the current word index during playback using Howler's seek position + useEffect(() => { + if (!activeHowl || !isPlaying || !currentSentenceAlignment || !currentSentenceAlignment.words.length) { + setCurrentWordIndex(null); + return; + } + + let frameId: number; + + const tick = () => { + try { + const pos = activeHowl.seek() as number; + if (typeof pos === 'number' && Number.isFinite(pos)) { + const words = currentSentenceAlignment.words; + let idx = -1; + for (let i = 0; i < words.length; i++) { + const w = words[i]; + if (pos >= w.startSec && pos < w.endSec) { + idx = i; + break; + } + } + if (idx !== -1) { + setCurrentWordIndex((prev) => (prev === idx ? prev : idx)); + } + } + } catch { + // ignore seek errors + } + frameId = requestAnimationFrame(tick); + }; + + frameId = requestAnimationFrame(tick); + + return () => { + if (frameId) { + cancelAnimationFrame(frameId); + } + }; + }, [activeHowl, isPlaying, currentSentenceAlignment]); + /** * Preloads the next sentence's audio */ @@ -1085,6 +1232,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setCurrDocPages(undefined); setIsProcessing(false); setIsEPUB(false); + sentenceAlignmentCacheRef.current.clear(); + setCurrentSentenceAlignment(undefined); + setCurrentWordIndex(null); }, [abortAudio]); /** @@ -1205,6 +1355,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement isProcessing, isBackgrounded, currentSentence: sentences[currentIndex] || '', + currentSentenceAlignment, + currentWordIndex, currDocPage, currDocPageNumber, currDocPages, @@ -1248,7 +1400,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement skipToLocation, registerLocationChangeHandler, registerVisualPageChangeHandler, - setIsEPUB + setIsEPUB, + currentSentenceAlignment, + currentWordIndex ]); // Use media session hook diff --git a/src/lib/dexie.ts b/src/lib/dexie.ts index ee62a29..3fe1d72 100644 --- a/src/lib/dexie.ts +++ b/src/lib/dexie.ts @@ -136,6 +136,12 @@ function buildAppConfigFromRaw(raw: RawConfigMap): AppConfigRow { savedVoices, pdfHighlightEnabled: raw.pdfHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfHighlightEnabled, + pdfWordHighlightEnabled: + raw.pdfWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfWordHighlightEnabled, + epubHighlightEnabled: + raw.epubHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubHighlightEnabled, + epubWordHighlightEnabled: + raw.epubWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubWordHighlightEnabled, firstVisit: raw.firstVisit === 'true', documentListState, }; diff --git a/src/lib/pdf.ts b/src/lib/pdf.ts index cfcb945..a8e938b 100644 --- a/src/lib/pdf.ts +++ b/src/lib/pdf.ts @@ -3,9 +3,10 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist'; import "core-js/proposals/promise-with-resolvers"; import { processTextToSentences } from '@/lib/nlp'; +import type { TTSSentenceAlignment } from '@/types/tts'; import { CmpStr } from 'cmpstr'; -const cmp = CmpStr.create().setMetric( 'dice' ).setFlags( 'itw' ); +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); // Worker coordination for offloading highlight token matching interface HighlightTokenMatchRequest { @@ -141,12 +142,25 @@ try { console.error('Error patching TextLayer.render:', e); } -interface TextMatch { - elements: HTMLElement[]; - rating: number; +type PDFToken = { + spanIndex: number; + textNode: Text; text: string; - lengthDiff: number; -} + startOffset: number; + endOffset: number; +}; + +let lastSpanNodes: HTMLElement[] = []; +let lastTokens: PDFToken[] = []; +let lastSentenceTokenWindow: { start: number; end: number } | null = null; +let lastSentencePattern: string | null = null; +let lastSentenceWordToTokenMap: number[] | null = null; + +const normalizeWordForMatch = (text: string): string => + text + .trim() + .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '') + .toLowerCase(); // Text Processing functions export async function extractTextFromPDF( @@ -279,50 +293,23 @@ export function clearHighlights() { element.parentElement.removeChild(element); } }); + const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay'); + wordOverlays.forEach((node) => { + const element = node as HTMLElement; + if (element.parentElement) { + element.parentElement.removeChild(element); + } + }); } -export function findBestTextMatch( - elements: Array<{ element: HTMLElement; text: string }>, - targetText: string, - maxCombinedLength: number -): TextMatch { - let bestMatch = { - elements: [] as HTMLElement[], - rating: 0, - text: '', - lengthDiff: Infinity, - }; - - const SPAN_SEARCH_LIMIT = 10; - - for (let i = 0; i < elements.length; i++) { - let combinedText = ''; - const currentElements = []; - for (let j = i; j < Math.min(i + SPAN_SEARCH_LIMIT, elements.length); j++) { - const node = elements[j]; - const newText = combinedText ? `${combinedText} ${node.text}` : node.text; - if (newText.length > maxCombinedLength) break; - - combinedText = newText; - currentElements.push(node.element); - - const similarity = cmp.compare(combinedText, targetText); - const lengthDiff = Math.abs(combinedText.length - targetText.length); - const lengthPenalty = lengthDiff / targetText.length; - const adjustedRating = similarity * (1 - lengthPenalty * 0.5); - - if (adjustedRating > bestMatch.rating) { - bestMatch = { - elements: [...currentElements], - rating: adjustedRating, - text: combinedText, - lengthDiff, - }; - } +export function clearWordHighlights() { + const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay'); + wordOverlays.forEach((node) => { + const element = node as HTMLElement; + if (element.parentElement) { + element.parentElement.removeChild(element); } - } - - return bestMatch; + }); } export function highlightPattern( @@ -338,22 +325,18 @@ export function highlightPattern( const cleanPattern = pattern.trim().replace(/\s+/g, ' '); if (!cleanPattern) return; + lastSentencePattern = cleanPattern; + lastSentenceWordToTokenMap = null; + lastSentenceTokenWindow = null; const spanNodes = Array.from( container.querySelectorAll('.react-pdf__Page__textContent span') ) as HTMLElement[]; if (!spanNodes.length) return; + lastSpanNodes = spanNodes; - type Token = { - spanIndex: number; - textNode: Text; - text: string; - startOffset: number; - endOffset: number; - }; - - const tokens: Token[] = []; + const tokens: PDFToken[] = []; spanNodes.forEach((span, spanIndex) => { const node = span.firstChild; @@ -377,6 +360,7 @@ export function highlightPattern( }); if (!tokens.length) return; + lastTokens = tokens; const patternLen = cleanPattern.length; @@ -415,6 +399,11 @@ export function highlightPattern( bestLengthDiff < patternLen * 0.3 ? 0.3 : 0.5; if (hasTokenMatch && bestRating >= similarityThreshold) { + lastSentenceTokenWindow = { + start: bestStart, + end: bestEnd, + }; + const rangesBySpan = new Map< number, { startOffset: number; endOffset: number } @@ -454,65 +443,6 @@ export function highlightPattern( }); } - // Fallback: if token-level matching failed, use span-based fuzzy matching - if (!highlightRanges.length) { - const spanEntries = spanNodes - .map((node) => ({ - element: node as HTMLElement, - text: (node.textContent || '').trim(), - })) - .filter((entry) => entry.text.length > 0); - - const containerRect = container.getBoundingClientRect(); - const visibleTop = container.scrollTop; - const visibleBottom = visibleTop + containerRect.height; - const bufferSize = containerRect.height; - - const visibleNodes = spanEntries.filter(({ element }) => { - const rect = element.getBoundingClientRect(); - const elementTop = - rect.top - containerRect.top + container.scrollTop; - return ( - elementTop >= visibleTop - bufferSize && - elementTop <= visibleBottom + bufferSize - ); - }); - - let bestMatch = findBestTextMatch( - visibleNodes, - cleanPattern, - cleanPattern.length * 2 - ); - - if (bestMatch.rating < 0.3) { - bestMatch = findBestTextMatch( - spanEntries, - cleanPattern, - cleanPattern.length * 2 - ); - } - - const spanSimilarityThreshold = - bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5; - - if (bestMatch.rating >= spanSimilarityThreshold) { - bestMatch.elements.forEach((element) => { - const node = element.firstChild; - if (!node || node.nodeType !== Node.TEXT_NODE) return; - const textNode = node as Text; - const content = textNode.textContent || ''; - if (!content) return; - - highlightRanges.push({ - textNode, - startOffset: 0, - endOffset: content.length, - span: element, - }); - }); - } - } - if (!highlightRanges.length) return; // Create overlay rectangles for each range, relative to its page text layer @@ -577,7 +507,7 @@ export function highlightPattern( runHighlightTokenMatch(cleanPattern, tokenTexts) .then((result) => { if (!result || result.bestStart === -1) { - // No worker result or no good match; rely on span-level fallback + // No worker result or no good match; nothing to highlight applyHighlightFromTokens(null); } else { applyHighlightFromTokens({ @@ -590,95 +520,199 @@ export function highlightPattern( }) .catch((error) => { console.error( - 'Error in PDF highlight worker, falling back to span-based matching:', + 'Error in PDF highlight worker; no highlights applied:', error ); applyHighlightFromTokens(null); }); } -// Text Click Handler -export function handleTextClick( - event: MouseEvent, - pdfText: string, - containerRef: React.RefObject, - stopAndPlayFromIndex: (index: number) => void, - isProcessing: boolean, - enableHighlight = true +export function highlightWordIndex( + alignment: TTSSentenceAlignment | undefined, + wordIndex: number | null | undefined, + sentence: string | null | undefined, + containerRef: React.RefObject ) { - if (isProcessing) return; + clearWordHighlights(); - const target = event.target as HTMLElement; - if (!target.matches('.react-pdf__Page__textContent span')) return; - - const parentElement = target.closest('.react-pdf__Page__textContent'); - if (!parentElement) return; - - const spans = Array.from(parentElement.querySelectorAll('span')); - const clickedIndex = spans.indexOf(target); - const contextWindow = 3; - const startIndex = Math.max(0, clickedIndex - contextWindow); - const endIndex = Math.min(spans.length - 1, clickedIndex + contextWindow); - const contextText = spans - .slice(startIndex, endIndex + 1) - .map((span) => span.textContent) - .join(' ') - .trim(); - - if (!contextText?.trim()) return; - - const cleanContext = contextText.trim().replace(/\s+/g, ' '); - - // Fast path when highlight overlays are disabled: - // avoid expensive span-level fuzzy matching and just map - // the clicked context to a sentence using cheap string checks. - if (!enableHighlight) { - const sentences = processTextToSentences(pdfText); - const idx = sentences.findIndex((sentence) => { - const cleanSentence = sentence.trim().replace(/\s+/g, ' '); - return ( - cleanSentence.includes(cleanContext) || - cleanContext.includes(cleanSentence) - ); - }); - - if (idx !== -1) { - stopAndPlayFromIndex(idx); - } + if (!alignment) return; + if (wordIndex === null || wordIndex === undefined || wordIndex < 0) { return; } - const allText = Array.from(parentElement.querySelectorAll('span')).map((node) => ({ - element: node as HTMLElement, - text: (node.textContent || '').trim(), - })).filter((node) => node.text.length > 0); + const words = alignment.words || []; + if (!words.length || wordIndex >= words.length) return; - const bestMatch = findBestTextMatch(allText, cleanContext, cleanContext.length * 2); - const similarityThreshold = bestMatch.lengthDiff < cleanContext.length * 0.3 ? 0.3 : 0.5; + const container = containerRef.current; + if (!container) return; + if (!lastSentenceTokenWindow) return; + if (!lastTokens.length || !lastSpanNodes.length) return; - if (bestMatch.rating >= similarityThreshold) { - const matchText = bestMatch.text; - // Use the same sentence processing logic as TTSContext for consistency - const sentences = processTextToSentences(pdfText); - console.log("sentences inside handleTextClick: %d", sentences.length) - let bestSentenceMatch = { sentence: '', rating: 0 }; + const cleanSentence = + sentence && sentence.trim() + ? sentence.trim().replace(/\s+/g, ' ') + : null; + if (!cleanSentence || !lastSentencePattern) return; + if (cleanSentence !== lastSentencePattern) return; - for (const sentence of sentences) { - const rating = cmp.compare(matchText, sentence); - if (rating > bestSentenceMatch.rating) { - bestSentenceMatch = { sentence, rating }; - } + const start = lastSentenceTokenWindow.start; + const end = lastSentenceTokenWindow.end; + if (end < start) return; + + // Lazily build or refresh the mapping from alignment word + // indices to PDF token indices for this sentence window. + if ( + !lastSentenceWordToTokenMap || + lastSentenceWordToTokenMap.length !== words.length + ) { + const pdfFiltered: { tokenIndex: number; norm: string }[] = []; + for (let i = start; i <= end; i++) { + const norm = normalizeWordForMatch(lastTokens[i].text); + if (!norm) continue; + pdfFiltered.push({ tokenIndex: i, norm }); } - if (bestSentenceMatch.rating >= 0.5) { - const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence); - if (sentenceIndex !== -1) { - stopAndPlayFromIndex(sentenceIndex); - if (enableHighlight) { - highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef); + const ttsFiltered: { wordIndex: number; norm: string }[] = []; + for (let i = 0; i < words.length; i++) { + const norm = normalizeWordForMatch(words[i].text); + if (!norm) continue; + ttsFiltered.push({ wordIndex: i, norm }); + } + + const wordToToken = new Array(words.length).fill(-1); + + const m = pdfFiltered.length; + const n = ttsFiltered.length; + + if (m && n) { + const dp: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(Number.POSITIVE_INFINITY) + ); + const bt: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(0) + ); // 0=diag,1=up,2=left + + dp[0][0] = 0; + const GAP_COST = 0.7; + + for (let i = 0; i <= m; i++) { + for (let j = 0; j <= n; j++) { + if (i > 0 && j > 0) { + const a = pdfFiltered[i - 1].norm; + const b = ttsFiltered[j - 1].norm; + const sim = cmp.compare(a, b); + const subCost = 1 - sim; + const cand = dp[i - 1][j - 1] + subCost; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 0; + } + } + if (i > 0) { + const cand = dp[i - 1][j] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 1; + } + } + if (j > 0) { + const cand = dp[i][j - 1] + GAP_COST; + if (cand < dp[i][j]) { + dp[i][j] = cand; + bt[i][j] = 2; + } + } + } + } + + let i = m; + let j = n; + while (i > 0 || j > 0) { + const move = bt[i][j]; + if (i > 0 && j > 0 && move === 0) { + const pdfIdx = pdfFiltered[i - 1].tokenIndex; + const ttsIdx = ttsFiltered[j - 1].wordIndex; + if (wordToToken[ttsIdx] === -1) { + wordToToken[ttsIdx] = pdfIdx; + } + i -= 1; + j -= 1; + } else if (i > 0 && (move === 1 || j === 0)) { + i -= 1; + } else if (j > 0 && (move === 2 || i === 0)) { + j -= 1; + } else { + break; + } + } + + // Propagate nearest known mapping to fill gaps + let lastSeen = -1; + for (let k = 0; k < wordToToken.length; k++) { + if (wordToToken[k] !== -1) { + lastSeen = wordToToken[k]; + } else if (lastSeen !== -1) { + wordToToken[k] = lastSeen; + } + } + let nextSeen = -1; + for (let k = wordToToken.length - 1; k >= 0; k--) { + if (wordToToken[k] !== -1) { + nextSeen = wordToToken[k]; + } else if (nextSeen !== -1) { + wordToToken[k] = nextSeen; } } } + + lastSentenceWordToTokenMap = wordToToken; + } + + const mappedIndex = + lastSentenceWordToTokenMap && wordIndex < lastSentenceWordToTokenMap.length + ? lastSentenceWordToTokenMap[wordIndex] + : -1; + + if (mappedIndex === -1) return; + + const chosenTokenIndex = mappedIndex; + + const token = lastTokens[chosenTokenIndex]; + const span = lastSpanNodes[token.spanIndex]; + if (!span) return; + + const node = token.textNode; + if (!node || node.nodeType !== Node.TEXT_NODE) return; + + try { + const range = document.createRange(); + range.setStart(node, token.startOffset); + range.setEnd(node, token.endOffset); + + const pageLayer = span.closest( + '.react-pdf__Page__textContent' + ) as HTMLElement | null; + if (!pageLayer) return; + + const pageRect = pageLayer.getBoundingClientRect(); + const rects = Array.from(range.getClientRects()); + + rects.forEach((rect) => { + const highlight = document.createElement('div'); + highlight.className = 'pdf-word-highlight-overlay'; + highlight.style.position = 'absolute'; + highlight.style.backgroundColor = 'var(--accent)'; + highlight.style.opacity = '0.4'; + highlight.style.pointerEvents = 'none'; + highlight.style.left = `${rect.left - pageRect.left}px`; + highlight.style.top = `${rect.top - pageRect.top}px`; + highlight.style.width = `${rect.width}px`; + highlight.style.height = `${rect.height}px`; + highlight.style.zIndex = '2'; + pageLayer.appendChild(highlight); + }); + } catch { + // Ignore range errors } } diff --git a/src/types/config.ts b/src/types/config.ts index 30401c3..950aaa3 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -23,7 +23,9 @@ export interface AppConfigValues { savedVoices: SavedVoices; smartSentenceSplitting: boolean; pdfHighlightEnabled: boolean; + pdfWordHighlightEnabled: boolean; epubHighlightEnabled: boolean; + epubWordHighlightEnabled: boolean; firstVisit: boolean; documentListState: DocumentListState; } @@ -47,7 +49,9 @@ export const APP_CONFIG_DEFAULTS: AppConfigValues = { savedVoices: {}, smartSentenceSplitting: true, pdfHighlightEnabled: true, + pdfWordHighlightEnabled: true, epubHighlightEnabled: true, + epubWordHighlightEnabled: true, firstVisit: false, documentListState: { sortBy: 'name', diff --git a/src/types/tts.ts b/src/types/tts.ts index 6cbec2c..a2744b2 100644 --- a/src/types/tts.ts +++ b/src/types/tts.ts @@ -63,6 +63,22 @@ export interface TTSPageTurnEstimate { fraction: number; } +// Word-level alignment within a single spoken sentence/block +export interface TTSSentenceWord { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; +} + +// Alignment metadata for a single TTS sentence/block +export interface TTSSentenceAlignment { + sentence: string; + sentenceIndex: number; + words: TTSSentenceWord[]; +} + // Metadata for an audiobook chapter export interface TTSAudiobookChapter { index: number; @@ -71,4 +87,4 @@ export interface TTSAudiobookChapter { status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b'; -} \ No newline at end of file +}