From c28c074e434b0caf853048e3bd7fa7a785f22273 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 21 Jan 2026 12:09:59 -0700 Subject: [PATCH 1/6] refactor(nlp): improve text splitting and normalization for TTS blocks - Rename and enhance text processing functions in nlp.ts for better handling of oversized texts, sentence boundaries, and PDF artifacts - Update PDFViewer to add layout-aware highlighting with retry logic for sentence and word highlights - Adjust PDFContext and TTSContext to use new normalized text functions - Expand unit tests for new splitting behaviors, including long texts and punctuation preferences --- src/components/PDFViewer.tsx | 116 ++++++++++++++++++++++--- src/contexts/PDFContext.tsx | 8 +- src/contexts/TTSContext.tsx | 10 +-- src/lib/nlp.ts | 163 +++++++++++++++++++++++++++++------ tests/unit/nlp.spec.ts | 75 ++++++++++++++-- 5 files changed, 316 insertions(+), 56 deletions(-) diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index de88b1d..94ac3f6 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -24,6 +24,12 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { const containerRef = useRef(null); const scaleRef = useRef(1); const { containerWidth } = usePDFResize(containerRef); + const sentenceHighlightSeqRef = useRef(0); + const wordHighlightSeqRef = useRef(0); + const sentenceHighlightTimeoutsRef = useRef[]>([]); + const wordHighlightTimeoutsRef = useRef[]>([]); + const lastSentenceLayoutKeyRef = useRef(''); + const lastWordLayoutKeyRef = useRef(''); // Config context const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig(); @@ -49,6 +55,28 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { currDocPage, } = usePDF(); + const layoutKey = `${zoomLevel}:${containerWidth}:${viewType}:${currDocPage}`; + + const clearSentenceHighlightTimeouts = useCallback(() => { + for (const t of sentenceHighlightTimeoutsRef.current) clearTimeout(t); + sentenceHighlightTimeoutsRef.current = []; + }, []); + + const clearWordHighlightTimeouts = useCallback(() => { + for (const t of wordHighlightTimeoutsRef.current) clearTimeout(t); + wordHighlightTimeoutsRef.current = []; + }, []); + + const scheduleSentenceTimeout = useCallback((fn: () => void, ms: number) => { + const t = setTimeout(fn, ms); + sentenceHighlightTimeoutsRef.current.push(t); + }, []); + + const scheduleWordTimeout = useCallback((fn: () => void, ms: number) => { + const t = setTimeout(fn, ms); + wordHighlightTimeoutsRef.current.push(t); + }, []); + useEffect(() => { /* * Handles highlighting the current sentence being read by TTS. @@ -66,17 +94,47 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { return; } - const highlightTimeout = setTimeout(() => { - if (containerRef.current) { - highlightPattern(currDocText, currentSentence || '', containerRef as RefObject); + clearSentenceHighlightTimeouts(); + + const seq = ++sentenceHighlightSeqRef.current; + const isLayoutChange = layoutKey !== lastSentenceLayoutKeyRef.current; + lastSentenceLayoutKeyRef.current = layoutKey; + + if (isLayoutChange) { + clearHighlights(); + } + + const tryApply = (attempt: number) => { + if (seq !== sentenceHighlightSeqRef.current) return; + const container = containerRef.current; + if (!container) return; + if (!currentSentence) return; + + const spans = container.querySelectorAll('.react-pdf__Page__textContent span'); + if (!spans.length) { + if (attempt < 10) scheduleSentenceTimeout(() => tryApply(attempt + 1), 75); + return; } - }, 200); + + highlightPattern(currDocText, currentSentence, containerRef as RefObject); + }; + + scheduleSentenceTimeout(() => tryApply(0), 200); return () => { - clearTimeout(highlightTimeout); + clearSentenceHighlightTimeouts(); clearHighlights(); }; - }, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]); + }, [ + currDocText, + currentSentence, + highlightPattern, + clearHighlights, + pdfHighlightEnabled, + layoutKey, + clearSentenceHighlightTimeouts, + scheduleSentenceTimeout + ]); // Word-level highlight layered on top of the block highlight useEffect(() => { @@ -102,12 +160,41 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { return; } - highlightWordIndex( - currentSentenceAlignment, - currentWordIndex, - currentSentence || '', - containerRef as RefObject - ); + clearWordHighlightTimeouts(); + + const seq = ++wordHighlightSeqRef.current; + const isLayoutChange = layoutKey !== lastWordLayoutKeyRef.current; + lastWordLayoutKeyRef.current = layoutKey; + + const tryApplyWord = (attempt: number) => { + if (seq !== wordHighlightSeqRef.current) return; + const container = containerRef.current; + if (!container) return; + + highlightWordIndex( + currentSentenceAlignment, + currentWordIndex, + currentSentence || '', + containerRef as RefObject + ); + + if (isLayoutChange) { + // If we don't see a word overlay yet, the sentence highlight worker may not + // have produced `lastSentenceTokenWindow` (or the text layer isn't ready). + const overlayCount = container.querySelectorAll('.pdf-word-highlight-overlay').length; + if (overlayCount === 0 && attempt < 12) { + scheduleWordTimeout(() => tryApplyWord(attempt + 1), 75); + } + } + }; + + if (isLayoutChange) { + clearWordHighlights(); + scheduleWordTimeout(() => tryApplyWord(0), 250); + return; + } + + tryApplyWord(0); }, [ currentWordIndex, currentSentence, @@ -115,7 +202,10 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { pdfHighlightEnabled, pdfWordHighlightEnabled, clearWordHighlights, - highlightWordIndex + highlightWordIndex, + layoutKey, + clearWordHighlightTimeouts, + scheduleWordTimeout ]); // Add page dimensions state diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index a771452..ede6879 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -30,7 +30,7 @@ import type { PDFDocumentProxy } from 'pdfjs-dist'; import { getPdfDocument } from '@/lib/dexie'; import { useTTS } from '@/contexts/TTSContext'; import { useConfig } from '@/contexts/ConfigContext'; -import { processTextToSentences } from '@/lib/nlp'; +import { normalizeTextForTts } from '@/lib/nlp'; import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client'; import { extractTextFromPDF, @@ -312,9 +312,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { }); const trimmedText = rawText.trim(); if (trimmedText) { - const processedText = smartSentenceSplitting - ? processTextToSentences(trimmedText).join(' ') - : trimmedText; + const processedText = smartSentenceSplitting ? normalizeTextForTts(trimmedText) : trimmedText; textPerPage.push(processedText); totalLength += processedText.length; @@ -532,7 +530,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { } const textForTTS = smartSentenceSplitting - ? processTextToSentences(trimmedText).join(' ') + ? normalizeTextForTts(trimmedText) : trimmedText; // Use logical chapter numbering (index + 1) to match original generation titles diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 293821f..2c03d1e 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -37,7 +37,7 @@ import { useAudioContext } from '@/hooks/audio/useAudioContext'; import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/dexie'; import { useBackgroundState } from '@/hooks/audio/useBackgroundState'; import { withRetry, generateTTS, alignAudio } from '@/lib/client'; -import { preprocessSentenceForAudio, processTextToSentences } from '@/lib/nlp'; +import { preprocessSentenceForAudio, splitTextToTtsBlocks } from '@/lib/nlp'; import { isKokoroModel } from '@/utils/voice'; import type { TTSLocation, @@ -379,13 +379,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {string} text - The text to be processed * @returns {Promise} Array of processed sentences */ - const processTextToSentencesLocal = useCallback(async (text: string): Promise => { + const splitTextToTtsBlocksLocal = useCallback(async (text: string): Promise => { if (text.length < 1) { return []; } // Use the shared utility directly instead of making an API call - return processTextToSentences(text); + return splitTextToTtsBlocks(text); }, []); /** @@ -579,7 +579,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement abortAudio(true); // Clear pending requests since text is changing setIsProcessing(true); // Set processing state before text processing starts - processTextToSentencesLocal(workingText) + splitTextToTtsBlocksLocal(workingText) .then(newSentences => { if (newSentences.length === 0) { console.warn('No sentences found in text'); @@ -658,7 +658,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement duration: 3000, }); }); - }, [isPlaying, handleBlankSection, abortAudio, processTextToSentencesLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]); + }, [isPlaying, handleBlankSection, abortAudio, splitTextToTtsBlocksLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]); /** * Toggles the playback state between playing and paused diff --git a/src/lib/nlp.ts b/src/lib/nlp.ts index 13f0167..69cfe99 100644 --- a/src/lib/nlp.ts +++ b/src/lib/nlp.ts @@ -9,6 +9,113 @@ import nlp from 'compromise'; const MAX_BLOCK_LENGTH = 450; +const splitOversizedText = (text: string, maxLen: number): string[] => { + const normalized = text.replace(/\s+/g, ' ').trim(); + if (!normalized) return []; + if (normalized.length <= maxLen) return [normalized]; + + const parts: string[] = []; + const MAX_OVERFLOW = maxLen; // allow finishing the sentence up to +maxLen chars + const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']); + const BREAK_CHARS = new Set(['.', '!', '?']); + const SOFT_BREAK_CHARS = new Set([';', ':']); + + const findPunctuationCut = (s: string, limit: number): number | null => { + for (let i = limit; i >= 0; i--) { + const ch = s[i]; + if (!BREAK_CHARS.has(ch)) continue; + + const prev = i > 0 ? s[i - 1] : ''; + const next = i + 1 < s.length ? s[i + 1] : ''; + + // Avoid splitting inside decimals like 3.14 + if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue; + + let end = i + 1; + while (end < s.length && CLOSERS.has(s[end])) end++; + const after = end < s.length ? s[end] : ''; + + // Allow a boundary at end/whitespace, or common PDF artifact where + // the next sentence starts immediately with an uppercase letter. + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end; + } + return null; + }; + + const findForwardPunctuationCut = ( + s: string, + startIndex: number, + endIndex: number, + chars: Set + ): number | null => { + const start = Math.max(0, startIndex); + const end = Math.min(endIndex, s.length - 1); + for (let i = start; i <= end; i++) { + const ch = s[i]; + if (!chars.has(ch)) continue; + + const prev = i > 0 ? s[i - 1] : ''; + const next = i + 1 < s.length ? s[i + 1] : ''; + + if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue; + + let cut = i + 1; + while (cut < s.length && CLOSERS.has(s[cut])) cut++; + const after = cut < s.length ? s[cut] : ''; + + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return cut; + } + return null; + }; + + const findSoftPunctuationCut = (s: string, limit: number): number | null => { + for (let i = limit; i >= 0; i--) { + const ch = s[i]; + if (!SOFT_BREAK_CHARS.has(ch)) continue; + + let end = i + 1; + while (end < s.length && CLOSERS.has(s[end])) end++; + const after = end < s.length ? s[end] : ''; + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end; + } + return null; + }; + + let remaining = normalized; + while (remaining.length > maxLen) { + const backwardLimit = Math.min(maxLen, remaining.length - 1); + const forwardLimit = Math.min(maxLen + MAX_OVERFLOW, remaining.length - 1); + + let cut = + findPunctuationCut(remaining, backwardLimit) ?? + findForwardPunctuationCut(remaining, maxLen, forwardLimit, BREAK_CHARS) ?? + findSoftPunctuationCut(remaining, backwardLimit) ?? + findForwardPunctuationCut(remaining, maxLen, forwardLimit, SOFT_BREAK_CHARS) ?? + remaining.lastIndexOf(' ', maxLen); + + if (cut === 0 || cut === -1) { + // No whitespace or punctuation; hard-cut for extremely long tokens. + cut = maxLen; + } + + const chunk = remaining.slice(0, cut).trim(); + if (chunk) parts.push(chunk); + remaining = remaining.slice(cut).trim(); + } + + if (remaining) parts.push(remaining); + return parts; +}; + +const normalizeSentenceBoundariesForNlp = (text: string): string => { + // PDF extraction sometimes yields "...end.Next..." with no whitespace. + // Insert a space only when it looks like a sentence boundary (lower/digit before, + // uppercase after) to avoid breaking abbreviations like "U.S.A". + return text + .replace(/([a-z0-9])([.!?])(?=[A-Z])/g, '$1$2 ') + .replace(/([a-z0-9][.!?][\"”’)\]])(?=[A-Z])/g, '$1 '); +}; + /** * Preprocesses text for audio generation by cleaning up various text artifacts * @@ -31,14 +138,18 @@ export const preprocessSentenceForAudio = (text: string): string => { * @param {string} text - The text to split into sentences * @returns {string[]} Array of sentence blocks */ -export const splitIntoSentences = (text: string): string[] => { - const paragraphs = text.split(/\n+/); +export const splitTextToTtsBlocks = (text: string): string[] => { + // Treat double-newlines as paragraph boundaries; single newlines are usually + // just PDF line wrapping and should not force sentence/block boundaries. + const paragraphs = text.split(/\n{2,}/); const blocks: string[] = []; for (const paragraph of paragraphs) { if (!paragraph.trim()) continue; - const cleanedText = preprocessSentenceForAudio(paragraph); + const cleanedText = normalizeSentenceBoundariesForNlp( + preprocessSentenceForAudio(paragraph) + ); const doc = nlp(cleanedText); const rawSentences = doc.sentences().out('array') as string[]; @@ -49,14 +160,17 @@ export const splitIntoSentences = (text: string): string[] => { for (const sentence of mergedSentences) { const trimmedSentence = sentence.trim(); + const sentenceParts = splitOversizedText(trimmedSentence, MAX_BLOCK_LENGTH); - if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) { - blocks.push(currentBlock.trim()); - currentBlock = trimmedSentence; - } else { - currentBlock = currentBlock - ? `${currentBlock} ${trimmedSentence}` - : trimmedSentence; + for (const sentencePart of sentenceParts) { + if (currentBlock && (currentBlock.length + sentencePart.length + 1) > MAX_BLOCK_LENGTH) { + blocks.push(currentBlock.trim()); + currentBlock = sentencePart; + } else { + currentBlock = currentBlock + ? `${currentBlock} ${sentencePart}` + : sentencePart; + } } } @@ -69,29 +183,23 @@ export const splitIntoSentences = (text: string): string[] => { }; /** - * Main sentence processing function that handles both short and long texts + * Normalizes text for single-shot TTS generation (e.g., a whole PDF page). + * Uses the same logic as `splitTextToTtsBlocks`, but returns a single string. * * @param {string} text - The text to process - * @returns {string[]} Array of processed sentences/blocks + * @returns {string} Normalized text */ -export const processTextToSentences = (text: string): string[] => { - if (!text || text.length < 1) { - return []; - } - - // Always use the full splitting logic so we consistently respect - // sentence boundaries and quoted dialogue, even for shorter texts. - return splitIntoSentences(text); -}; +export const normalizeTextForTts = (text: string): string => + splitTextToTtsBlocks(text).join(' '); /** - * Gets raw sentences from text without preprocessing or grouping - * This is useful for text matching and highlighting + * Extracts raw sentence strings from text without preprocessing or block grouping. + * Useful for text matching and highlighting. * * @param {string} text - The text to extract sentences from * @returns {string[]} Array of raw sentences */ -export const getRawSentences = (text: string): string[] => { +export const extractRawSentences = (text: string): string[] => { if (!text || text.length < 1) { return []; } @@ -111,8 +219,8 @@ export const processTextWithMapping = (text: string): { rawSentences: string[]; sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }>; } => { - const rawSentences = getRawSentences(text); - const processedSentences = processTextToSentences(text); + const rawSentences = extractRawSentences(text); + const processedSentences = splitTextToTtsBlocks(text); // Create a mapping between processed sentences and raw sentences const sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }> = []; @@ -149,6 +257,7 @@ export const processTextWithMapping = (text: string): { sentenceMapping }; }; + // Helper functions to merge quoted dialogue across sentences const countDoubleQuotes = (s: string): number => { const matches = s.match(/["“”]/g); @@ -216,4 +325,4 @@ const mergeQuotedDialogue = (rawSentences: string[]): string[] => { } return result; -}; \ No newline at end of file +}; diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.spec.ts index 4bb5485..becc95e 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.spec.ts @@ -1,7 +1,7 @@ import { test, expect } from '@playwright/test'; import { preprocessSentenceForAudio, - splitIntoSentences, + splitTextToTtsBlocks, processTextWithMapping } from '../../src/lib/nlp'; @@ -31,24 +31,24 @@ test.describe('preprocessSentenceForAudio', () => { }); }); -test.describe('splitIntoSentences', () => { +test.describe('splitTextToTtsBlocks', () => { test('groups short sentences into single block', () => { const input = 'First sentence. Second sentence.'; - const result = splitIntoSentences(input); + const result = splitTextToTtsBlocks(input); expect(result).toHaveLength(1); expect(result[0]).toBe('First sentence. Second sentence.'); }); test('merges quoted dialogue (double quotes)', () => { const input = 'He said, "This should be one block." and walked away.'; - const result = splitIntoSentences(input); + const result = splitTextToTtsBlocks(input); expect(result).toHaveLength(1); expect(result[0]).toBe('He said, "This should be one block." and walked away.'); }); test('merges quoted dialogue (curly quotes)', () => { const input = 'She replied, “This also should be merged.” then smiled.'; - const result = splitIntoSentences(input); + const result = splitTextToTtsBlocks(input); expect(result).toHaveLength(1); expect(result[0]).toBe('She replied, “This also should be merged.” then smiled.'); }); @@ -64,12 +64,75 @@ test.describe('splitIntoSentences', () => { // 5 sentences = 505 chars + 4 spaces = 509 chars (> 450). Should split. const input = Array(5).fill(sentence).join(' '); - const result = splitIntoSentences(input); + const result = splitTextToTtsBlocks(input); expect(result.length).toBeGreaterThan(1); // The first block should contain as many as possible expect(result[0].length).toBeLessThanOrEqual(450); }); + + test('splits a single oversized sentence into multiple blocks', () => { + // Some PDF pages (e.g. research papers) can extract into one massive "sentence" + // with few or no punctuation marks; we still must respect MAX_BLOCK_LENGTH. + const input = Array(1200).fill('word').join(' '); // no punctuation + const result = splitTextToTtsBlocks(input); + expect(result.length).toBeGreaterThan(1); + for (const block of result) { + expect(block.length).toBeGreaterThan(0); + expect(block.length).toBeLessThanOrEqual(450); + } + }); + + test('splits extremely long unbroken tokens', () => { + const input = 'A'.repeat(1200); // no spaces, no punctuation + const result = splitTextToTtsBlocks(input); + expect(result.length).toBeGreaterThan(1); + for (const block of result) { + expect(block.length).toBeGreaterThan(0); + expect(block.length).toBeLessThanOrEqual(450); + } + }); + + test('prefers sentence punctuation when chunking long PDF-like text', () => { + const sentences = Array.from({ length: 80 }, (_, i) => + `Sentence ${i} has some filler words to keep the length varying slightly number ${i}.` + ); + // Simulate a common PDF extraction artifact: no whitespace after '.' before the next sentence. + const input = sentences.join(''); + const result = splitTextToTtsBlocks(input); + expect(result.length).toBeGreaterThan(1); + for (const block of result) { + expect(block.length).toBeGreaterThan(0); + expect(block.length).toBeLessThanOrEqual(450); + expect(block.endsWith('.')).toBe(true); + } + }); + + test('does not treat single newlines as paragraph boundaries', () => { + // Many PDFs contain hard-wrapped lines; we should not break blocks/sentences + // just because of a newline. + const input = + 'The first line ends with a comma,\n' + + 'but the sentence continues on the next line and ends here.\n' + + 'And this is the second sentence.'; + const result = splitTextToTtsBlocks(input); + expect(result).toHaveLength(1); + expect(result[0]).toBe( + 'The first line ends with a comma, but the sentence continues on the next line and ends here. And this is the second sentence.' + ); + }); + + test('allows long sentences to reach their ending punctuation', () => { + const longSentence = + `${'word '.repeat(110)}` + // ~550 chars before period + 'end.' + + ' Next.'; + const result = splitTextToTtsBlocks(longSentence); + // The first block should end at a period, not be cut mid-sentence at a space boundary. + expect(result.length).toBeGreaterThan(1); + expect(result[0].endsWith('.')).toBe(true); + expect(result[0].includes('end.')).toBe(true); + }); }); test.describe('processTextWithMapping', () => { From 5a40e4b00a416ef86f14eda45c7e7827fce593d3 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 21 Jan 2026 13:17:57 -0700 Subject: [PATCH 2/6] fix: improve word highlight cleanup in PDFViewer and update test - Add cleanup function to prevent memory leaks in word highlighting - Update test to allow blocks ending with ., !, or ? for better accuracy --- package.json | 2 +- src/components/PDFViewer.tsx | 14 +++++++++----- tests/unit/nlp.spec.ts | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 25d17db..735c6aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openreader-webui", - "version": "v1.2.0", + "version": "v1.2.1", "private": true, "scripts": { "dev": "next dev --turbopack -p 3003", diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index 94ac3f6..6668299 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -138,6 +138,8 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { // Word-level highlight layered on top of the block highlight useEffect(() => { + clearWordHighlightTimeouts(); + if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) { clearWordHighlights(); return; @@ -149,8 +151,7 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { } const wordEntry = - currentSentenceAlignment && - currentWordIndex < currentSentenceAlignment.words.length + currentSentenceAlignment && currentWordIndex < currentSentenceAlignment.words.length ? currentSentenceAlignment.words[currentWordIndex] : undefined; const wordText = wordEntry?.text || null; @@ -160,8 +161,6 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { return; } - clearWordHighlightTimeouts(); - const seq = ++wordHighlightSeqRef.current; const isLayoutChange = layoutKey !== lastWordLayoutKeyRef.current; lastWordLayoutKeyRef.current = layoutKey; @@ -188,13 +187,18 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { } }; + const cleanup = () => { + clearWordHighlightTimeouts(); + }; + if (isLayoutChange) { clearWordHighlights(); scheduleWordTimeout(() => tryApplyWord(0), 250); - return; + return cleanup; } tryApplyWord(0); + return cleanup; }, [ currentWordIndex, currentSentence, diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.spec.ts index becc95e..14dbd86 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.spec.ts @@ -104,7 +104,7 @@ test.describe('splitTextToTtsBlocks', () => { for (const block of result) { expect(block.length).toBeGreaterThan(0); expect(block.length).toBeLessThanOrEqual(450); - expect(block.endsWith('.')).toBe(true); + expect(block).toMatch(/[.!?]$/); } }); From 63c316fc4b26d626be7c716c846801516bdebf42 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 21 Jan 2026 13:32:04 -0700 Subject: [PATCH 3/6] fix(pdfviewer): improve sentence highlight cleanup when current sentence is null Handle case where currentSentence is null by canceling retry loops and clearing stale highlights to prevent lingering highlights. Remove redundant check inside the highlight function. --- src/components/PDFViewer.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index 6668299..cfcd958 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -96,6 +96,14 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { clearSentenceHighlightTimeouts(); + if (!currentSentence) { + // Cancel any in-flight retry loops and ensure stale highlights don't remain + // when the current sentence becomes null/undefined. + sentenceHighlightSeqRef.current += 1; + clearHighlights(); + return; + } + const seq = ++sentenceHighlightSeqRef.current; const isLayoutChange = layoutKey !== lastSentenceLayoutKeyRef.current; lastSentenceLayoutKeyRef.current = layoutKey; @@ -108,7 +116,6 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { if (seq !== sentenceHighlightSeqRef.current) return; const container = containerRef.current; if (!container) return; - if (!currentSentence) return; const spans = container.querySelectorAll('.react-pdf__Page__textContent span'); if (!spans.length) { From efe6ffec86ba4d59a459a378773a739c6ca5877c Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 21 Jan 2026 14:04:19 -0700 Subject: [PATCH 4/6] fix(nlp): add EPUB-specific text splitting for TTS blocks Introduce `splitTextToTtsBlocksEPUB` function to handle EPUB text splitting, treating single newlines as paragraph boundaries for precise highlighting. Update TTS context to conditionally use EPUB splitting based on document type. Enhance PDFViewer to clear highlights when current sentence is null. Add comprehensive tests for the new functionality and refactor existing ones. --- src/components/PDFViewer.tsx | 2 +- src/contexts/TTSContext.tsx | 6 +- src/lib/nlp.ts | 42 ++++- tests/unit/nlp.spec.ts | 290 +++++++++++++++++++++-------------- 4 files changed, 222 insertions(+), 118 deletions(-) diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index cfcd958..0a20958 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -152,7 +152,7 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { return; } - if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { + if (!currentSentence || currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { clearWordHighlights(); return; } diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 2c03d1e..1644c77 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -37,7 +37,7 @@ import { useAudioContext } from '@/hooks/audio/useAudioContext'; import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/dexie'; import { useBackgroundState } from '@/hooks/audio/useBackgroundState'; import { withRetry, generateTTS, alignAudio } from '@/lib/client'; -import { preprocessSentenceForAudio, splitTextToTtsBlocks } from '@/lib/nlp'; +import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/nlp'; import { isKokoroModel } from '@/utils/voice'; import type { TTSLocation, @@ -385,8 +385,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement } // Use the shared utility directly instead of making an API call - return splitTextToTtsBlocks(text); - }, []); + return isEPUB ? splitTextToTtsBlocksEPUB(text) : splitTextToTtsBlocks(text); + }, [isEPUB]); /** * Stops the current audio playback and clears the active Howl instance diff --git a/src/lib/nlp.ts b/src/lib/nlp.ts index 69cfe99..89f0c5e 100644 --- a/src/lib/nlp.ts +++ b/src/lib/nlp.ts @@ -7,7 +7,7 @@ import nlp from 'compromise'; -const MAX_BLOCK_LENGTH = 450; +export const MAX_BLOCK_LENGTH = 450; const splitOversizedText = (text: string, maxLen: number): string[] => { const normalized = text.replace(/\s+/g, ' ').trim(); @@ -182,6 +182,46 @@ export const splitTextToTtsBlocks = (text: string): string[] => { return blocks; }; +/** + * EPUB block splitting used where we want the produced sentences + * to closely match the original DOM text (for exact-match highlighting). + */ +export const splitTextToTtsBlocksEPUB = (text: string): string[] => { + const paragraphs = text.split(/\n+/); + const blocks: string[] = []; + + for (const paragraph of paragraphs) { + if (!paragraph.trim()) continue; + + const cleanedText = preprocessSentenceForAudio(paragraph); + const doc = nlp(cleanedText); + const rawSentences = doc.sentences().out('array') as string[]; + + const mergedSentences = mergeQuotedDialogue(rawSentences); + + let currentBlock = ''; + + for (const sentence of mergedSentences) { + const trimmedSentence = sentence.trim(); + + if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) { + blocks.push(currentBlock.trim()); + currentBlock = trimmedSentence; + } else { + currentBlock = currentBlock + ? `${currentBlock} ${trimmedSentence}` + : trimmedSentence; + } + } + + if (currentBlock) { + blocks.push(currentBlock.trim()); + } + } + + return blocks; +}; + /** * Normalizes text for single-shot TTS generation (e.g., a whole PDF page). * Uses the same logic as `splitTextToTtsBlocks`, but returns a single string. diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.spec.ts index 14dbd86..ada924f 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.spec.ts @@ -2,150 +2,214 @@ import { test, expect } from '@playwright/test'; import { preprocessSentenceForAudio, splitTextToTtsBlocks, - processTextWithMapping + splitTextToTtsBlocksEPUB, + normalizeTextForTts, + extractRawSentences, + processTextWithMapping, + MAX_BLOCK_LENGTH } from '../../src/lib/nlp'; +const PDF_MAX_BLOCK_LENGTH = MAX_BLOCK_LENGTH * 2; // splitTextToTtsBlocks can overflow to reach punctuation + +const expectNormalizedBlocks = (blocks: string[], maxLen = Number.POSITIVE_INFINITY) => { + for (const block of blocks) { + expect(block.trim().length).toBeGreaterThan(0); + expect(block.length).toBeLessThanOrEqual(maxLen); + expect(block).not.toMatch(/\n/); + expect(block).not.toMatch(/\s{2,}/); + } +}; + test.describe('preprocessSentenceForAudio', () => { - test('removes URLs', () => { - const input = 'Check out https://example.com/page for more info'; - const expected = 'Check out - (link to example.com) - for more info'; - expect(preprocessSentenceForAudio(input)).toBe(expected); - }); + test('normalizes common extraction artifacts', () => { + const cases: Array<{ input: string; expected: string }> = [ + { + input: 'Check out https://example.com/page for more info', + expected: 'Check out - (link to example.com) - for more info', + }, + { + input: 'This is a hyp- henated word', + expected: 'This is a hyphenated word', + }, + { + input: 'This is *bold* text', + expected: 'This is bold text', + }, + { + input: 'Multiple spaces', + expected: 'Multiple spaces', + }, + ]; - test('removes hyphenation', () => { - const input = 'This is a hyp- henated word'; - const expected = 'This is a hyphenated word'; - expect(preprocessSentenceForAudio(input)).toBe(expected); - }); - - test('removes asterisks', () => { - const input = 'This is *bold* text'; - const expected = 'This is bold text'; - expect(preprocessSentenceForAudio(input)).toBe(expected); - }); - - test('collapses whitespace', () => { - const input = 'Multiple spaces'; - const expected = 'Multiple spaces'; - expect(preprocessSentenceForAudio(input)).toBe(expected); + for (const { input, expected } of cases) { + expect(preprocessSentenceForAudio(input)).toBe(expected); + } }); }); -test.describe('splitTextToTtsBlocks', () => { - test('groups short sentences into single block', () => { - const input = 'First sentence. Second sentence.'; - const result = splitTextToTtsBlocks(input); - expect(result).toHaveLength(1); - expect(result[0]).toBe('First sentence. Second sentence.'); - }); - - test('merges quoted dialogue (double quotes)', () => { - const input = 'He said, "This should be one block." and walked away.'; - const result = splitTextToTtsBlocks(input); - expect(result).toHaveLength(1); - expect(result[0]).toBe('He said, "This should be one block." and walked away.'); - }); - - test('merges quoted dialogue (curly quotes)', () => { - const input = 'She replied, “This also should be merged.” then smiled.'; - const result = splitTextToTtsBlocks(input); - expect(result).toHaveLength(1); - expect(result[0]).toBe('She replied, “This also should be merged.” then smiled.'); - }); - - test('respects max block length for long text', () => { - // MAX_BLOCK_LENGTH is 450 in nlp.ts - // We construct distinct sentences. - // If we make sentences short enough individually but long enough combined, - // they should be grouped until the limit is reached. - - const sentence = 'A'.repeat(100) + '.'; // 101 chars - // 4 sentences = 404 chars + 3 spaces = 407 chars (< 450). Should fit in one block. - // 5 sentences = 505 chars + 4 spaces = 509 chars (> 450). Should split. - - const input = Array(5).fill(sentence).join(' '); - const result = splitTextToTtsBlocks(input); - - expect(result.length).toBeGreaterThan(1); - // The first block should contain as many as possible - expect(result[0].length).toBeLessThanOrEqual(450); - }); - - test('splits a single oversized sentence into multiple blocks', () => { - // Some PDF pages (e.g. research papers) can extract into one massive "sentence" - // with few or no punctuation marks; we still must respect MAX_BLOCK_LENGTH. - const input = Array(1200).fill('word').join(' '); // no punctuation - const result = splitTextToTtsBlocks(input); - expect(result.length).toBeGreaterThan(1); - for (const block of result) { - expect(block.length).toBeGreaterThan(0); - expect(block.length).toBeLessThanOrEqual(450); - } - }); - - test('splits extremely long unbroken tokens', () => { - const input = 'A'.repeat(1200); // no spaces, no punctuation - const result = splitTextToTtsBlocks(input); - expect(result.length).toBeGreaterThan(1); - for (const block of result) { - expect(block.length).toBeGreaterThan(0); - expect(block.length).toBeLessThanOrEqual(450); - } - }); - - test('prefers sentence punctuation when chunking long PDF-like text', () => { - const sentences = Array.from({ length: 80 }, (_, i) => - `Sentence ${i} has some filler words to keep the length varying slightly number ${i}.` - ); - // Simulate a common PDF extraction artifact: no whitespace after '.' before the next sentence. - const input = sentences.join(''); - const result = splitTextToTtsBlocks(input); - expect(result.length).toBeGreaterThan(1); - for (const block of result) { - expect(block.length).toBeGreaterThan(0); - expect(block.length).toBeLessThanOrEqual(450); - expect(block).toMatch(/[.!?]$/); - } +test.describe('splitTextToTtsBlocks (PDF-oriented)', () => { + test('returns [] for empty input', () => { + expect(splitTextToTtsBlocks('')).toEqual([]); + expect(splitTextToTtsBlocks(' ')).toEqual([]); + expect(splitTextToTtsBlocks('\n\n')).toEqual([]); }); test('does not treat single newlines as paragraph boundaries', () => { - // Many PDFs contain hard-wrapped lines; we should not break blocks/sentences - // just because of a newline. const input = 'The first line ends with a comma,\n' + 'but the sentence continues on the next line and ends here.\n' + 'And this is the second sentence.'; const result = splitTextToTtsBlocks(input); + expect(result).toHaveLength(1); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); expect(result[0]).toBe( 'The first line ends with a comma, but the sentence continues on the next line and ends here. And this is the second sentence.' ); }); - test('allows long sentences to reach their ending punctuation', () => { - const longSentence = - `${'word '.repeat(110)}` + // ~550 chars before period - 'end.' + - ' Next.'; - const result = splitTextToTtsBlocks(longSentence); - // The first block should end at a period, not be cut mid-sentence at a space boundary. + test('treats blank lines (double newlines) as paragraph boundaries', () => { + const input = 'First paragraph.\n\nSecond paragraph.'; + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThanOrEqual(2); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + }); + + test('repairs missing whitespace between sentences (common PDF artifact)', () => { + const input = 'This ends.Here starts.'; + const normalized = normalizeTextForTts(input); + expect(normalized).toContain('ends. Here'); + }); + + test('does not break decimals when repairing sentence boundaries', () => { + const input = 'Pi is 3.14.Next sentence.'; + const normalized = normalizeTextForTts(input); + expect(normalized).toContain('3.14'); + }); + + test('enforces max block length on long content', () => { + const sentence = `${'A'.repeat(100)}.`; // 101 chars + const input = Array(8).fill(sentence).join(' '); // guaranteed to exceed MAX_BLOCK_LENGTH + const result = splitTextToTtsBlocks(input); + expect(result.length).toBeGreaterThan(1); - expect(result[0].endsWith('.')).toBe(true); - expect(result[0].includes('end.')).toBe(true); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); + + test('splits oversized content with no punctuation', () => { + const input = Array(1200).fill('word').join(' '); + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); + + test('splits extremely long unbroken tokens', () => { + const input = 'A'.repeat(1200); + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); + + test('prefers sentence punctuation when chunking long PDF-like text', () => { + const sentences = Array.from( + { length: 80 }, + (_, i) => `Sentence ${i} has filler words to vary length slightly number ${i}.` + ); + const input = sentences.join(''); // no whitespace after '.' between sentences + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + + // When sentence punctuation exists, blocks should usually end at punctuation/closers. + // This guards against regressions where we cut mid-word/mid-sentence too often. + for (const block of result) { + expect(block).toMatch(/[.!?]["'”’)\]]*$/); + } + }); + + test('allows a long sentence to extend to its ending punctuation', () => { + // Create a single sentence that exceeds MAX_BLOCK_LENGTH, but ends with a period + // within the forward-search overflow window. + const input = `${'word '.repeat(110)}end. Next.`; + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + // This case is specifically asserting we may exceed MAX_BLOCK_LENGTH to reach punctuation, + // but should still remain bounded by the overflow policy. + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + expect(result[0]).toMatch(/end\.$/); + }); + + test('merges multi-sentence quoted dialogue', () => { + const input = 'He said, "First. Second." Then left.'; + const result = splitTextToTtsBlocks(input); + + expect(result).toHaveLength(1); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + expect(result[0]).toContain('"First. Second."'); + }); +}); + +test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { + test('returns [] for empty input', () => { + expect(splitTextToTtsBlocksEPUB('')).toEqual([]); + expect(splitTextToTtsBlocksEPUB(' ')).toEqual([]); + expect(splitTextToTtsBlocksEPUB('\n\n')).toEqual([]); + }); + + test('treats single newlines as paragraph boundaries', () => { + const input = 'One.\nTwo.'; + const result = splitTextToTtsBlocksEPUB(input); + expect(result).toHaveLength(2); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + expect(result[0]).toBe('One.'); + expect(result[1]).toBe('Two.'); + }); +}); + +test.describe('normalizeTextForTts', () => { + test('returns a single normalized string without newlines', () => { + const input = 'Hello.\nWorld.\n\nNext paragraph.'; + const normalized = normalizeTextForTts(input); + expect(normalized).not.toMatch(/\n/); + expect(normalized).not.toMatch(/\s{2,}/); + expect(normalized.length).toBeGreaterThan(0); + }); +}); + +test.describe('extractRawSentences', () => { + test('returns [] for empty input', () => { + expect(extractRawSentences('')).toEqual([]); + }); + + test('returns sentence-like strings without preprocessing', () => { + const input = 'First sentence. Second sentence.'; + const result = extractRawSentences(input); + expect(result.length).toBeGreaterThanOrEqual(2); + expect(result[0]).toContain('First'); }); }); test.describe('processTextWithMapping', () => { - test('maps raw sentences to processed ones', () => { + test('returns mapping entries with valid indices', () => { const text = 'First (1). Second (2).'; const { processedSentences, rawSentences, sentenceMapping } = processTextWithMapping(text); expect(processedSentences.length).toBeGreaterThan(0); expect(rawSentences.length).toBeGreaterThan(0); expect(sentenceMapping).toHaveLength(processedSentences.length); - - // Check structure of mapping - expect(sentenceMapping[0]).toHaveProperty('processedIndex'); - expect(sentenceMapping[0]).toHaveProperty('rawIndices'); + + for (let i = 0; i < sentenceMapping.length; i++) { + const entry = sentenceMapping[i]; + expect(entry.processedIndex).toBe(i); + for (const rawIndex of entry.rawIndices) { + expect(rawIndex).toBeGreaterThanOrEqual(0); + expect(rawIndex).toBeLessThan(rawSentences.length); + } + } }); }); From 54b96915240665d87b9bd912eab710577d2dc31a Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 21 Jan 2026 14:27:20 -0700 Subject: [PATCH 5/6] refactor(nlp): enhance EPUB text splitting for oversized sentences and remove unused functions - Add logic to split oversized sentences in `splitTextToTtsBlocksEPUB` using `splitOversizedText` to ensure blocks stay within `MAX_BLOCK_LENGTH` - Remove `extractRawSentences` and `processTextWithMapping` functions as they are no longer needed - Update tests to reflect the changes, including a new test for oversized sentence splitting and removal of related test suites --- src/lib/nlp.ts | 86 +++++++----------------------------------- tests/unit/nlp.spec.ts | 43 ++++----------------- 2 files changed, 21 insertions(+), 108 deletions(-) diff --git a/src/lib/nlp.ts b/src/lib/nlp.ts index 89f0c5e..6b42697 100644 --- a/src/lib/nlp.ts +++ b/src/lib/nlp.ts @@ -203,14 +203,20 @@ export const splitTextToTtsBlocksEPUB = (text: string): string[] => { for (const sentence of mergedSentences) { const trimmedSentence = sentence.trim(); + const sentenceParts = + trimmedSentence.length > MAX_BLOCK_LENGTH + ? splitOversizedText(trimmedSentence, MAX_BLOCK_LENGTH) + : [trimmedSentence]; - if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) { - blocks.push(currentBlock.trim()); - currentBlock = trimmedSentence; - } else { - currentBlock = currentBlock - ? `${currentBlock} ${trimmedSentence}` - : trimmedSentence; + for (const sentencePart of sentenceParts) { + if (currentBlock && (currentBlock.length + sentencePart.length + 1) > MAX_BLOCK_LENGTH) { + blocks.push(currentBlock.trim()); + currentBlock = sentencePart; + } else { + currentBlock = currentBlock + ? `${currentBlock} ${sentencePart}` + : sentencePart; + } } } @@ -232,72 +238,6 @@ export const splitTextToTtsBlocksEPUB = (text: string): string[] => { export const normalizeTextForTts = (text: string): string => splitTextToTtsBlocks(text).join(' '); -/** - * Extracts raw sentence strings from text without preprocessing or block grouping. - * Useful for text matching and highlighting. - * - * @param {string} text - The text to extract sentences from - * @returns {string[]} Array of raw sentences - */ -export const extractRawSentences = (text: string): string[] => { - if (!text || text.length < 1) { - return []; - } - - return nlp(text).sentences().out('array') as string[]; -}; - -/** - * Enhanced sentence processing that returns both processed sentences and raw sentences - * This allows for better mapping between the two for click-to-highlight functionality - * - * @param {string} text - The text to process - * @returns {Object} Object containing processed sentences and raw sentences with mapping - */ -export const processTextWithMapping = (text: string): { - processedSentences: string[]; - rawSentences: string[]; - sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }>; -} => { - const rawSentences = extractRawSentences(text); - const processedSentences = splitTextToTtsBlocks(text); - - // Create a mapping between processed sentences and raw sentences - const sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }> = []; - - // For simple mapping, we'll track which raw sentences contributed to each processed sentence - let rawIndex = 0; - - for (let processedIndex = 0; processedIndex < processedSentences.length; processedIndex++) { - const processedSentence = processedSentences[processedIndex]; - const rawIndices: number[] = []; - - // Find which raw sentences are contained in this processed sentence - const remainingText = processedSentence; - - while (rawIndex < rawSentences.length && remainingText.length > 0) { - const rawSentence = rawSentences[rawIndex]; - const cleanedRawSentence = preprocessSentenceForAudio(rawSentence); - - if (remainingText.includes(cleanedRawSentence) || cleanedRawSentence.includes(remainingText)) { - rawIndices.push(rawIndex); - rawIndex++; - break; - } else { - rawIndex++; - } - } - - sentenceMapping.push({ processedIndex, rawIndices }); - } - - return { - processedSentences, - rawSentences, - sentenceMapping - }; -}; - // Helper functions to merge quoted dialogue across sentences const countDoubleQuotes = (s: string): number => { const matches = s.match(/["“”]/g); diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.spec.ts index ada924f..cb7164b 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.spec.ts @@ -4,8 +4,6 @@ import { splitTextToTtsBlocks, splitTextToTtsBlocksEPUB, normalizeTextForTts, - extractRawSentences, - processTextWithMapping, MAX_BLOCK_LENGTH } from '../../src/lib/nlp'; @@ -169,6 +167,14 @@ test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { expect(result[0]).toBe('One.'); expect(result[1]).toBe('Two.'); }); + + test('splits oversized sentences to keep blocks bounded', () => { + const input = Array(1200).fill('word').join(' '); // no punctuation; guaranteed to exceed MAX_BLOCK_LENGTH + const result = splitTextToTtsBlocksEPUB(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); }); test.describe('normalizeTextForTts', () => { @@ -180,36 +186,3 @@ test.describe('normalizeTextForTts', () => { expect(normalized.length).toBeGreaterThan(0); }); }); - -test.describe('extractRawSentences', () => { - test('returns [] for empty input', () => { - expect(extractRawSentences('')).toEqual([]); - }); - - test('returns sentence-like strings without preprocessing', () => { - const input = 'First sentence. Second sentence.'; - const result = extractRawSentences(input); - expect(result.length).toBeGreaterThanOrEqual(2); - expect(result[0]).toContain('First'); - }); -}); - -test.describe('processTextWithMapping', () => { - test('returns mapping entries with valid indices', () => { - const text = 'First (1). Second (2).'; - const { processedSentences, rawSentences, sentenceMapping } = processTextWithMapping(text); - - expect(processedSentences.length).toBeGreaterThan(0); - expect(rawSentences.length).toBeGreaterThan(0); - expect(sentenceMapping).toHaveLength(processedSentences.length); - - for (let i = 0; i < sentenceMapping.length; i++) { - const entry = sentenceMapping[i]; - expect(entry.processedIndex).toBe(i); - for (const rawIndex of entry.rawIndices) { - expect(rawIndex).toBeGreaterThanOrEqual(0); - expect(rawIndex).toBeLessThan(rawSentences.length); - } - } - }); -}); From ddf8912092ecbdc99cadc812a3d9fb8759023c06 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 21 Jan 2026 14:42:13 -0700 Subject: [PATCH 6/6] chore(config): ignore unit tests in firefox and webkit browsers --- playwright.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playwright.config.ts b/playwright.config.ts index 7b20bed..877a6a7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -44,6 +44,7 @@ export default defineConfig({ { name: 'firefox', + testIgnore: '**/unit/**', use: { ...devices['Desktop Firefox'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'firefox' }, @@ -52,6 +53,7 @@ export default defineConfig({ { name: 'webkit', + testIgnore: '**/unit/**', use: { ...devices['Desktop Safari'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'webkit' },