diff --git a/package.json b/package.json index 25d17db..735c6aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openreader-webui", - "version": "v1.2.0", + "version": "v1.2.1", "private": true, "scripts": { "dev": "next dev --turbopack -p 3003", diff --git a/playwright.config.ts b/playwright.config.ts index 7b20bed..877a6a7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -44,6 +44,7 @@ export default defineConfig({ { name: 'firefox', + testIgnore: '**/unit/**', use: { ...devices['Desktop Firefox'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'firefox' }, @@ -52,6 +53,7 @@ export default defineConfig({ { name: 'webkit', + testIgnore: '**/unit/**', use: { ...devices['Desktop Safari'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'webkit' }, diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index de88b1d..0a20958 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -24,6 +24,12 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { const containerRef = useRef(null); const scaleRef = useRef(1); const { containerWidth } = usePDFResize(containerRef); + const sentenceHighlightSeqRef = useRef(0); + const wordHighlightSeqRef = useRef(0); + const sentenceHighlightTimeoutsRef = useRef[]>([]); + const wordHighlightTimeoutsRef = useRef[]>([]); + const lastSentenceLayoutKeyRef = useRef(''); + const lastWordLayoutKeyRef = useRef(''); // Config context const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig(); @@ -49,6 +55,28 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { currDocPage, } = usePDF(); + const layoutKey = `${zoomLevel}:${containerWidth}:${viewType}:${currDocPage}`; + + const clearSentenceHighlightTimeouts = useCallback(() => { + for (const t of sentenceHighlightTimeoutsRef.current) clearTimeout(t); + sentenceHighlightTimeoutsRef.current = []; + }, []); + + const clearWordHighlightTimeouts = useCallback(() => { + for (const t of wordHighlightTimeoutsRef.current) clearTimeout(t); + wordHighlightTimeoutsRef.current = []; + }, []); + + const scheduleSentenceTimeout = useCallback((fn: () => void, ms: number) => { + const t = setTimeout(fn, ms); + sentenceHighlightTimeoutsRef.current.push(t); + }, []); + + const scheduleWordTimeout = useCallback((fn: () => void, ms: number) => { + const t = setTimeout(fn, ms); + wordHighlightTimeoutsRef.current.push(t); + }, []); + useEffect(() => { /* * Handles highlighting the current sentence being read by TTS. @@ -66,33 +94,71 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { return; } - const highlightTimeout = setTimeout(() => { - if (containerRef.current) { - highlightPattern(currDocText, currentSentence || '', containerRef as RefObject); + clearSentenceHighlightTimeouts(); + + if (!currentSentence) { + // Cancel any in-flight retry loops and ensure stale highlights don't remain + // when the current sentence becomes null/undefined. + sentenceHighlightSeqRef.current += 1; + clearHighlights(); + return; + } + + const seq = ++sentenceHighlightSeqRef.current; + const isLayoutChange = layoutKey !== lastSentenceLayoutKeyRef.current; + lastSentenceLayoutKeyRef.current = layoutKey; + + if (isLayoutChange) { + clearHighlights(); + } + + const tryApply = (attempt: number) => { + if (seq !== sentenceHighlightSeqRef.current) return; + const container = containerRef.current; + if (!container) return; + + const spans = container.querySelectorAll('.react-pdf__Page__textContent span'); + if (!spans.length) { + if (attempt < 10) scheduleSentenceTimeout(() => tryApply(attempt + 1), 75); + return; } - }, 200); + + highlightPattern(currDocText, currentSentence, containerRef as RefObject); + }; + + scheduleSentenceTimeout(() => tryApply(0), 200); return () => { - clearTimeout(highlightTimeout); + clearSentenceHighlightTimeouts(); clearHighlights(); }; - }, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]); + }, [ + currDocText, + currentSentence, + highlightPattern, + clearHighlights, + pdfHighlightEnabled, + layoutKey, + clearSentenceHighlightTimeouts, + scheduleSentenceTimeout + ]); // Word-level highlight layered on top of the block highlight useEffect(() => { + clearWordHighlightTimeouts(); + if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) { clearWordHighlights(); return; } - if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { + if (!currentSentence || currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) { clearWordHighlights(); return; } const wordEntry = - currentSentenceAlignment && - currentWordIndex < currentSentenceAlignment.words.length + currentSentenceAlignment && currentWordIndex < currentSentenceAlignment.words.length ? currentSentenceAlignment.words[currentWordIndex] : undefined; const wordText = wordEntry?.text || null; @@ -102,12 +168,44 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { return; } - highlightWordIndex( - currentSentenceAlignment, - currentWordIndex, - currentSentence || '', - containerRef as RefObject - ); + const seq = ++wordHighlightSeqRef.current; + const isLayoutChange = layoutKey !== lastWordLayoutKeyRef.current; + lastWordLayoutKeyRef.current = layoutKey; + + const tryApplyWord = (attempt: number) => { + if (seq !== wordHighlightSeqRef.current) return; + const container = containerRef.current; + if (!container) return; + + highlightWordIndex( + currentSentenceAlignment, + currentWordIndex, + currentSentence || '', + containerRef as RefObject + ); + + if (isLayoutChange) { + // If we don't see a word overlay yet, the sentence highlight worker may not + // have produced `lastSentenceTokenWindow` (or the text layer isn't ready). + const overlayCount = container.querySelectorAll('.pdf-word-highlight-overlay').length; + if (overlayCount === 0 && attempt < 12) { + scheduleWordTimeout(() => tryApplyWord(attempt + 1), 75); + } + } + }; + + const cleanup = () => { + clearWordHighlightTimeouts(); + }; + + if (isLayoutChange) { + clearWordHighlights(); + scheduleWordTimeout(() => tryApplyWord(0), 250); + return cleanup; + } + + tryApplyWord(0); + return cleanup; }, [ currentWordIndex, currentSentence, @@ -115,7 +213,10 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { pdfHighlightEnabled, pdfWordHighlightEnabled, clearWordHighlights, - highlightWordIndex + highlightWordIndex, + layoutKey, + clearWordHighlightTimeouts, + scheduleWordTimeout ]); // Add page dimensions state diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index a771452..ede6879 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -30,7 +30,7 @@ import type { PDFDocumentProxy } from 'pdfjs-dist'; import { getPdfDocument } from '@/lib/dexie'; import { useTTS } from '@/contexts/TTSContext'; import { useConfig } from '@/contexts/ConfigContext'; -import { processTextToSentences } from '@/lib/nlp'; +import { normalizeTextForTts } from '@/lib/nlp'; import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client'; import { extractTextFromPDF, @@ -312,9 +312,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { }); const trimmedText = rawText.trim(); if (trimmedText) { - const processedText = smartSentenceSplitting - ? processTextToSentences(trimmedText).join(' ') - : trimmedText; + const processedText = smartSentenceSplitting ? normalizeTextForTts(trimmedText) : trimmedText; textPerPage.push(processedText); totalLength += processedText.length; @@ -532,7 +530,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { } const textForTTS = smartSentenceSplitting - ? processTextToSentences(trimmedText).join(' ') + ? normalizeTextForTts(trimmedText) : trimmedText; // Use logical chapter numbering (index + 1) to match original generation titles diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 293821f..1644c77 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -37,7 +37,7 @@ import { useAudioContext } from '@/hooks/audio/useAudioContext'; import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/dexie'; import { useBackgroundState } from '@/hooks/audio/useBackgroundState'; import { withRetry, generateTTS, alignAudio } from '@/lib/client'; -import { preprocessSentenceForAudio, processTextToSentences } from '@/lib/nlp'; +import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/nlp'; import { isKokoroModel } from '@/utils/voice'; import type { TTSLocation, @@ -379,14 +379,14 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * @param {string} text - The text to be processed * @returns {Promise} Array of processed sentences */ - const processTextToSentencesLocal = useCallback(async (text: string): Promise => { + const splitTextToTtsBlocksLocal = useCallback(async (text: string): Promise => { if (text.length < 1) { return []; } // Use the shared utility directly instead of making an API call - return processTextToSentences(text); - }, []); + return isEPUB ? splitTextToTtsBlocksEPUB(text) : splitTextToTtsBlocks(text); + }, [isEPUB]); /** * Stops the current audio playback and clears the active Howl instance @@ -579,7 +579,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement abortAudio(true); // Clear pending requests since text is changing setIsProcessing(true); // Set processing state before text processing starts - processTextToSentencesLocal(workingText) + splitTextToTtsBlocksLocal(workingText) .then(newSentences => { if (newSentences.length === 0) { console.warn('No sentences found in text'); @@ -658,7 +658,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement duration: 3000, }); }); - }, [isPlaying, handleBlankSection, abortAudio, processTextToSentencesLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]); + }, [isPlaying, handleBlankSection, abortAudio, splitTextToTtsBlocksLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]); /** * Toggles the playback state between playing and paused diff --git a/src/lib/nlp.ts b/src/lib/nlp.ts index 13f0167..6b42697 100644 --- a/src/lib/nlp.ts +++ b/src/lib/nlp.ts @@ -7,7 +7,114 @@ import nlp from 'compromise'; -const MAX_BLOCK_LENGTH = 450; +export const MAX_BLOCK_LENGTH = 450; + +const splitOversizedText = (text: string, maxLen: number): string[] => { + const normalized = text.replace(/\s+/g, ' ').trim(); + if (!normalized) return []; + if (normalized.length <= maxLen) return [normalized]; + + const parts: string[] = []; + const MAX_OVERFLOW = maxLen; // allow finishing the sentence up to +maxLen chars + const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']); + const BREAK_CHARS = new Set(['.', '!', '?']); + const SOFT_BREAK_CHARS = new Set([';', ':']); + + const findPunctuationCut = (s: string, limit: number): number | null => { + for (let i = limit; i >= 0; i--) { + const ch = s[i]; + if (!BREAK_CHARS.has(ch)) continue; + + const prev = i > 0 ? s[i - 1] : ''; + const next = i + 1 < s.length ? s[i + 1] : ''; + + // Avoid splitting inside decimals like 3.14 + if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue; + + let end = i + 1; + while (end < s.length && CLOSERS.has(s[end])) end++; + const after = end < s.length ? s[end] : ''; + + // Allow a boundary at end/whitespace, or common PDF artifact where + // the next sentence starts immediately with an uppercase letter. + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end; + } + return null; + }; + + const findForwardPunctuationCut = ( + s: string, + startIndex: number, + endIndex: number, + chars: Set + ): number | null => { + const start = Math.max(0, startIndex); + const end = Math.min(endIndex, s.length - 1); + for (let i = start; i <= end; i++) { + const ch = s[i]; + if (!chars.has(ch)) continue; + + const prev = i > 0 ? s[i - 1] : ''; + const next = i + 1 < s.length ? s[i + 1] : ''; + + if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue; + + let cut = i + 1; + while (cut < s.length && CLOSERS.has(s[cut])) cut++; + const after = cut < s.length ? s[cut] : ''; + + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return cut; + } + return null; + }; + + const findSoftPunctuationCut = (s: string, limit: number): number | null => { + for (let i = limit; i >= 0; i--) { + const ch = s[i]; + if (!SOFT_BREAK_CHARS.has(ch)) continue; + + let end = i + 1; + while (end < s.length && CLOSERS.has(s[end])) end++; + const after = end < s.length ? s[end] : ''; + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end; + } + return null; + }; + + let remaining = normalized; + while (remaining.length > maxLen) { + const backwardLimit = Math.min(maxLen, remaining.length - 1); + const forwardLimit = Math.min(maxLen + MAX_OVERFLOW, remaining.length - 1); + + let cut = + findPunctuationCut(remaining, backwardLimit) ?? + findForwardPunctuationCut(remaining, maxLen, forwardLimit, BREAK_CHARS) ?? + findSoftPunctuationCut(remaining, backwardLimit) ?? + findForwardPunctuationCut(remaining, maxLen, forwardLimit, SOFT_BREAK_CHARS) ?? + remaining.lastIndexOf(' ', maxLen); + + if (cut === 0 || cut === -1) { + // No whitespace or punctuation; hard-cut for extremely long tokens. + cut = maxLen; + } + + const chunk = remaining.slice(0, cut).trim(); + if (chunk) parts.push(chunk); + remaining = remaining.slice(cut).trim(); + } + + if (remaining) parts.push(remaining); + return parts; +}; + +const normalizeSentenceBoundariesForNlp = (text: string): string => { + // PDF extraction sometimes yields "...end.Next..." with no whitespace. + // Insert a space only when it looks like a sentence boundary (lower/digit before, + // uppercase after) to avoid breaking abbreviations like "U.S.A". + return text + .replace(/([a-z0-9])([.!?])(?=[A-Z])/g, '$1$2 ') + .replace(/([a-z0-9][.!?][\"”’)\]])(?=[A-Z])/g, '$1 '); +}; /** * Preprocesses text for audio generation by cleaning up various text artifacts @@ -31,14 +138,18 @@ export const preprocessSentenceForAudio = (text: string): string => { * @param {string} text - The text to split into sentences * @returns {string[]} Array of sentence blocks */ -export const splitIntoSentences = (text: string): string[] => { - const paragraphs = text.split(/\n+/); +export const splitTextToTtsBlocks = (text: string): string[] => { + // Treat double-newlines as paragraph boundaries; single newlines are usually + // just PDF line wrapping and should not force sentence/block boundaries. + const paragraphs = text.split(/\n{2,}/); const blocks: string[] = []; for (const paragraph of paragraphs) { if (!paragraph.trim()) continue; - const cleanedText = preprocessSentenceForAudio(paragraph); + const cleanedText = normalizeSentenceBoundariesForNlp( + preprocessSentenceForAudio(paragraph) + ); const doc = nlp(cleanedText); const rawSentences = doc.sentences().out('array') as string[]; @@ -49,14 +160,17 @@ export const splitIntoSentences = (text: string): string[] => { for (const sentence of mergedSentences) { const trimmedSentence = sentence.trim(); + const sentenceParts = splitOversizedText(trimmedSentence, MAX_BLOCK_LENGTH); - if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) { - blocks.push(currentBlock.trim()); - currentBlock = trimmedSentence; - } else { - currentBlock = currentBlock - ? `${currentBlock} ${trimmedSentence}` - : trimmedSentence; + for (const sentencePart of sentenceParts) { + if (currentBlock && (currentBlock.length + sentencePart.length + 1) > MAX_BLOCK_LENGTH) { + blocks.push(currentBlock.trim()); + currentBlock = sentencePart; + } else { + currentBlock = currentBlock + ? `${currentBlock} ${sentencePart}` + : sentencePart; + } } } @@ -69,86 +183,61 @@ export const splitIntoSentences = (text: string): string[] => { }; /** - * Main sentence processing function that handles both short and long texts - * - * @param {string} text - The text to process - * @returns {string[]} Array of processed sentences/blocks + * EPUB block splitting used where we want the produced sentences + * to closely match the original DOM text (for exact-match highlighting). */ -export const processTextToSentences = (text: string): string[] => { - if (!text || text.length < 1) { - return []; - } +export const splitTextToTtsBlocksEPUB = (text: string): string[] => { + const paragraphs = text.split(/\n+/); + const blocks: string[] = []; - // Always use the full splitting logic so we consistently respect - // sentence boundaries and quoted dialogue, even for shorter texts. - return splitIntoSentences(text); -}; + for (const paragraph of paragraphs) { + if (!paragraph.trim()) continue; -/** - * Gets raw sentences from text without preprocessing or grouping - * This is useful for text matching and highlighting - * - * @param {string} text - The text to extract sentences from - * @returns {string[]} Array of raw sentences - */ -export const getRawSentences = (text: string): string[] => { - if (!text || text.length < 1) { - return []; - } - - return nlp(text).sentences().out('array') as string[]; -}; + const cleanedText = preprocessSentenceForAudio(paragraph); + const doc = nlp(cleanedText); + const rawSentences = doc.sentences().out('array') as string[]; -/** - * Enhanced sentence processing that returns both processed sentences and raw sentences - * This allows for better mapping between the two for click-to-highlight functionality - * - * @param {string} text - The text to process - * @returns {Object} Object containing processed sentences and raw sentences with mapping - */ -export const processTextWithMapping = (text: string): { - processedSentences: string[]; - rawSentences: string[]; - sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }>; -} => { - const rawSentences = getRawSentences(text); - const processedSentences = processTextToSentences(text); - - // Create a mapping between processed sentences and raw sentences - const sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }> = []; - - // For simple mapping, we'll track which raw sentences contributed to each processed sentence - let rawIndex = 0; - - for (let processedIndex = 0; processedIndex < processedSentences.length; processedIndex++) { - const processedSentence = processedSentences[processedIndex]; - const rawIndices: number[] = []; - - // Find which raw sentences are contained in this processed sentence - const remainingText = processedSentence; - - while (rawIndex < rawSentences.length && remainingText.length > 0) { - const rawSentence = rawSentences[rawIndex]; - const cleanedRawSentence = preprocessSentenceForAudio(rawSentence); - - if (remainingText.includes(cleanedRawSentence) || cleanedRawSentence.includes(remainingText)) { - rawIndices.push(rawIndex); - rawIndex++; - break; - } else { - rawIndex++; + const mergedSentences = mergeQuotedDialogue(rawSentences); + + let currentBlock = ''; + + for (const sentence of mergedSentences) { + const trimmedSentence = sentence.trim(); + const sentenceParts = + trimmedSentence.length > MAX_BLOCK_LENGTH + ? splitOversizedText(trimmedSentence, MAX_BLOCK_LENGTH) + : [trimmedSentence]; + + for (const sentencePart of sentenceParts) { + if (currentBlock && (currentBlock.length + sentencePart.length + 1) > MAX_BLOCK_LENGTH) { + blocks.push(currentBlock.trim()); + currentBlock = sentencePart; + } else { + currentBlock = currentBlock + ? `${currentBlock} ${sentencePart}` + : sentencePart; + } } } - - sentenceMapping.push({ processedIndex, rawIndices }); + + if (currentBlock) { + blocks.push(currentBlock.trim()); + } } - - return { - processedSentences, - rawSentences, - sentenceMapping - }; -}; + + return blocks; +}; + +/** + * Normalizes text for single-shot TTS generation (e.g., a whole PDF page). + * Uses the same logic as `splitTextToTtsBlocks`, but returns a single string. + * + * @param {string} text - The text to process + * @returns {string} Normalized text + */ +export const normalizeTextForTts = (text: string): string => + splitTextToTtsBlocks(text).join(' '); + // Helper functions to merge quoted dialogue across sentences const countDoubleQuotes = (s: string): number => { const matches = s.match(/["“”]/g); @@ -216,4 +305,4 @@ const mergeQuotedDialogue = (rawSentences: string[]): string[] => { } return result; -}; \ No newline at end of file +}; diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.spec.ts index 4bb5485..cb7164b 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.spec.ts @@ -1,88 +1,188 @@ import { test, expect } from '@playwright/test'; import { preprocessSentenceForAudio, - splitIntoSentences, - processTextWithMapping + splitTextToTtsBlocks, + splitTextToTtsBlocksEPUB, + normalizeTextForTts, + MAX_BLOCK_LENGTH } from '../../src/lib/nlp'; +const PDF_MAX_BLOCK_LENGTH = MAX_BLOCK_LENGTH * 2; // splitTextToTtsBlocks can overflow to reach punctuation + +const expectNormalizedBlocks = (blocks: string[], maxLen = Number.POSITIVE_INFINITY) => { + for (const block of blocks) { + expect(block.trim().length).toBeGreaterThan(0); + expect(block.length).toBeLessThanOrEqual(maxLen); + expect(block).not.toMatch(/\n/); + expect(block).not.toMatch(/\s{2,}/); + } +}; + test.describe('preprocessSentenceForAudio', () => { - test('removes URLs', () => { - const input = 'Check out https://example.com/page for more info'; - const expected = 'Check out - (link to example.com) - for more info'; - expect(preprocessSentenceForAudio(input)).toBe(expected); - }); + test('normalizes common extraction artifacts', () => { + const cases: Array<{ input: string; expected: string }> = [ + { + input: 'Check out https://example.com/page for more info', + expected: 'Check out - (link to example.com) - for more info', + }, + { + input: 'This is a hyp- henated word', + expected: 'This is a hyphenated word', + }, + { + input: 'This is *bold* text', + expected: 'This is bold text', + }, + { + input: 'Multiple spaces', + expected: 'Multiple spaces', + }, + ]; - test('removes hyphenation', () => { - const input = 'This is a hyp- henated word'; - const expected = 'This is a hyphenated word'; - expect(preprocessSentenceForAudio(input)).toBe(expected); - }); - - test('removes asterisks', () => { - const input = 'This is *bold* text'; - const expected = 'This is bold text'; - expect(preprocessSentenceForAudio(input)).toBe(expected); - }); - - test('collapses whitespace', () => { - const input = 'Multiple spaces'; - const expected = 'Multiple spaces'; - expect(preprocessSentenceForAudio(input)).toBe(expected); + for (const { input, expected } of cases) { + expect(preprocessSentenceForAudio(input)).toBe(expected); + } }); }); -test.describe('splitIntoSentences', () => { - test('groups short sentences into single block', () => { - const input = 'First sentence. Second sentence.'; - const result = splitIntoSentences(input); - expect(result).toHaveLength(1); - expect(result[0]).toBe('First sentence. Second sentence.'); +test.describe('splitTextToTtsBlocks (PDF-oriented)', () => { + test('returns [] for empty input', () => { + expect(splitTextToTtsBlocks('')).toEqual([]); + expect(splitTextToTtsBlocks(' ')).toEqual([]); + expect(splitTextToTtsBlocks('\n\n')).toEqual([]); }); - test('merges quoted dialogue (double quotes)', () => { - const input = 'He said, "This should be one block." and walked away.'; - const result = splitIntoSentences(input); - expect(result).toHaveLength(1); - expect(result[0]).toBe('He said, "This should be one block." and walked away.'); - }); - - test('merges quoted dialogue (curly quotes)', () => { - const input = 'She replied, “This also should be merged.” then smiled.'; - const result = splitIntoSentences(input); - expect(result).toHaveLength(1); - expect(result[0]).toBe('She replied, “This also should be merged.” then smiled.'); - }); + test('does not treat single newlines as paragraph boundaries', () => { + const input = + 'The first line ends with a comma,\n' + + 'but the sentence continues on the next line and ends here.\n' + + 'And this is the second sentence.'; + const result = splitTextToTtsBlocks(input); + + expect(result).toHaveLength(1); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + expect(result[0]).toBe( + 'The first line ends with a comma, but the sentence continues on the next line and ends here. And this is the second sentence.' + ); + }); + + test('treats blank lines (double newlines) as paragraph boundaries', () => { + const input = 'First paragraph.\n\nSecond paragraph.'; + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThanOrEqual(2); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + }); + + test('repairs missing whitespace between sentences (common PDF artifact)', () => { + const input = 'This ends.Here starts.'; + const normalized = normalizeTextForTts(input); + expect(normalized).toContain('ends. Here'); + }); + + test('does not break decimals when repairing sentence boundaries', () => { + const input = 'Pi is 3.14.Next sentence.'; + const normalized = normalizeTextForTts(input); + expect(normalized).toContain('3.14'); + }); + + test('enforces max block length on long content', () => { + const sentence = `${'A'.repeat(100)}.`; // 101 chars + const input = Array(8).fill(sentence).join(' '); // guaranteed to exceed MAX_BLOCK_LENGTH + const result = splitTextToTtsBlocks(input); - test('respects max block length for long text', () => { - // MAX_BLOCK_LENGTH is 450 in nlp.ts - // We construct distinct sentences. - // If we make sentences short enough individually but long enough combined, - // they should be grouped until the limit is reached. - - const sentence = 'A'.repeat(100) + '.'; // 101 chars - // 4 sentences = 404 chars + 3 spaces = 407 chars (< 450). Should fit in one block. - // 5 sentences = 505 chars + 4 spaces = 509 chars (> 450). Should split. - - const input = Array(5).fill(sentence).join(' '); - const result = splitIntoSentences(input); - expect(result.length).toBeGreaterThan(1); - // The first block should contain as many as possible - expect(result[0].length).toBeLessThanOrEqual(450); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); + + test('splits oversized content with no punctuation', () => { + const input = Array(1200).fill('word').join(' '); + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); + + test('splits extremely long unbroken tokens', () => { + const input = 'A'.repeat(1200); + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); + + test('prefers sentence punctuation when chunking long PDF-like text', () => { + const sentences = Array.from( + { length: 80 }, + (_, i) => `Sentence ${i} has filler words to vary length slightly number ${i}.` + ); + const input = sentences.join(''); // no whitespace after '.' between sentences + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + + // When sentence punctuation exists, blocks should usually end at punctuation/closers. + // This guards against regressions where we cut mid-word/mid-sentence too often. + for (const block of result) { + expect(block).toMatch(/[.!?]["'”’)\]]*$/); + } + }); + + test('allows a long sentence to extend to its ending punctuation', () => { + // Create a single sentence that exceeds MAX_BLOCK_LENGTH, but ends with a period + // within the forward-search overflow window. + const input = `${'word '.repeat(110)}end. Next.`; + const result = splitTextToTtsBlocks(input); + + expect(result.length).toBeGreaterThan(1); + // This case is specifically asserting we may exceed MAX_BLOCK_LENGTH to reach punctuation, + // but should still remain bounded by the overflow policy. + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + expect(result[0]).toMatch(/end\.$/); + }); + + test('merges multi-sentence quoted dialogue', () => { + const input = 'He said, "First. Second." Then left.'; + const result = splitTextToTtsBlocks(input); + + expect(result).toHaveLength(1); + expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH); + expect(result[0]).toContain('"First. Second."'); }); }); -test.describe('processTextWithMapping', () => { - test('maps raw sentences to processed ones', () => { - const text = 'First (1). Second (2).'; - const { processedSentences, rawSentences, sentenceMapping } = processTextWithMapping(text); +test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { + test('returns [] for empty input', () => { + expect(splitTextToTtsBlocksEPUB('')).toEqual([]); + expect(splitTextToTtsBlocksEPUB(' ')).toEqual([]); + expect(splitTextToTtsBlocksEPUB('\n\n')).toEqual([]); + }); - expect(processedSentences.length).toBeGreaterThan(0); - expect(rawSentences.length).toBeGreaterThan(0); - expect(sentenceMapping).toHaveLength(processedSentences.length); - - // Check structure of mapping - expect(sentenceMapping[0]).toHaveProperty('processedIndex'); - expect(sentenceMapping[0]).toHaveProperty('rawIndices'); + test('treats single newlines as paragraph boundaries', () => { + const input = 'One.\nTwo.'; + const result = splitTextToTtsBlocksEPUB(input); + expect(result).toHaveLength(2); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + expect(result[0]).toBe('One.'); + expect(result[1]).toBe('Two.'); + }); + + test('splits oversized sentences to keep blocks bounded', () => { + const input = Array(1200).fill('word').join(' '); // no punctuation; guaranteed to exceed MAX_BLOCK_LENGTH + const result = splitTextToTtsBlocksEPUB(input); + + expect(result.length).toBeGreaterThan(1); + expectNormalizedBlocks(result, MAX_BLOCK_LENGTH); + }); +}); + +test.describe('normalizeTextForTts', () => { + test('returns a single normalized string without newlines', () => { + const input = 'Hello.\nWorld.\n\nNext paragraph.'; + const normalized = normalizeTextForTts(input); + expect(normalized).not.toMatch(/\n/); + expect(normalized).not.toMatch(/\s{2,}/); + expect(normalized.length).toBeGreaterThan(0); }); });