Merge pull request #75 from richardr1126/v1.2.1

* **Bug Fixes**
  * More reliable PDF sentence and word highlighting during layout changes and rapid updates, preventing stale or overlapping highlights.

* **Enhancements**
  * Revamped text normalization and TTS block splitting for better handling of long/complex content, EPUB texts, dialogue/quotes, and PDF artifacts.

* **Tests**
  * Expanded unit tests covering block-splitting, EPUB behavior, oversized inputs, and boundary cases.

* **Chores**
  * Package version bumped; test runner config updated to ignore unit tests for select browser projects.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "openreader-webui",
-  "version": "v1.2.0",
+  "version": "v1.2.1",
   "private": true,
   "scripts": {
     "dev": "next dev --turbopack -p 3003",

playwright.config.ts

@@ -44,6 +44,7 @@ export default defineConfig({
 
     {
       name: 'firefox',
+      testIgnore: '**/unit/**',
       use: {
         ...devices['Desktop Firefox'],
         extraHTTPHeaders: { 'x-openreader-test-namespace': 'firefox' },
@@ -52,6 +53,7 @@ export default defineConfig({
 
     {
       name: 'webkit',
+      testIgnore: '**/unit/**',
       use: {
         ...devices['Desktop Safari'],
         extraHTTPHeaders: { 'x-openreader-test-namespace': 'webkit' },

src/components/PDFViewer.tsx

@@ -24,6 +24,12 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
   const containerRef = useRef<HTMLDivElement>(null);
   const scaleRef = useRef<number>(1);
   const { containerWidth } = usePDFResize(containerRef);
+  const sentenceHighlightSeqRef = useRef(0);
+  const wordHighlightSeqRef = useRef(0);
+  const sentenceHighlightTimeoutsRef = useRef<ReturnType<typeof setTimeout>[]>([]);
+  const wordHighlightTimeoutsRef = useRef<ReturnType<typeof setTimeout>[]>([]);
+  const lastSentenceLayoutKeyRef = useRef<string>('');
+  const lastWordLayoutKeyRef = useRef<string>('');
 
   // Config context
   const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig();
@@ -49,6 +55,28 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
     currDocPage,
   } = usePDF();
 
+  const layoutKey = `${zoomLevel}:${containerWidth}:${viewType}:${currDocPage}`;
+
+  const clearSentenceHighlightTimeouts = useCallback(() => {
+    for (const t of sentenceHighlightTimeoutsRef.current) clearTimeout(t);
+    sentenceHighlightTimeoutsRef.current = [];
+  }, []);
+
+  const clearWordHighlightTimeouts = useCallback(() => {
+    for (const t of wordHighlightTimeoutsRef.current) clearTimeout(t);
+    wordHighlightTimeoutsRef.current = [];
+  }, []);
+
+  const scheduleSentenceTimeout = useCallback((fn: () => void, ms: number) => {
+    const t = setTimeout(fn, ms);
+    sentenceHighlightTimeoutsRef.current.push(t);
+  }, []);
+
+  const scheduleWordTimeout = useCallback((fn: () => void, ms: number) => {
+    const t = setTimeout(fn, ms);
+    wordHighlightTimeoutsRef.current.push(t);
+  }, []);
+
   useEffect(() => {
     /*
      * Handles highlighting the current sentence being read by TTS.
@@ -66,33 +94,71 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
       return;
     }
 
-    const highlightTimeout = setTimeout(() => {
-      if (containerRef.current) {
-        highlightPattern(currDocText, currentSentence || '', containerRef as RefObject<HTMLDivElement>);
+    clearSentenceHighlightTimeouts();
+
+    if (!currentSentence) {
+      // Cancel any in-flight retry loops and ensure stale highlights don't remain
+      // when the current sentence becomes null/undefined.
+      sentenceHighlightSeqRef.current += 1;
+      clearHighlights();
+      return;
+    }
+
+    const seq = ++sentenceHighlightSeqRef.current;
+    const isLayoutChange = layoutKey !== lastSentenceLayoutKeyRef.current;
+    lastSentenceLayoutKeyRef.current = layoutKey;
+
+    if (isLayoutChange) {
+      clearHighlights();
+    }
+
+    const tryApply = (attempt: number) => {
+      if (seq !== sentenceHighlightSeqRef.current) return;
+      const container = containerRef.current;
+      if (!container) return;
+
+      const spans = container.querySelectorAll('.react-pdf__Page__textContent span');
+      if (!spans.length) {
+        if (attempt < 10) scheduleSentenceTimeout(() => tryApply(attempt + 1), 75);
+        return;
       }
-    }, 200);
+
+      highlightPattern(currDocText, currentSentence, containerRef as RefObject<HTMLDivElement>);
+    };
+
+    scheduleSentenceTimeout(() => tryApply(0), 200);
 
     return () => {
-      clearTimeout(highlightTimeout);
+      clearSentenceHighlightTimeouts();
       clearHighlights();
     };
-  }, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]);
+  }, [
+    currDocText,
+    currentSentence,
+    highlightPattern,
+    clearHighlights,
+    pdfHighlightEnabled,
+    layoutKey,
+    clearSentenceHighlightTimeouts,
+    scheduleSentenceTimeout
+  ]);
 
   // Word-level highlight layered on top of the block highlight
   useEffect(() => {
+    clearWordHighlightTimeouts();
+
     if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) {
       clearWordHighlights();
       return;
     }
 
-    if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) {
+    if (!currentSentence || currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) {
       clearWordHighlights();
       return;
     }
 
     const wordEntry =
-      currentSentenceAlignment &&
-      currentWordIndex < currentSentenceAlignment.words.length
+      currentSentenceAlignment && currentWordIndex < currentSentenceAlignment.words.length
         ? currentSentenceAlignment.words[currentWordIndex]
         : undefined;
     const wordText = wordEntry?.text || null;
@@ -102,12 +168,44 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
       return;
     }
 
-    highlightWordIndex(
-      currentSentenceAlignment,
-      currentWordIndex,
-      currentSentence || '',
-      containerRef as RefObject<HTMLDivElement>
-    );
+    const seq = ++wordHighlightSeqRef.current;
+    const isLayoutChange = layoutKey !== lastWordLayoutKeyRef.current;
+    lastWordLayoutKeyRef.current = layoutKey;
+
+    const tryApplyWord = (attempt: number) => {
+      if (seq !== wordHighlightSeqRef.current) return;
+      const container = containerRef.current;
+      if (!container) return;
+
+      highlightWordIndex(
+        currentSentenceAlignment,
+        currentWordIndex,
+        currentSentence || '',
+        containerRef as RefObject<HTMLDivElement>
+      );
+
+      if (isLayoutChange) {
+        // If we don't see a word overlay yet, the sentence highlight worker may not
+        // have produced `lastSentenceTokenWindow` (or the text layer isn't ready).
+        const overlayCount = container.querySelectorAll('.pdf-word-highlight-overlay').length;
+        if (overlayCount === 0 && attempt < 12) {
+          scheduleWordTimeout(() => tryApplyWord(attempt + 1), 75);
+        }
+      }
+    };
+
+    const cleanup = () => {
+      clearWordHighlightTimeouts();
+    };
+
+    if (isLayoutChange) {
+      clearWordHighlights();
+      scheduleWordTimeout(() => tryApplyWord(0), 250);
+      return cleanup;
+    }
+
+    tryApplyWord(0);
+    return cleanup;
   }, [
     currentWordIndex,
     currentSentence,
@@ -115,7 +213,10 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
     pdfHighlightEnabled,
     pdfWordHighlightEnabled,
     clearWordHighlights,
-    highlightWordIndex
+    highlightWordIndex,
+    layoutKey,
+    clearWordHighlightTimeouts,
+    scheduleWordTimeout
   ]);
 
   // Add page dimensions state

src/contexts/PDFContext.tsx

@@ -30,7 +30,7 @@ import type { PDFDocumentProxy } from 'pdfjs-dist';
 import { getPdfDocument } from '@/lib/dexie';
 import { useTTS } from '@/contexts/TTSContext';
 import { useConfig } from '@/contexts/ConfigContext';
-import { processTextToSentences } from '@/lib/nlp';
+import { normalizeTextForTts } from '@/lib/nlp';
 import { withRetry, getAudiobookStatus, generateTTS, createAudiobookChapter } from '@/lib/client';
 import {
   extractTextFromPDF,
@@ -312,9 +312,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
         });
         const trimmedText = rawText.trim();
         if (trimmedText) {
-          const processedText = smartSentenceSplitting
-            ? processTextToSentences(trimmedText).join(' ')
-            : trimmedText;
+          const processedText = smartSentenceSplitting ? normalizeTextForTts(trimmedText) : trimmedText;
 
           textPerPage.push(processedText);
           totalLength += processedText.length;
@@ -532,7 +530,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
       }
 
       const textForTTS = smartSentenceSplitting
-        ? processTextToSentences(trimmedText).join(' ')
+        ? normalizeTextForTts(trimmedText)
         : trimmedText;
 
       // Use logical chapter numbering (index + 1) to match original generation titles

src/contexts/TTSContext.tsx

@@ -37,7 +37,7 @@ import { useAudioContext } from '@/hooks/audio/useAudioContext';
 import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/dexie';
 import { useBackgroundState } from '@/hooks/audio/useBackgroundState';
 import { withRetry, generateTTS, alignAudio } from '@/lib/client';
-import { preprocessSentenceForAudio, processTextToSentences } from '@/lib/nlp';
+import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/nlp';
 import { isKokoroModel } from '@/utils/voice';
 import type {
   TTSLocation,
@@ -379,14 +379,14 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
    * @param {string} text - The text to be processed
    * @returns {Promise<string[]>} Array of processed sentences
    */
-  const processTextToSentencesLocal = useCallback(async (text: string): Promise<string[]> => {
+  const splitTextToTtsBlocksLocal = useCallback(async (text: string): Promise<string[]> => {
     if (text.length < 1) {
       return [];
     }
 
     // Use the shared utility directly instead of making an API call
-    return processTextToSentences(text);
-  }, []);
+    return isEPUB ? splitTextToTtsBlocksEPUB(text) : splitTextToTtsBlocks(text);
+  }, [isEPUB]);
 
   /**
    * Stops the current audio playback and clears the active Howl instance
@@ -579,7 +579,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     abortAudio(true); // Clear pending requests since text is changing
     setIsProcessing(true); // Set processing state before text processing starts
 
-    processTextToSentencesLocal(workingText)
+    splitTextToTtsBlocksLocal(workingText)
       .then(newSentences => {
         if (newSentences.length === 0) {
           console.warn('No sentences found in text');
@@ -658,7 +658,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
           duration: 3000,
         });
       });
-  }, [isPlaying, handleBlankSection, abortAudio, processTextToSentencesLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]);
+  }, [isPlaying, handleBlankSection, abortAudio, splitTextToTtsBlocksLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]);
 
   /**
    * Toggles the playback state between playing and paused

src/lib/nlp.ts

@@ -7,7 +7,114 @@
 
 import nlp from 'compromise';
 
-const MAX_BLOCK_LENGTH = 450;
+export const MAX_BLOCK_LENGTH = 450;
+
+const splitOversizedText = (text: string, maxLen: number): string[] => {
+  const normalized = text.replace(/\s+/g, ' ').trim();
+  if (!normalized) return [];
+  if (normalized.length <= maxLen) return [normalized];
+
+  const parts: string[] = [];
+  const MAX_OVERFLOW = maxLen; // allow finishing the sentence up to +maxLen chars
+  const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']);
+  const BREAK_CHARS = new Set(['.', '!', '?']);
+  const SOFT_BREAK_CHARS = new Set([';', ':']);
+
+  const findPunctuationCut = (s: string, limit: number): number | null => {
+    for (let i = limit; i >= 0; i--) {
+      const ch = s[i];
+      if (!BREAK_CHARS.has(ch)) continue;
+
+      const prev = i > 0 ? s[i - 1] : '';
+      const next = i + 1 < s.length ? s[i + 1] : '';
+
+      // Avoid splitting inside decimals like 3.14
+      if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue;
+
+      let end = i + 1;
+      while (end < s.length && CLOSERS.has(s[end])) end++;
+      const after = end < s.length ? s[end] : '';
+
+      // Allow a boundary at end/whitespace, or common PDF artifact where
+      // the next sentence starts immediately with an uppercase letter.
+      if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end;
+    }
+    return null;
+  };
+
+  const findForwardPunctuationCut = (
+    s: string,
+    startIndex: number,
+    endIndex: number,
+    chars: Set<string>
+  ): number | null => {
+    const start = Math.max(0, startIndex);
+    const end = Math.min(endIndex, s.length - 1);
+    for (let i = start; i <= end; i++) {
+      const ch = s[i];
+      if (!chars.has(ch)) continue;
+
+      const prev = i > 0 ? s[i - 1] : '';
+      const next = i + 1 < s.length ? s[i + 1] : '';
+
+      if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue;
+
+      let cut = i + 1;
+      while (cut < s.length && CLOSERS.has(s[cut])) cut++;
+      const after = cut < s.length ? s[cut] : '';
+
+      if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return cut;
+    }
+    return null;
+  };
+
+  const findSoftPunctuationCut = (s: string, limit: number): number | null => {
+    for (let i = limit; i >= 0; i--) {
+      const ch = s[i];
+      if (!SOFT_BREAK_CHARS.has(ch)) continue;
+
+      let end = i + 1;
+      while (end < s.length && CLOSERS.has(s[end])) end++;
+      const after = end < s.length ? s[end] : '';
+      if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end;
+    }
+    return null;
+  };
+
+  let remaining = normalized;
+  while (remaining.length > maxLen) {
+    const backwardLimit = Math.min(maxLen, remaining.length - 1);
+    const forwardLimit = Math.min(maxLen + MAX_OVERFLOW, remaining.length - 1);
+
+    let cut =
+      findPunctuationCut(remaining, backwardLimit) ??
+      findForwardPunctuationCut(remaining, maxLen, forwardLimit, BREAK_CHARS) ??
+      findSoftPunctuationCut(remaining, backwardLimit) ??
+      findForwardPunctuationCut(remaining, maxLen, forwardLimit, SOFT_BREAK_CHARS) ??
+      remaining.lastIndexOf(' ', maxLen);
+
+    if (cut === 0 || cut === -1) {
+      // No whitespace or punctuation; hard-cut for extremely long tokens.
+      cut = maxLen;
+    }
+
+    const chunk = remaining.slice(0, cut).trim();
+    if (chunk) parts.push(chunk);
+    remaining = remaining.slice(cut).trim();
+  }
+
+  if (remaining) parts.push(remaining);
+  return parts;
+};
+
+const normalizeSentenceBoundariesForNlp = (text: string): string => {
+  // PDF extraction sometimes yields "...end.Next..." with no whitespace.
+  // Insert a space only when it looks like a sentence boundary (lower/digit before,
+  // uppercase after) to avoid breaking abbreviations like "U.S.A".
+  return text
+    .replace(/([a-z0-9])([.!?])(?=[A-Z])/g, '$1$2 ')
+    .replace(/([a-z0-9][.!?][\"”’)\]])(?=[A-Z])/g, '$1 ');
+};
 
 /**
  * Preprocesses text for audio generation by cleaning up various text artifacts
@@ -31,14 +138,18 @@ export const preprocessSentenceForAudio = (text: string): string => {
  * @param {string} text - The text to split into sentences
  * @returns {string[]} Array of sentence blocks
  */
-export const splitIntoSentences = (text: string): string[] => {
-  const paragraphs = text.split(/\n+/);
+export const splitTextToTtsBlocks = (text: string): string[] => {
+  // Treat double-newlines as paragraph boundaries; single newlines are usually
+  // just PDF line wrapping and should not force sentence/block boundaries.
+  const paragraphs = text.split(/\n{2,}/);
   const blocks: string[] = [];
 
   for (const paragraph of paragraphs) {
     if (!paragraph.trim()) continue;
 
-    const cleanedText = preprocessSentenceForAudio(paragraph);
+    const cleanedText = normalizeSentenceBoundariesForNlp(
+      preprocessSentenceForAudio(paragraph)
+    );
     const doc = nlp(cleanedText);
     const rawSentences = doc.sentences().out('array') as string[];
     
@@ -49,14 +160,17 @@ export const splitIntoSentences = (text: string): string[] => {
 
     for (const sentence of mergedSentences) {
       const trimmedSentence = sentence.trim();
+      const sentenceParts = splitOversizedText(trimmedSentence, MAX_BLOCK_LENGTH);
 
-      if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
-        blocks.push(currentBlock.trim());
-        currentBlock = trimmedSentence;
-      } else {
-        currentBlock = currentBlock 
-          ? `${currentBlock} ${trimmedSentence}`
-          : trimmedSentence;
+      for (const sentencePart of sentenceParts) {
+        if (currentBlock && (currentBlock.length + sentencePart.length + 1) > MAX_BLOCK_LENGTH) {
+          blocks.push(currentBlock.trim());
+          currentBlock = sentencePart;
+        } else {
+          currentBlock = currentBlock 
+            ? `${currentBlock} ${sentencePart}`
+            : sentencePart;
+        }
       }
     }
 
@@ -69,86 +183,61 @@ export const splitIntoSentences = (text: string): string[] => {
 };
 
 /**
- * Main sentence processing function that handles both short and long texts
- * 
- * @param {string} text - The text to process
- * @returns {string[]} Array of processed sentences/blocks
+ * EPUB block splitting used where we want the produced sentences
+ * to closely match the original DOM text (for exact-match highlighting).
  */
-export const processTextToSentences = (text: string): string[] => {
-  if (!text || text.length < 1) {
-    return [];
-  }
+export const splitTextToTtsBlocksEPUB = (text: string): string[] => {
+  const paragraphs = text.split(/\n+/);
+  const blocks: string[] = [];
 
-  // Always use the full splitting logic so we consistently respect
-  // sentence boundaries and quoted dialogue, even for shorter texts.
-  return splitIntoSentences(text);
-};
+  for (const paragraph of paragraphs) {
+    if (!paragraph.trim()) continue;
 
-/**
- * Gets raw sentences from text without preprocessing or grouping
- * This is useful for text matching and highlighting
- * 
- * @param {string} text - The text to extract sentences from
- * @returns {string[]} Array of raw sentences
- */
-export const getRawSentences = (text: string): string[] => {
-  if (!text || text.length < 1) {
-    return [];
-  }
-  
-  return nlp(text).sentences().out('array') as string[];
-};
+    const cleanedText = preprocessSentenceForAudio(paragraph);
+    const doc = nlp(cleanedText);
+    const rawSentences = doc.sentences().out('array') as string[];
 
-/**
- * Enhanced sentence processing that returns both processed sentences and raw sentences
- * This allows for better mapping between the two for click-to-highlight functionality
- * 
- * @param {string} text - The text to process
- * @returns {Object} Object containing processed sentences and raw sentences with mapping
- */
-export const processTextWithMapping = (text: string): {
-  processedSentences: string[];
-  rawSentences: string[];
-  sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }>;
-} => {
-  const rawSentences = getRawSentences(text);
-  const processedSentences = processTextToSentences(text);
-  
-  // Create a mapping between processed sentences and raw sentences
-  const sentenceMapping: Array<{ processedIndex: number; rawIndices: number[] }> = [];
-  
-  // For simple mapping, we'll track which raw sentences contributed to each processed sentence
-  let rawIndex = 0;
-  
-  for (let processedIndex = 0; processedIndex < processedSentences.length; processedIndex++) {
-    const processedSentence = processedSentences[processedIndex];
-    const rawIndices: number[] = [];
-    
-    // Find which raw sentences are contained in this processed sentence
-    const remainingText = processedSentence;
-    
-    while (rawIndex < rawSentences.length && remainingText.length > 0) {
-      const rawSentence = rawSentences[rawIndex];
-      const cleanedRawSentence = preprocessSentenceForAudio(rawSentence);
-      
-      if (remainingText.includes(cleanedRawSentence) || cleanedRawSentence.includes(remainingText)) {
-        rawIndices.push(rawIndex);
-        rawIndex++;
-        break;
-      } else {
-        rawIndex++;
+    const mergedSentences = mergeQuotedDialogue(rawSentences);
+
+    let currentBlock = '';
+
+    for (const sentence of mergedSentences) {
+      const trimmedSentence = sentence.trim();
+      const sentenceParts =
+        trimmedSentence.length > MAX_BLOCK_LENGTH
+          ? splitOversizedText(trimmedSentence, MAX_BLOCK_LENGTH)
+          : [trimmedSentence];
+
+      for (const sentencePart of sentenceParts) {
+        if (currentBlock && (currentBlock.length + sentencePart.length + 1) > MAX_BLOCK_LENGTH) {
+          blocks.push(currentBlock.trim());
+          currentBlock = sentencePart;
+        } else {
+          currentBlock = currentBlock
+            ? `${currentBlock} ${sentencePart}`
+            : sentencePart;
+        }
       }
     }
-    
-    sentenceMapping.push({ processedIndex, rawIndices });
+
+    if (currentBlock) {
+      blocks.push(currentBlock.trim());
+    }
   }
-  
-  return {
-    processedSentences,
-    rawSentences,
-    sentenceMapping
-  };
-}; 
+
+  return blocks;
+};
+
+/**
+ * Normalizes text for single-shot TTS generation (e.g., a whole PDF page).
+ * Uses the same logic as `splitTextToTtsBlocks`, but returns a single string.
+ * 
+ * @param {string} text - The text to process
+ * @returns {string} Normalized text
+ */
+export const normalizeTextForTts = (text: string): string =>
+  splitTextToTtsBlocks(text).join(' ');
+
 // Helper functions to merge quoted dialogue across sentences
 const countDoubleQuotes = (s: string): number => {
   const matches = s.match(/["“”]/g);
@@ -216,4 +305,4 @@ const mergeQuotedDialogue = (rawSentences: string[]): string[] => {
   }
 
   return result;
-};
+};

tests/unit/nlp.spec.ts

@@ -1,88 +1,188 @@
 import { test, expect } from '@playwright/test';
 import { 
   preprocessSentenceForAudio, 
-  splitIntoSentences, 
-  processTextWithMapping 
+  splitTextToTtsBlocks, 
+  splitTextToTtsBlocksEPUB,
+  normalizeTextForTts,
+  MAX_BLOCK_LENGTH
 } from '../../src/lib/nlp';
 
+const PDF_MAX_BLOCK_LENGTH = MAX_BLOCK_LENGTH * 2; // splitTextToTtsBlocks can overflow to reach punctuation
+
+const expectNormalizedBlocks = (blocks: string[], maxLen = Number.POSITIVE_INFINITY) => {
+  for (const block of blocks) {
+    expect(block.trim().length).toBeGreaterThan(0);
+    expect(block.length).toBeLessThanOrEqual(maxLen);
+    expect(block).not.toMatch(/\n/);
+    expect(block).not.toMatch(/\s{2,}/);
+  }
+};
+
 test.describe('preprocessSentenceForAudio', () => {
-  test('removes URLs', () => {
-    const input = 'Check out https://example.com/page for more info';
-    const expected = 'Check out - (link to example.com) - for more info';
-    expect(preprocessSentenceForAudio(input)).toBe(expected);
-  });
+  test('normalizes common extraction artifacts', () => {
+    const cases: Array<{ input: string; expected: string }> = [
+      {
+        input: 'Check out https://example.com/page for more info',
+        expected: 'Check out - (link to example.com) - for more info',
+      },
+      {
+        input: 'This is a hyp- henated word',
+        expected: 'This is a hyphenated word',
+      },
+      {
+        input: 'This is *bold* text',
+        expected: 'This is bold text',
+      },
+      {
+        input: 'Multiple    spaces',
+        expected: 'Multiple spaces',
+      },
+    ];
 
-  test('removes hyphenation', () => {
-    const input = 'This is a hyp- henated word';
-    const expected = 'This is a hyphenated word';
-    expect(preprocessSentenceForAudio(input)).toBe(expected);
-  });
-
-  test('removes asterisks', () => {
-    const input = 'This is *bold* text';
-    const expected = 'This is bold text';
-    expect(preprocessSentenceForAudio(input)).toBe(expected);
-  });
-
-  test('collapses whitespace', () => {
-    const input = 'Multiple    spaces';
-    const expected = 'Multiple spaces';
-    expect(preprocessSentenceForAudio(input)).toBe(expected);
+    for (const { input, expected } of cases) {
+      expect(preprocessSentenceForAudio(input)).toBe(expected);
+    }
   });
 });
 
-test.describe('splitIntoSentences', () => {
-  test('groups short sentences into single block', () => {
-    const input = 'First sentence. Second sentence.';
-    const result = splitIntoSentences(input);
-    expect(result).toHaveLength(1);
-    expect(result[0]).toBe('First sentence. Second sentence.');
+test.describe('splitTextToTtsBlocks (PDF-oriented)', () => {
+  test('returns [] for empty input', () => {
+    expect(splitTextToTtsBlocks('')).toEqual([]);
+    expect(splitTextToTtsBlocks('   ')).toEqual([]);
+    expect(splitTextToTtsBlocks('\n\n')).toEqual([]);
   });
 
-  test('merges quoted dialogue (double quotes)', () => {
-    const input = 'He said, "This should be one block." and walked away.';
-    const result = splitIntoSentences(input);
-    expect(result).toHaveLength(1);
-    expect(result[0]).toBe('He said, "This should be one block." and walked away.');
-  });
-  
-  test('merges quoted dialogue (curly quotes)', () => {
-      const input = 'She replied, “This also should be merged.” then smiled.';
-      const result = splitIntoSentences(input);
-      expect(result).toHaveLength(1);
-      expect(result[0]).toBe('She replied, “This also should be merged.” then smiled.');
-    });
+  test('does not treat single newlines as paragraph boundaries', () => {
+    const input =
+      'The first line ends with a comma,\n' +
+      'but the sentence continues on the next line and ends here.\n' +
+      'And this is the second sentence.';
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result).toHaveLength(1);
+    expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH);
+    expect(result[0]).toBe(
+      'The first line ends with a comma, but the sentence continues on the next line and ends here. And this is the second sentence.'
+    );
+  });
+
+  test('treats blank lines (double newlines) as paragraph boundaries', () => {
+    const input = 'First paragraph.\n\nSecond paragraph.';
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result.length).toBeGreaterThanOrEqual(2);
+    expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH);
+  });
+
+  test('repairs missing whitespace between sentences (common PDF artifact)', () => {
+    const input = 'This ends.Here starts.';
+    const normalized = normalizeTextForTts(input);
+    expect(normalized).toContain('ends. Here');
+  });
+
+  test('does not break decimals when repairing sentence boundaries', () => {
+    const input = 'Pi is 3.14.Next sentence.';
+    const normalized = normalizeTextForTts(input);
+    expect(normalized).toContain('3.14');
+  });
+
+  test('enforces max block length on long content', () => {
+    const sentence = `${'A'.repeat(100)}.`; // 101 chars
+    const input = Array(8).fill(sentence).join(' '); // guaranteed to exceed MAX_BLOCK_LENGTH
+    const result = splitTextToTtsBlocks(input);
 
-  test('respects max block length for long text', () => {
-    // MAX_BLOCK_LENGTH is 450 in nlp.ts
-    // We construct distinct sentences. 
-    // If we make sentences short enough individually but long enough combined, 
-    // they should be grouped until the limit is reached.
-    
-    const sentence = 'A'.repeat(100) + '.'; // 101 chars
-    // 4 sentences = 404 chars + 3 spaces = 407 chars (< 450). Should fit in one block.
-    // 5 sentences = 505 chars + 4 spaces = 509 chars (> 450). Should split.
-    
-    const input = Array(5).fill(sentence).join(' ');
-    const result = splitIntoSentences(input);
-    
     expect(result.length).toBeGreaterThan(1);
-    // The first block should contain as many as possible
-    expect(result[0].length).toBeLessThanOrEqual(450);
+    expectNormalizedBlocks(result, MAX_BLOCK_LENGTH);
+  });
+
+  test('splits oversized content with no punctuation', () => {
+    const input = Array(1200).fill('word').join(' ');
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result.length).toBeGreaterThan(1);
+    expectNormalizedBlocks(result, MAX_BLOCK_LENGTH);
+  });
+
+  test('splits extremely long unbroken tokens', () => {
+    const input = 'A'.repeat(1200);
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result.length).toBeGreaterThan(1);
+    expectNormalizedBlocks(result, MAX_BLOCK_LENGTH);
+  });
+
+  test('prefers sentence punctuation when chunking long PDF-like text', () => {
+    const sentences = Array.from(
+      { length: 80 },
+      (_, i) => `Sentence ${i} has filler words to vary length slightly number ${i}.`
+    );
+    const input = sentences.join(''); // no whitespace after '.' between sentences
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result.length).toBeGreaterThan(1);
+    expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH);
+
+    // When sentence punctuation exists, blocks should usually end at punctuation/closers.
+    // This guards against regressions where we cut mid-word/mid-sentence too often.
+    for (const block of result) {
+      expect(block).toMatch(/[.!?]["'”’)\]]*$/);
+    }
+  });
+
+  test('allows a long sentence to extend to its ending punctuation', () => {
+    // Create a single sentence that exceeds MAX_BLOCK_LENGTH, but ends with a period
+    // within the forward-search overflow window.
+    const input = `${'word '.repeat(110)}end. Next.`;
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result.length).toBeGreaterThan(1);
+    // This case is specifically asserting we may exceed MAX_BLOCK_LENGTH to reach punctuation,
+    // but should still remain bounded by the overflow policy.
+    expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH);
+    expect(result[0]).toMatch(/end\.$/);
+  });
+
+  test('merges multi-sentence quoted dialogue', () => {
+    const input = 'He said, "First. Second." Then left.';
+    const result = splitTextToTtsBlocks(input);
+
+    expect(result).toHaveLength(1);
+    expectNormalizedBlocks(result, PDF_MAX_BLOCK_LENGTH);
+    expect(result[0]).toContain('"First. Second."');
   });
 });
 
-test.describe('processTextWithMapping', () => {
-  test('maps raw sentences to processed ones', () => {
-    const text = 'First (1). Second (2).';
-    const { processedSentences, rawSentences, sentenceMapping } = processTextWithMapping(text);
+test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => {
+  test('returns [] for empty input', () => {
+    expect(splitTextToTtsBlocksEPUB('')).toEqual([]);
+    expect(splitTextToTtsBlocksEPUB('   ')).toEqual([]);
+    expect(splitTextToTtsBlocksEPUB('\n\n')).toEqual([]);
+  });
 
-    expect(processedSentences.length).toBeGreaterThan(0);
-    expect(rawSentences.length).toBeGreaterThan(0);
-    expect(sentenceMapping).toHaveLength(processedSentences.length);
-    
-    // Check structure of mapping
-    expect(sentenceMapping[0]).toHaveProperty('processedIndex');
-    expect(sentenceMapping[0]).toHaveProperty('rawIndices');
+  test('treats single newlines as paragraph boundaries', () => {
+    const input = 'One.\nTwo.';
+    const result = splitTextToTtsBlocksEPUB(input);
+    expect(result).toHaveLength(2);
+    expectNormalizedBlocks(result, MAX_BLOCK_LENGTH);
+    expect(result[0]).toBe('One.');
+    expect(result[1]).toBe('Two.');
+  });
+
+  test('splits oversized sentences to keep blocks bounded', () => {
+    const input = Array(1200).fill('word').join(' '); // no punctuation; guaranteed to exceed MAX_BLOCK_LENGTH
+    const result = splitTextToTtsBlocksEPUB(input);
+
+    expect(result.length).toBeGreaterThan(1);
+    expectNormalizedBlocks(result, MAX_BLOCK_LENGTH);
+  });
+});
+
+test.describe('normalizeTextForTts', () => {
+  test('returns a single normalized string without newlines', () => {
+    const input = 'Hello.\nWorld.\n\nNext paragraph.';
+    const normalized = normalizeTextForTts(input);
+    expect(normalized).not.toMatch(/\n/);
+    expect(normalized).not.toMatch(/\s{2,}/);
+    expect(normalized.length).toBeGreaterThan(0);
   });
 });