feat(tts): add word-by-word highlighting with whisper.cpp

- Introduce `/api/whisper` endpoint which uses `whisper.cpp` (via a `WHISPER_CPP_BIN` executable) and `ffmpeg` to generate word-level audio alignments from provided audio and text.
- Integrate word-level alignments into `TTSContext`, tracking the currently spoken word based on audio seek position and provided timestamps. Alignments are cached in-memory and fetched asynchronously.
- Add new configuration options (`pdfWordHighlightEnabled`, `epubWordHighlightEnabled`) to `ConfigContext` and `Dexie` for enabling/disabling the feature.
- Implement visual word highlighting in both `PDFViewer` and `EPUBViewer` by mapping TTS-aligned words to rendered text elements.
- Enhance `EPUBContext` and `PDFContext` with new `highlightWordIndex` and `clearWordHighlights` functions, utilizing fuzzy string matching (`cmpstr`) to robustly align spoken words with displayed text for accurate highlighting.
- Update `DocumentSettings` to include user-facing toggles for the new highlighting modes.

src/app/api/whisper/route.ts

@@ -0,0 +1,451 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createHash, randomUUID } from 'crypto';
+import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { spawn } from 'child_process';
+import type { TTSSentenceAlignment } from '@/types/tts';
+import { preprocessSentenceForAudio } from '@/lib/nlp';
+
+export const runtime = 'nodejs';
+
+interface WhisperRequestBody {
+  text: string;
+  audio: number[]; // raw bytes from Uint8Array
+  lang?: string;
+}
+
+interface WhisperAlignmentOptions {
+  engine?: 'whisper.cpp';
+  lang?: string;
+}
+
+interface WhisperWord {
+  start: number;
+  end: number;
+  word: string;
+}
+
+// Simple in-memory cache keyed by a hash of text+lang+audio length
+const alignmentCache = new Map<string, TTSSentenceAlignment[]>();
+
+// Model management using a fixed tiny.en GGML model stored under docstore/model
+const MODEL_NAME = 'ggml-tiny.en.bin';
+const MODEL_URL =
+  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin';
+const DOCSTORE_DIR = join(process.cwd(), 'docstore');
+const MODEL_DIR = join(DOCSTORE_DIR, 'model');
+const MODEL_PATH = join(MODEL_DIR, MODEL_NAME);
+const modelReadyPromises = new Map<string, Promise<void>>();
+
+async function ensureModelAvailable(): Promise<void> {
+  // Fast path: model already present
+  try {
+    await access(MODEL_PATH);
+    return;
+  } catch {
+    // fall through to download
+  }
+
+  const existing = modelReadyPromises.get(MODEL_PATH);
+  if (existing) return existing;
+
+  const promise = (async () => {
+    try {
+      await access(MODEL_PATH);
+      return;
+    } catch {
+      // still missing
+    }
+
+    await mkdir(MODEL_DIR, { recursive: true });
+
+    const res = await fetch(MODEL_URL);
+    if (!res.ok) {
+      throw new Error(
+        `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}`
+      );
+    }
+
+    const arrayBuffer = await res.arrayBuffer();
+    await writeFile(MODEL_PATH, Buffer.from(arrayBuffer));
+  })();
+
+  modelReadyPromises.set(MODEL_PATH, promise);
+  return promise;
+}
+
+async function runWhisperCpp(
+  wavPath: string,
+  opts: WhisperAlignmentOptions
+): Promise<WhisperWord[]> {
+  const binary = process.env.WHISPER_CPP_BIN;
+  if (!binary) {
+    throw new Error(
+      'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.'
+    );
+  }
+
+  await ensureModelAvailable();
+
+  return new Promise((resolve, reject) => {
+    const jsonBase = `${wavPath}.json_out`;
+    const jsonPath = `${jsonBase}.json`;
+    // Request full JSON output (including token-level details when available)
+    // and ask whisper-cli not to print anything to stdout.
+    const args = [
+      '-m',
+      MODEL_PATH,
+      '-f',
+      wavPath,
+      '-of',
+      jsonBase,
+      '-ojf',
+      '-np',
+    ];
+
+    if (opts.lang) {
+      args.push('-l', opts.lang);
+    }
+
+    const child = spawn(binary, args);
+
+    let stdout = '';
+    let stderr = '';
+
+    child.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+
+    child.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    child.on('error', (err) => {
+      reject(err);
+    });
+
+    child.on('close', (code) => {
+      if (code !== 0) {
+        console.error('whisper-cli exited with error', {
+          code,
+          wavPath,
+          stdout: stdout.slice(0, 500),
+          stderr: stderr.slice(0, 500),
+        });
+        return reject(
+          new Error(
+            `whisper.cpp exited with code ${code}: ${stderr || stdout}`
+          )
+        );
+      }
+
+      readFile(jsonPath, 'utf-8')
+        .then((content: string) => {
+          const words: WhisperWord[] = [];
+          const parsed = JSON.parse(content) as {
+            transcription?: Array<{
+              text?: string;
+              timestamps?: { from?: string; to?: string };
+              offsets?: { from?: number; to?: number };
+              tokens?: Array<{
+                text?: string;
+                timestamps?: { from?: string; to?: string };
+                offsets?: { from?: number; to?: number };
+              }>;
+            }>;
+          };
+
+          const transcription = parsed.transcription;
+
+          const parseTimecode = (value?: string): number | null => {
+            if (!value) return null;
+            const m = value.match(/(\d+):(\d+):(\d+),(\d+)/);
+            if (!m) return null;
+            const h = Number(m[1]);
+            const min = Number(m[2]);
+            const s = Number(m[3]);
+            const ms = Number(m[4]);
+            if (
+              Number.isNaN(h) ||
+              Number.isNaN(min) ||
+              Number.isNaN(s) ||
+              Number.isNaN(ms)
+            ) {
+              return null;
+            }
+            return h * 3600 + min * 60 + s + ms / 1000;
+          };
+
+          if (Array.isArray(transcription)) {
+            for (const seg of transcription) {
+              const segText = (seg.text || '').trim();
+              const segStartSecFromTs = parseTimecode(
+                seg.timestamps?.from
+              );
+              const segEndSecFromTs = parseTimecode(seg.timestamps?.to);
+              const segStartSecFromMs =
+                typeof seg.offsets?.from === 'number'
+                  ? seg.offsets.from / 1000
+                  : null;
+              const segEndSecFromMs =
+                typeof seg.offsets?.to === 'number'
+                  ? seg.offsets.to / 1000
+                  : null;
+
+              const segStartSec =
+                segStartSecFromTs ??
+                segStartSecFromMs ??
+                0;
+              const segEndSec =
+                segEndSecFromTs ??
+                segEndSecFromMs ??
+                segStartSec;
+
+              const tokens = Array.isArray(seg.tokens)
+                ? seg.tokens
+                : [];
+
+              if (tokens.length > 0) {
+                for (const token of tokens) {
+                  const rawText = token.text || '';
+                  const tokenText = rawText.trim();
+                  // Skip special markers like [_BEG_]
+                  if (!tokenText || /^\[.*\]$/.test(tokenText)) continue;
+
+                  const tokStartSecFromTs = parseTimecode(
+                    token.timestamps?.from
+                  );
+                  const tokEndSecFromTs = parseTimecode(
+                    token.timestamps?.to
+                  );
+                  const tokStartSecFromMs =
+                    typeof token.offsets?.from === 'number'
+                      ? token.offsets.from / 1000
+                      : null;
+                  const tokEndSecFromMs =
+                    typeof token.offsets?.to === 'number'
+                      ? token.offsets.to / 1000
+                      : null;
+
+                  const startSec =
+                    tokStartSecFromTs ??
+                    tokStartSecFromMs ??
+                    segStartSec;
+                  const endSec =
+                    tokEndSecFromTs ??
+                    tokEndSecFromMs ??
+                    segEndSec;
+
+                  words.push({
+                    word: tokenText,
+                    start: startSec,
+                    end: endSec,
+                  });
+                }
+              } else if (segText) {
+                // Fallback: no token list, approximate per-word timing within the segment
+                const segTokens = segText.split(/\s+/).filter(Boolean);
+                if (segTokens.length) {
+                  const totalDur = Math.max(segEndSec - segStartSec, 0);
+                  const step =
+                    segTokens.length > 0
+                      ? totalDur / segTokens.length
+                      : 0;
+                  segTokens.forEach((token, index) => {
+                    const wStart =
+                      step > 0
+                        ? segStartSec + step * index
+                        : segStartSec;
+                    const wEnd =
+                      step > 0
+                        ? index === segTokens.length - 1
+                          ? segEndSec
+                          : segStartSec + step * (index + 1)
+                        : segEndSec;
+                    words.push({
+                      word: token,
+                      start: wStart,
+                      end: wEnd,
+                    });
+                  });
+                }
+              }
+            }
+          }
+
+          resolve(words);
+        })
+        .catch((err: unknown) => {
+          reject(err);
+        });
+    });
+  });
+}
+
+function mapWordsToSentenceOffsets(
+  sentence: string,
+  words: WhisperWord[]
+): TTSSentenceAlignment {
+  const normalizedSentence = preprocessSentenceForAudio(sentence);
+  let cursor = 0;
+
+  const alignedWords = words.map((w) => {
+    const token = w.word.trim();
+    if (!token) {
+      return {
+        text: '',
+        startSec: w.start,
+        endSec: w.end,
+        charStart: cursor,
+        charEnd: cursor,
+      };
+    }
+
+    const idx = normalizedSentence
+      .toLowerCase()
+      .indexOf(token.toLowerCase(), cursor);
+
+    const start =
+      idx !== -1
+        ? idx
+        : cursor;
+    const end = start + token.length;
+
+    cursor = end;
+
+    return {
+      text: token,
+      startSec: w.start,
+      endSec: w.end,
+      charStart: start,
+      charEnd: end,
+    };
+  });
+
+  return {
+    sentence,
+    sentenceIndex: 0,
+    words: alignedWords.filter((w) => w.text.length > 0),
+  };
+}
+
+async function alignAudioWithText(
+  audioBuffer: ArrayBuffer,
+  text: string,
+  cacheKey?: string,
+  opts: WhisperAlignmentOptions = {}
+): Promise<TTSSentenceAlignment[]> {
+  if (!text.trim()) {
+    return [];
+  }
+
+  if (cacheKey && alignmentCache.has(cacheKey)) {
+    return alignmentCache.get(cacheKey)!;
+  }
+
+  const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-'));
+  const inputPath = join(tmpBase, `${randomUUID()}-input.bin`);
+  const wavPath = join(tmpBase, `${randomUUID()}-input.wav`);
+
+  try {
+    await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer)));
+
+    await new Promise<void>((resolve, reject) => {
+      const ffmpeg = spawn('ffmpeg', [
+        '-y',
+        '-i',
+        inputPath,
+        '-ar',
+        '16000',
+        '-ac',
+        '1',
+        wavPath,
+      ]);
+
+      let stderr = '';
+      ffmpeg.stderr.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      ffmpeg.on('error', (err) => {
+        reject(err);
+      });
+
+      ffmpeg.on('close', (code) => {
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(
+            new Error(`ffmpeg failed with code ${code}: ${stderr}`)
+          );
+        }
+      });
+    });
+
+    const words = await runWhisperCpp(wavPath, opts);
+    const alignment = mapWordsToSentenceOffsets(text, words);
+    const result: TTSSentenceAlignment[] = [alignment];
+
+    if (cacheKey) {
+      alignmentCache.set(cacheKey, result);
+    }
+
+    return result;
+  } finally {
+    try {
+      await rm(tmpBase, { recursive: true, force: true });
+    } catch {
+      // ignore
+    }
+  }
+}
+
+function makeCacheKey(input: WhisperRequestBody) {
+  const hash = createHash('sha256')
+    .update(
+      JSON.stringify({
+        text: input.text,
+        lang: input.lang || '',
+        audioLen: input.audio?.length || 0,
+      })
+    )
+    .digest('hex');
+  return hash;
+}
+
+export async function POST(req: NextRequest) {
+  try {
+    const body = (await req.json()) as WhisperRequestBody;
+    const { text, audio, lang } = body;
+
+    if (!text || !audio || !Array.isArray(audio)) {
+      return NextResponse.json(
+        { error: 'Missing text or audio in request body' },
+        { status: 400 }
+      );
+    }
+
+    const cacheKey = makeCacheKey(body);
+    const audioBuffer = new Uint8Array(audio).buffer;
+
+    const alignments: TTSSentenceAlignment[] = await alignAudioWithText(
+      audioBuffer,
+      text,
+      cacheKey,
+      { engine: 'whisper.cpp', lang }
+    );
+
+    return NextResponse.json({ alignments }, { status: 200 });
+  } catch (error) {
+    console.error('Error in whisper route:', error);
+    return NextResponse.json(
+      {
+        error: 'WHISPER_ALIGNMENT_FAILED',
+        message: 'Failed to compute word-level alignment',
+      },
+      { status: 500 }
+    );
+  }
+}
+

src/components/DocumentSettings.tsx

@@ -35,6 +35,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
     updateConfigKey,
     pdfHighlightEnabled,
     epubHighlightEnabled,
+    pdfWordHighlightEnabled,
+    epubWordHighlightEnabled,
   } = useConfig();
   const { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter } = useEPUB();
   const { createFullAudioBook: createPDFAudioBook, regenerateChapter: regeneratePDFChapter } = usePDF();
@@ -324,40 +326,82 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
                           </span>
                         </label>
                         <p className="text-sm text-muted pl-6">
-                          Merge sentences across page or section breaks for smoother TTS.
+                          Merge sentences across page or section breaks
                         </p>
                       </div>
                     )}
                     {!epub && !html && (
-                      <div className="space-y-1">
+                      <div className="space-y-2">
-                        <label className="flex items-center space-x-2">
+                        <div className="space-y-1">
-                          <input
+                          <label className="flex items-center space-x-2">
-                            type="checkbox"
+                            <input
-                            checked={pdfHighlightEnabled}
+                              type="checkbox"
-                            onChange={(e) => updateConfigKey('pdfHighlightEnabled', e.target.checked)}
+                              checked={pdfHighlightEnabled}
-                            className="form-checkbox h-4 w-4 text-accent rounded border-muted"
+                              onChange={(e) => updateConfigKey('pdfHighlightEnabled', e.target.checked)}
-                          />
+                              className="form-checkbox h-4 w-4 text-accent rounded border-muted"
-                          <span className="text-sm font-medium text-foreground">Highlight text during playback</span>
+                            />
-                        </label>
+                            <span className="text-sm font-medium text-foreground">Highlight text during playback</span>
-                        <p className="text-sm text-muted pl-6">
+                          </label>
-                          Show visual highlighting in the PDF viewer while TTS is reading.
+                          <p className="text-sm text-muted pl-6">
-                        </p>
+                            Visual text playback highlighting in the PDF viewer
+                          </p>
+                        </div>
+                        <div className="space-y-1 pl-6">
+                          <label className="flex items-center space-x-2">
+                            <input
+                              type="checkbox"
+                              checked={pdfWordHighlightEnabled && pdfHighlightEnabled}
+                              disabled={!pdfHighlightEnabled}
+                              onChange={(e) =>
+                                updateConfigKey('pdfWordHighlightEnabled', e.target.checked)
+                              }
+                              className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50"
+                            />
+                            <span className="text-sm font-medium text-foreground">
+                              Word-by-word
+                            </span>
+                          </label>
+                          <p className="text-sm text-muted pl-6">
+                            Highlight individual words using audio timestamps generated by whisper.cpp
+                          </p>
+                        </div>
                       </div>
                     )}
                     {epub && (
-                      <div className="space-y-1">
+                      <div className="space-y-2">
-                        <label className="flex items-center space-x-2">
+                        <div className="space-y-1">
-                          <input
+                          <label className="flex items-center space-x-2">
-                            type="checkbox"
+                            <input
-                            checked={epubHighlightEnabled}
+                              type="checkbox"
-                            onChange={(e) => updateConfigKey('epubHighlightEnabled', e.target.checked)}
+                              checked={epubHighlightEnabled}
-                            className="form-checkbox h-4 w-4 text-accent rounded border-muted"
+                              onChange={(e) => updateConfigKey('epubHighlightEnabled', e.target.checked)}
-                          />
+                              className="form-checkbox h-4 w-4 text-accent rounded border-muted"
-                          <span className="text-sm font-medium text-foreground">Highlight text during playback</span>
+                            />
-                        </label>
+                            <span className="text-sm font-medium text-foreground">Highlight text during playback</span>
-                        <p className="text-sm text-muted pl-6">
+                          </label>
-                          Show visual highlighting in the EPUB viewer while TTS is reading.
+                          <p className="text-sm text-muted pl-6">
-                        </p>
+                            Visual text playback highlighting in the EPUB viewer
+                          </p>
+                        </div>
+                        <div className="space-y-1 pl-6">
+                          <label className="flex items-center space-x-2">
+                            <input
+                              type="checkbox"
+                              checked={epubWordHighlightEnabled && epubHighlightEnabled}
+                              disabled={!epubHighlightEnabled}
+                              onChange={(e) =>
+                                updateConfigKey('epubWordHighlightEnabled', e.target.checked)
+                              }
+                              className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50"
+                            />
+                            <span className="text-sm font-medium text-foreground">
+                              Word-by-word
+                            </span>
+                          </label>
+                          <p className="text-sm text-muted pl-6">
+                            Highlight individual words using audio timestamps generated by whisper.cpp
+                          </p>
+                        </div>
                       </div>
                     )}
                     {epub && (

src/components/EPUBViewer.tsx

@@ -30,10 +30,18 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
     setRendition,
     extractPageText,
     highlightPattern,
-    clearHighlights
+    clearHighlights,
+    highlightWordIndex,
+    clearWordHighlights
   } = useEPUB();
-  const { registerLocationChangeHandler, pause, currentSentence } = useTTS();
+  const {
-  const { epubTheme } = useConfig();
+    registerLocationChangeHandler,
+    pause,
+    currentSentence,
+    currentSentenceAlignment,
+    currentWordIndex
+  } = useTTS();
+  const { epubTheme, epubHighlightEnabled, epubWordHighlightEnabled } = useConfig();
   const { updateTheme } = useEPUBTheme(epubTheme, renditionRef.current);
   const containerRef = useRef<HTMLDivElement>(null);
   const { isResizing, setIsResizing, dimensions } = useEPUBResize(containerRef);
@@ -70,6 +78,38 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
     }
   }, [currentSentence, highlightPattern, clearHighlights]);
 
+  // Word-level highlight layered on top of the block highlight
+  useEffect(() => {
+    if (!epubHighlightEnabled || !epubWordHighlightEnabled) {
+      clearWordHighlights();
+      return;
+    }
+
+    if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) {
+      clearWordHighlights();
+      return;
+    }
+
+    if (!currentSentenceAlignment) {
+      clearWordHighlights();
+      return;
+    }
+
+    highlightWordIndex(
+      currentSentenceAlignment,
+      currentWordIndex,
+      currentSentence || ''
+    );
+  }, [
+    currentWordIndex,
+    currentSentence,
+    currentSentenceAlignment,
+    epubHighlightEnabled,
+    epubWordHighlightEnabled,
+    clearWordHighlights,
+    highlightWordIndex
+  ]);
+
   if (!currDocData) {
     return <DocumentSkeleton />;
   }
@@ -95,4 +135,4 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
       </div>
     </div>
   );
-}
+}

src/components/PDFViewer.tsx

@@ -26,11 +26,13 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
   const { containerWidth } = usePDFResize(containerRef);
 
   // Config context
-  const { viewType, pdfHighlightEnabled } = useConfig();
+  const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig();
 
   // TTS context
   const {
     currentSentence,
+    currentWordIndex,
+    currentSentenceAlignment,
     skipToLocation,
   } = useTTS();
 
@@ -38,6 +40,8 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
   const {
     highlightPattern,
     clearHighlights,
+    clearWordHighlights,
+    highlightWordIndex,
     onDocumentLoadSuccess,
     currDocData,
     currDocPages,
@@ -74,6 +78,46 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
     };
   }, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]);
 
+  // Word-level highlight layered on top of the block highlight
+  useEffect(() => {
+    if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) {
+      clearWordHighlights();
+      return;
+    }
+
+    if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) {
+      clearWordHighlights();
+      return;
+    }
+
+    const wordEntry =
+      currentSentenceAlignment &&
+      currentWordIndex < currentSentenceAlignment.words.length
+        ? currentSentenceAlignment.words[currentWordIndex]
+        : undefined;
+    const wordText = wordEntry?.text || null;
+
+    if (!wordText) {
+      clearWordHighlights();
+      return;
+    }
+
+    highlightWordIndex(
+      currentSentenceAlignment,
+      currentWordIndex,
+      currentSentence || '',
+      containerRef as RefObject<HTMLDivElement>
+    );
+  }, [
+    currentWordIndex,
+    currentSentence,
+    currentSentenceAlignment,
+    pdfHighlightEnabled,
+    pdfWordHighlightEnabled,
+    clearWordHighlights,
+    highlightWordIndex
+  ]);
+
   // Add page dimensions state
   const [pageWidth, setPageWidth] = useState<number>(595); // default A4 width
   const [pageHeight, setPageHeight] = useState<number>(842); // default A4 height

src/contexts/ConfigContext.tsx

@@ -32,7 +32,9 @@ interface ConfigContextType {
   isLoading: boolean;
   isDBReady: boolean;
   pdfHighlightEnabled: boolean;
+  pdfWordHighlightEnabled: boolean;
   epubHighlightEnabled: boolean;
+  epubWordHighlightEnabled: boolean;
 }
 
 const ConfigContext = createContext<ConfigContextType | undefined>(undefined);
@@ -103,7 +105,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
     savedVoices,
     smartSentenceSplitting,
     pdfHighlightEnabled,
+    pdfWordHighlightEnabled,
     epubHighlightEnabled,
+    epubWordHighlightEnabled,
   } = config || APP_CONFIG_DEFAULTS;
 
   /**
@@ -201,7 +205,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
       isLoading,
       isDBReady,
       pdfHighlightEnabled,
-      epubHighlightEnabled
+      pdfWordHighlightEnabled,
+      epubHighlightEnabled,
+      epubWordHighlightEnabled
     }}>
       {children}
     </ConfigContext.Provider>

src/contexts/EPUBContext.tsx

@@ -22,7 +22,13 @@ import { createRangeCfi } from '@/lib/epub';
 import { useParams } from 'next/navigation';
 import { useConfig } from './ConfigContext';
 import { withRetry } from '@/utils/audio';
-import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts';
+import { CmpStr } from 'cmpstr';
+import type {
+  TTSRequestHeaders,
+  TTSRequestPayload,
+  TTSRetryOptions,
+  TTSSentenceAlignment
+} from '@/types/tts';
 
 interface EPUBContextType {
   currDocData: ArrayBuffer | undefined;
@@ -44,12 +50,26 @@ interface EPUBContextType {
   isAudioCombining: boolean;
   highlightPattern: (text: string) => void;
   clearHighlights: () => void;
+  highlightWordIndex: (
+    alignment: TTSSentenceAlignment | undefined,
+    wordIndex: number | null | undefined,
+    sentence: string | null | undefined
+  ) => void;
+  clearWordHighlights: () => void;
 }
 
 const EPUBContext = createContext<EPUBContextType | undefined>(undefined);
 
 const EPUB_CONTINUATION_CHARS = 600;
 
+const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
+
+const normalizeWordForMatch = (text: string): string =>
+  text
+    .trim()
+    .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '')
+    .toLowerCase();
+
 const stepToNextNode = (node: Node | null, root: Node): Node | null => {
   if (!node) return null;
   if (node.firstChild) {
@@ -160,6 +180,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
   const shouldPauseRef = useRef(true);
   // Track current highlight CFI for removal
   const currentHighlightCfi = useRef<string | null>(null);
+  const currentWordHighlightCfi = useRef<string | null>(null);
 
   /**
    * Clears all current document state and stops any active TTS
@@ -711,15 +732,22 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
     }
   }, [id, skipToLocation, extractPageText, setIsEPUB]);
 
-  const clearHighlights = useCallback(() => {
+  const clearWordHighlights = useCallback(() => {
-    if (renditionRef.current) {
+    if (!renditionRef.current) return;
-      if (currentHighlightCfi.current) {
+    if (currentWordHighlightCfi.current) {
-        renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight');
+      renditionRef.current.annotations.remove(currentWordHighlightCfi.current, 'highlight');
-        currentHighlightCfi.current = null;
+      currentWordHighlightCfi.current = null;
-      }
     }
   }, []);
 
+  const clearHighlights = useCallback(() => {
+    if (renditionRef.current && currentHighlightCfi.current) {
+      renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight');
+      currentHighlightCfi.current = null;
+    }
+    clearWordHighlights();
+  }, [clearWordHighlights]);
+
   const highlightPattern = useCallback(async (text: string) => {
     if (!renditionRef.current) return;
 
@@ -772,6 +800,262 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
     }
   }, [epubHighlightEnabled, clearHighlights]);
 
+  const highlightWordIndex = useCallback((
+    alignment: TTSSentenceAlignment | undefined,
+    wordIndex: number | null | undefined,
+    sentence: string | null | undefined
+  ) => {
+    clearWordHighlights();
+
+    if (!epubHighlightEnabled) return;
+    if (!alignment) return;
+    if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return;
+
+    const words = alignment.words || [];
+    if (!words.length || wordIndex >= words.length) return;
+
+    if (!renditionRef.current) return;
+    if (!currentHighlightCfi.current) return;
+
+    const cleanSentence =
+      sentence && sentence.trim()
+        ? sentence.trim().replace(/\s+/g, ' ')
+        : null;
+    if (!cleanSentence) return;
+
+    const alignmentSentenceClean = alignment.sentence
+      ? alignment.sentence.trim().replace(/\s+/g, ' ')
+      : null;
+    if (!alignmentSentenceClean || alignmentSentenceClean !== cleanSentence) {
+      return;
+    }
+
+    const contents = renditionRef.current.getContents();
+    const contentsArray = Array.isArray(contents) ? contents : [contents];
+
+    for (const content of contentsArray) {
+      let range: Range | null = null;
+      try {
+        range = content.range(currentHighlightCfi.current as string);
+      } catch {
+        range = null;
+      }
+      if (!range) continue;
+
+      const root = range.commonAncestorContainer;
+      if (!root) continue;
+
+      const domTokens: Array<{
+        node: Text;
+        startOffset: number;
+        endOffset: number;
+        norm: string;
+      }> = [];
+
+      const addTokensFromNode = (textNode: Text, start: number, end: number) => {
+        const full = textNode.textContent || '';
+        const safeStart = Math.max(0, Math.min(start, full.length));
+        const safeEnd = Math.max(safeStart, Math.min(end, full.length));
+        if (safeEnd <= safeStart) return;
+
+        const slice = full.slice(safeStart, safeEnd);
+        const wordRegex = /\S+/g;
+        let match: RegExpExecArray | null;
+        while ((match = wordRegex.exec(slice)) !== null) {
+          const raw = match[0];
+          const norm = normalizeWordForMatch(raw);
+          if (!norm) continue;
+          const tokenStart = safeStart + match.index;
+          const tokenEnd = tokenStart + raw.length;
+          domTokens.push({
+            node: textNode,
+            startOffset: tokenStart,
+            endOffset: tokenEnd,
+            norm,
+          });
+        }
+      };
+
+      const nextTextNode = (node: Node | null): Text | null => {
+        let next = getNextTextNode(node, root);
+        while (next) {
+          if (next.nodeType === Node.TEXT_NODE) {
+            return next as Text;
+          }
+          next = getNextTextNode(next, root);
+        }
+        return null;
+      };
+
+      // Collect tokens within the sentence range
+      if (range.startContainer === range.endContainer && range.startContainer.nodeType === Node.TEXT_NODE) {
+        addTokensFromNode(range.startContainer as Text, range.startOffset, range.endOffset);
+      } else {
+        let current: Text | null = null;
+
+        if (range.startContainer.nodeType === Node.TEXT_NODE) {
+          const startText = range.startContainer as Text;
+          const isEnd = range.endContainer === startText && range.endContainer.nodeType === Node.TEXT_NODE;
+          const endOffset = isEnd ? range.endOffset : (startText.textContent || '').length;
+          addTokensFromNode(startText, range.startOffset, endOffset);
+          if (isEnd) {
+            current = null;
+          } else {
+            current = nextTextNode(startText);
+          }
+        } else {
+          current = nextTextNode(range.startContainer);
+        }
+
+        while (current) {
+          if (range.endContainer.nodeType === Node.TEXT_NODE && current === range.endContainer) {
+            addTokensFromNode(current, 0, range.endOffset);
+            break;
+          } else {
+            addTokensFromNode(current, 0, (current.textContent || '').length);
+          }
+          current = nextTextNode(current);
+        }
+      }
+
+      if (!domTokens.length) {
+        return;
+      }
+
+      const domFiltered: Array<{ tokenIndex: number; norm: string }> = [];
+      for (let i = 0; i < domTokens.length; i++) {
+        const norm = domTokens[i].norm;
+        if (!norm) continue;
+        domFiltered.push({ tokenIndex: i, norm });
+      }
+
+      const ttsFiltered: Array<{ wordIndex: number; norm: string }> = [];
+      for (let i = 0; i < words.length; i++) {
+        const norm = normalizeWordForMatch(words[i].text);
+        if (!norm) continue;
+        ttsFiltered.push({ wordIndex: i, norm });
+      }
+
+      const wordToToken = new Array<number>(words.length).fill(-1);
+      const m = domFiltered.length;
+      const n = ttsFiltered.length;
+
+      if (m && n) {
+        const dp: number[][] = Array.from({ length: m + 1 }, () =>
+          new Array<number>(n + 1).fill(Number.POSITIVE_INFINITY)
+        );
+        const bt: number[][] = Array.from({ length: m + 1 }, () =>
+          new Array<number>(n + 1).fill(0)
+        ); // 0=diag,1=up,2=left
+
+        dp[0][0] = 0;
+        const GAP_COST = 0.7;
+
+        for (let i = 0; i <= m; i++) {
+          for (let j = 0; j <= n; j++) {
+            if (i > 0 && j > 0) {
+              const a = domFiltered[i - 1].norm;
+              const b = ttsFiltered[j - 1].norm;
+              const sim = cmp.compare(a, b);
+              const subCost = 1 - sim;
+              const cand = dp[i - 1][j - 1] + subCost;
+              if (cand < dp[i][j]) {
+                dp[i][j] = cand;
+                bt[i][j] = 0;
+              }
+            }
+            if (i > 0) {
+              const cand = dp[i - 1][j] + GAP_COST;
+              if (cand < dp[i][j]) {
+                dp[i][j] = cand;
+                bt[i][j] = 1;
+              }
+            }
+            if (j > 0) {
+              const cand = dp[i][j - 1] + GAP_COST;
+              if (cand < dp[i][j]) {
+                dp[i][j] = cand;
+                bt[i][j] = 2;
+              }
+            }
+          }
+        }
+
+        let i = m;
+        let j = n;
+        while (i > 0 || j > 0) {
+          const move = bt[i][j];
+          if (i > 0 && j > 0 && move === 0) {
+            const domIdx = domFiltered[i - 1].tokenIndex;
+            const ttsIdx = ttsFiltered[j - 1].wordIndex;
+            if (wordToToken[ttsIdx] === -1) {
+              wordToToken[ttsIdx] = domIdx;
+            }
+            i -= 1;
+            j -= 1;
+          } else if (i > 0 && (move === 1 || j === 0)) {
+            i -= 1;
+          } else if (j > 0 && (move === 2 || i === 0)) {
+            j -= 1;
+          } else {
+            break;
+          }
+        }
+
+        // Propagate nearest known mapping to fill gaps
+        let lastSeen = -1;
+        for (let k = 0; k < wordToToken.length; k++) {
+          if (wordToToken[k] !== -1) {
+            lastSeen = wordToToken[k];
+          } else if (lastSeen !== -1) {
+            wordToToken[k] = lastSeen;
+          }
+        }
+        let nextSeen = -1;
+        for (let k = wordToToken.length - 1; k >= 0; k--) {
+          if (wordToToken[k] !== -1) {
+            nextSeen = wordToToken[k];
+          } else if (nextSeen !== -1) {
+            wordToToken[k] = nextSeen;
+          }
+        }
+      }
+
+      const mappedIndex =
+        wordIndex < wordToToken.length ? wordToToken[wordIndex] : -1;
+      if (mappedIndex === -1) {
+        return;
+      }
+
+      const token = domTokens[mappedIndex];
+      const doc = token.node.ownerDocument || (range.commonAncestorContainer as Document);
+      const wordRange = doc.createRange();
+      wordRange.setStart(token.node, token.startOffset);
+      wordRange.setEnd(token.node, token.endOffset);
+
+      try {
+        const wordCfi = content.cfiFromRange(wordRange);
+        currentWordHighlightCfi.current = wordCfi;
+        renditionRef.current.annotations.add(
+          'highlight',
+          wordCfi,
+          {},
+          () => {},
+          '',
+          {
+            fill: 'var(--accent)',
+            'fill-opacity': '0.4',
+            'mix-blend-mode': 'multiply',
+          }
+        );
+      } catch (error) {
+        console.error('Error highlighting EPUB word:', error);
+      }
+
+      break;
+    }
+  }, [epubHighlightEnabled, clearWordHighlights]);
+
 
 
   // Context value memoization
@@ -796,6 +1080,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
       isAudioCombining,
       highlightPattern,
       clearHighlights,
+      highlightWordIndex,
+      clearWordHighlights,
     }),
     [
       setCurrentDocument,
@@ -813,6 +1099,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
       isAudioCombining,
       highlightPattern,
       clearHighlights,
+      highlightWordIndex,
+      clearWordHighlights,
     ]
   );
 

src/contexts/PDFContext.tsx

@@ -36,10 +36,16 @@ import {
   extractTextFromPDF,
   highlightPattern,
   clearHighlights,
-  handleTextClick,
+  clearWordHighlights,
+  highlightWordIndex,
 } from '@/lib/pdf';
 
-import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts';
+import type {
+  TTSRequestHeaders,
+  TTSRequestPayload,
+  TTSRetryOptions,
+  TTSSentenceAlignment
+} from '@/types/tts';
 
 /**
  * Interface defining all available methods and properties in the PDF context
@@ -59,13 +65,12 @@ interface PDFContextType {
   onDocumentLoadSuccess: (pdf: PDFDocumentProxy) => void;
   highlightPattern: (text: string, pattern: string, containerRef: RefObject<HTMLDivElement>) => void;
   clearHighlights: () => void;
-  handleTextClick: (
+  clearWordHighlights: () => void;
-    event: MouseEvent,
+  highlightWordIndex: (
-    pdfText: string,
+    alignment: TTSSentenceAlignment | undefined,
-    containerRef: RefObject<HTMLDivElement>,
+    wordIndex: number | null | undefined,
-    stopAndPlayFromIndex: (index: number) => void,
+    sentence: string | null | undefined,
-    isProcessing: boolean,
+    containerRef: RefObject<HTMLDivElement>
-    enableHighlight?: boolean
   ) => void;
   createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise<string>;
   regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>;
@@ -655,7 +660,8 @@ export function PDFProvider({ children }: { children: ReactNode }) {
       clearCurrDoc,
       highlightPattern,
       clearHighlights,
-      handleTextClick,
+      clearWordHighlights,
+      highlightWordIndex,
       pdfDocument,
       createFullAudioBook,
       regenerateChapter,

src/contexts/TTSContext.tsx

@@ -47,6 +47,7 @@ import type {
   TTSRequestPayload,
   TTSRequestHeaders,
   TTSRetryOptions,
+  TTSSentenceAlignment,
 } from '@/types/tts';
 
 // Media globals
@@ -63,6 +64,10 @@ interface TTSContextType extends TTSPlaybackState {
   // Voice settings
   availableVoices: string[];
 
+  // Alignment metadata for the current sentence
+  currentSentenceAlignment?: TTSSentenceAlignment;
+  currentWordIndex?: number | null;
+
   // Control functions
   togglePlay: () => void;
   skipForward: () => void;
@@ -264,6 +269,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     updateConfigKey,
     skipBlank,
     smartSentenceSplitting,
+    pdfHighlightEnabled,
+    pdfWordHighlightEnabled,
+    epubHighlightEnabled,
+    epubWordHighlightEnabled,
   } = useConfig();
 
   // Audio and voice management hooks
@@ -331,6 +340,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
   const epubContinuationRef = useRef<string | null>(null);
   const pageTurnEstimateRef = useRef<TTSPageTurnEstimate | null>(null);
   const pageTurnTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const sentenceAlignmentCacheRef = useRef<Map<string, TTSSentenceAlignment>>(new Map());
+  const [currentSentenceAlignment, setCurrentSentenceAlignment] = useState<TTSSentenceAlignment | undefined>();
+  const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
+  const sentencesRef = useRef<string[]>([]);
+  const currentIndexRef = useRef(0);
+
+  useEffect(() => {
+    sentencesRef.current = sentences;
+  }, [sentences]);
+
+  useEffect(() => {
+    currentIndexRef.current = currentIndex;
+  }, [currentIndex]);
 
   /**
    * Processes text into sentences using the shared NLP utility
@@ -370,6 +392,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
       clearTimeout(pageTurnTimeoutRef.current);
       pageTurnTimeoutRef.current = null;
     }
+    setCurrentWordIndex(null);
   }, [activeHowl]);
 
   /**
@@ -558,6 +581,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
           setCurrentIndex(0);
         }
 
+        // Reset alignment state whenever the text block changes
+        sentenceAlignmentCacheRef.current.clear();
+        setCurrentSentenceAlignment(undefined);
+        setCurrentWordIndex(null);
+
         // Compute auto page-turn estimate for PDFs when we have a continuation
         if (smartSentenceSplitting && !isEPUB && continuationCarried && normalizedOptions.nextLocation !== undefined) {
           const continuationNormalized = preprocessSentenceForAudio(continuationCarried);
@@ -706,10 +734,61 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
    * @returns {Promise<ArrayBuffer | undefined>} The generated audio buffer
    */
   const getAudio = useCallback(async (sentence: string): Promise<ArrayBuffer | undefined> => {
+    const alignmentEnabledForCurrentDoc =
+      (!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) ||
+      (isEPUB && epubHighlightEnabled && epubWordHighlightEnabled);
+    // Helper to ensure we have an alignment for a given
+    // sentence/audio pair, even when the audio itself is
+    // served from the local cache.
+    const ensureAlignment = (arrayBuffer: ArrayBuffer) => {
+      if (!alignmentEnabledForCurrentDoc) return;
+      if (sentenceAlignmentCacheRef.current.has(sentence)) return;
+
+      try {
+        const audioBytes = Array.from(new Uint8Array(arrayBuffer));
+        const alignmentBody = {
+          text: sentence,
+          audio: audioBytes,
+        };
+
+        void fetch('/api/whisper', {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(alignmentBody),
+        })
+          .then(async (res) => {
+            if (!res.ok) return;
+            const data = await res.json().catch(() => null);
+            if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) {
+              return;
+            }
+            const alignment = data.alignments[0] as TTSSentenceAlignment;
+            sentenceAlignmentCacheRef.current.set(sentence, alignment);
+
+            const currentSentence = sentencesRef.current[currentIndexRef.current];
+            if (currentSentence === sentence) {
+              setCurrentSentenceAlignment(alignment);
+              setCurrentWordIndex(null);
+            }
+          })
+          .catch((err) => {
+            console.warn('Alignment request failed:', err);
+          });
+      } catch (err) {
+        console.warn('Failed to start alignment request:', err);
+      }
+    };
+
     // Check if the audio is already cached
     const cachedAudio = audioCache.get(sentence);
     if (cachedAudio) {
       console.log('Using cached audio for sentence:', sentence.substring(0, 20));
+      // If we have audio but no alignment (e.g. after a
+      // navigation or TTS reset), kick off a fresh alignment
+      // request using the cached audio buffer.
+      ensureAlignment(cachedAudio);
       return cachedAudio;
     }
 
@@ -769,6 +848,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
       // Cache the array buffer
       audioCache.set(sentence, arrayBuffer);
 
+      // Fire-and-forget alignment request; do not block audio playback
+      ensureAlignment(arrayBuffer);
+
       return arrayBuffer;
     } catch (error) {
       // Check if this was an abort error
@@ -788,7 +870,21 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
       });
       throw error;
     }
-  }, [voice, speed, ttsModel, ttsInstructions, audioCache, openApiKey, openApiBaseUrl, configTTSProvider]);
+  }, [
+    voice,
+    speed,
+    ttsModel,
+    ttsInstructions,
+    audioCache,
+    openApiKey,
+    openApiBaseUrl,
+    configTTSProvider,
+    isEPUB,
+    pdfHighlightEnabled,
+    pdfWordHighlightEnabled,
+    epubHighlightEnabled,
+    epubWordHighlightEnabled
+  ]);
 
   /**
    * Processes and plays the current sentence
@@ -1004,7 +1100,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
   }, [isPlaying, advance, activeHowl, processSentence, audioSpeed]);
 
   const playAudio = useCallback(async () => {
-    const howl = await playSentenceWithHowl(sentences[currentIndex], currentIndex);
+    const sentence = sentences[currentIndex];
+    const cachedAlignment = sentenceAlignmentCacheRef.current.get(sentence);
+    if (cachedAlignment) {
+      setCurrentSentenceAlignment(cachedAlignment);
+      setCurrentWordIndex(null);
+    } else {
+      setCurrentSentenceAlignment(undefined);
+      setCurrentWordIndex(null);
+    }
+
+    const howl = await playSentenceWithHowl(sentence, currentIndex);
     if (howl) {
       howl.play();
     }
@@ -1017,6 +1123,47 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     playAudio,
   });
 
+  // Track the current word index during playback using Howler's seek position
+  useEffect(() => {
+    if (!activeHowl || !isPlaying || !currentSentenceAlignment || !currentSentenceAlignment.words.length) {
+      setCurrentWordIndex(null);
+      return;
+    }
+
+    let frameId: number;
+
+    const tick = () => {
+      try {
+        const pos = activeHowl.seek() as number;
+        if (typeof pos === 'number' && Number.isFinite(pos)) {
+          const words = currentSentenceAlignment.words;
+          let idx = -1;
+          for (let i = 0; i < words.length; i++) {
+            const w = words[i];
+            if (pos >= w.startSec && pos < w.endSec) {
+              idx = i;
+              break;
+            }
+          }
+          if (idx !== -1) {
+            setCurrentWordIndex((prev) => (prev === idx ? prev : idx));
+          }
+        }
+      } catch {
+        // ignore seek errors
+      }
+      frameId = requestAnimationFrame(tick);
+    };
+
+    frameId = requestAnimationFrame(tick);
+
+    return () => {
+      if (frameId) {
+        cancelAnimationFrame(frameId);
+      }
+    };
+  }, [activeHowl, isPlaying, currentSentenceAlignment]);
+
   /**
    * Preloads the next sentence's audio
    */
@@ -1085,6 +1232,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     setCurrDocPages(undefined);
     setIsProcessing(false);
     setIsEPUB(false);
+    sentenceAlignmentCacheRef.current.clear();
+    setCurrentSentenceAlignment(undefined);
+    setCurrentWordIndex(null);
   }, [abortAudio]);
 
   /**
@@ -1205,6 +1355,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     isProcessing,
     isBackgrounded,
     currentSentence: sentences[currentIndex] || '',
+    currentSentenceAlignment,
+    currentWordIndex,
     currDocPage,
     currDocPageNumber,
     currDocPages,
@@ -1248,7 +1400,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     skipToLocation,
     registerLocationChangeHandler,
     registerVisualPageChangeHandler,
-    setIsEPUB
+    setIsEPUB,
+    currentSentenceAlignment,
+    currentWordIndex
   ]);
 
   // Use media session hook

src/lib/dexie.ts

@@ -136,6 +136,12 @@ function buildAppConfigFromRaw(raw: RawConfigMap): AppConfigRow {
     savedVoices,
     pdfHighlightEnabled:
       raw.pdfHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfHighlightEnabled,
+    pdfWordHighlightEnabled:
+      raw.pdfWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfWordHighlightEnabled,
+    epubHighlightEnabled:
+      raw.epubHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubHighlightEnabled,
+    epubWordHighlightEnabled:
+      raw.epubWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubWordHighlightEnabled,
     firstVisit: raw.firstVisit === 'true',
     documentListState,
   };

src/lib/pdf.ts

@@ -3,9 +3,10 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api';
 import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist';
 import "core-js/proposals/promise-with-resolvers";
 import { processTextToSentences } from '@/lib/nlp';
+import type { TTSSentenceAlignment } from '@/types/tts';
 import { CmpStr } from 'cmpstr';
 
-const cmp = CmpStr.create().setMetric( 'dice' ).setFlags( 'itw' );
+const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
 
 // Worker coordination for offloading highlight token matching
 interface HighlightTokenMatchRequest {
@@ -141,12 +142,25 @@ try {
   console.error('Error patching TextLayer.render:', e);
 }
 
-interface TextMatch {
+type PDFToken = {
-  elements: HTMLElement[];
+  spanIndex: number;
-  rating: number;
+  textNode: Text;
   text: string;
-  lengthDiff: number;
+  startOffset: number;
-}
+  endOffset: number;
+};
+
+let lastSpanNodes: HTMLElement[] = [];
+let lastTokens: PDFToken[] = [];
+let lastSentenceTokenWindow: { start: number; end: number } | null = null;
+let lastSentencePattern: string | null = null;
+let lastSentenceWordToTokenMap: number[] | null = null;
+
+const normalizeWordForMatch = (text: string): string =>
+  text
+    .trim()
+    .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '')
+    .toLowerCase();
 
 // Text Processing functions
 export async function extractTextFromPDF(
@@ -279,50 +293,23 @@ export function clearHighlights() {
       element.parentElement.removeChild(element);
     }
   });
+  const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay');
+  wordOverlays.forEach((node) => {
+    const element = node as HTMLElement;
+    if (element.parentElement) {
+      element.parentElement.removeChild(element);
+    }
+  });
 }
 
-export function findBestTextMatch(
+export function clearWordHighlights() {
-  elements: Array<{ element: HTMLElement; text: string }>,
+  const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay');
-  targetText: string,
+  wordOverlays.forEach((node) => {
-  maxCombinedLength: number
+    const element = node as HTMLElement;
-): TextMatch {
+    if (element.parentElement) {
-  let bestMatch = {
+      element.parentElement.removeChild(element);
-    elements: [] as HTMLElement[],
-    rating: 0,
-    text: '',
-    lengthDiff: Infinity,
-  };
-
-  const SPAN_SEARCH_LIMIT = 10;
-
-  for (let i = 0; i < elements.length; i++) {
-    let combinedText = '';
-    const currentElements = [];
-    for (let j = i; j < Math.min(i + SPAN_SEARCH_LIMIT, elements.length); j++) {
-      const node = elements[j];
-      const newText = combinedText ? `${combinedText} ${node.text}` : node.text;
-      if (newText.length > maxCombinedLength) break;
-
-      combinedText = newText;
-      currentElements.push(node.element);
-
-      const similarity = cmp.compare(combinedText, targetText);
-      const lengthDiff = Math.abs(combinedText.length - targetText.length);
-      const lengthPenalty = lengthDiff / targetText.length;
-      const adjustedRating = similarity * (1 - lengthPenalty * 0.5);
-
-      if (adjustedRating > bestMatch.rating) {
-        bestMatch = {
-          elements: [...currentElements],
-          rating: adjustedRating,
-          text: combinedText,
-          lengthDiff,
-        };
-      }
     }
-  }
+  });
-
-  return bestMatch;
 }
 
 export function highlightPattern(
@@ -338,22 +325,18 @@ export function highlightPattern(
 
   const cleanPattern = pattern.trim().replace(/\s+/g, ' ');
   if (!cleanPattern) return;
+  lastSentencePattern = cleanPattern;
+  lastSentenceWordToTokenMap = null;
+  lastSentenceTokenWindow = null;
 
   const spanNodes = Array.from(
     container.querySelectorAll('.react-pdf__Page__textContent span')
   ) as HTMLElement[];
 
   if (!spanNodes.length) return;
+  lastSpanNodes = spanNodes;
 
-  type Token = {
+  const tokens: PDFToken[] = [];
-    spanIndex: number;
-    textNode: Text;
-    text: string;
-    startOffset: number;
-    endOffset: number;
-  };
-
-  const tokens: Token[] = [];
 
   spanNodes.forEach((span, spanIndex) => {
     const node = span.firstChild;
@@ -377,6 +360,7 @@ export function highlightPattern(
   });
 
   if (!tokens.length) return;
+  lastTokens = tokens;
 
   const patternLen = cleanPattern.length;
 
@@ -415,6 +399,11 @@ export function highlightPattern(
       bestLengthDiff < patternLen * 0.3 ? 0.3 : 0.5;
 
     if (hasTokenMatch && bestRating >= similarityThreshold) {
+      lastSentenceTokenWindow = {
+        start: bestStart,
+        end: bestEnd,
+      };
+
       const rangesBySpan = new Map<
         number,
         { startOffset: number; endOffset: number }
@@ -454,65 +443,6 @@ export function highlightPattern(
       });
     }
 
-    // Fallback: if token-level matching failed, use span-based fuzzy matching
-    if (!highlightRanges.length) {
-      const spanEntries = spanNodes
-        .map((node) => ({
-          element: node as HTMLElement,
-          text: (node.textContent || '').trim(),
-        }))
-        .filter((entry) => entry.text.length > 0);
-
-      const containerRect = container.getBoundingClientRect();
-      const visibleTop = container.scrollTop;
-      const visibleBottom = visibleTop + containerRect.height;
-      const bufferSize = containerRect.height;
-
-      const visibleNodes = spanEntries.filter(({ element }) => {
-        const rect = element.getBoundingClientRect();
-        const elementTop =
-          rect.top - containerRect.top + container.scrollTop;
-        return (
-          elementTop >= visibleTop - bufferSize &&
-          elementTop <= visibleBottom + bufferSize
-        );
-      });
-
-      let bestMatch = findBestTextMatch(
-        visibleNodes,
-        cleanPattern,
-        cleanPattern.length * 2
-      );
-
-      if (bestMatch.rating < 0.3) {
-        bestMatch = findBestTextMatch(
-          spanEntries,
-          cleanPattern,
-          cleanPattern.length * 2
-        );
-      }
-
-      const spanSimilarityThreshold =
-        bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5;
-
-      if (bestMatch.rating >= spanSimilarityThreshold) {
-        bestMatch.elements.forEach((element) => {
-          const node = element.firstChild;
-          if (!node || node.nodeType !== Node.TEXT_NODE) return;
-          const textNode = node as Text;
-          const content = textNode.textContent || '';
-          if (!content) return;
-
-          highlightRanges.push({
-            textNode,
-            startOffset: 0,
-            endOffset: content.length,
-            span: element,
-          });
-        });
-      }
-    }
-
     if (!highlightRanges.length) return;
 
     // Create overlay rectangles for each range, relative to its page text layer
@@ -577,7 +507,7 @@ export function highlightPattern(
   runHighlightTokenMatch(cleanPattern, tokenTexts)
     .then((result) => {
       if (!result || result.bestStart === -1) {
-        // No worker result or no good match; rely on span-level fallback
+        // No worker result or no good match; nothing to highlight
         applyHighlightFromTokens(null);
       } else {
         applyHighlightFromTokens({
@@ -590,95 +520,199 @@ export function highlightPattern(
     })
     .catch((error) => {
       console.error(
-        'Error in PDF highlight worker, falling back to span-based matching:',
+        'Error in PDF highlight worker; no highlights applied:',
         error
       );
       applyHighlightFromTokens(null);
     });
 }
 
-// Text Click Handler
+export function highlightWordIndex(
-export function handleTextClick(
+  alignment: TTSSentenceAlignment | undefined,
-  event: MouseEvent,
+  wordIndex: number | null | undefined,
-  pdfText: string,
+  sentence: string | null | undefined,
-  containerRef: React.RefObject<HTMLDivElement>,
+  containerRef: React.RefObject<HTMLDivElement>
-  stopAndPlayFromIndex: (index: number) => void,
-  isProcessing: boolean,
-  enableHighlight = true
 ) {
-  if (isProcessing) return;
+  clearWordHighlights();
 
-  const target = event.target as HTMLElement;
+  if (!alignment) return;
-  if (!target.matches('.react-pdf__Page__textContent span')) return;
+  if (wordIndex === null || wordIndex === undefined || wordIndex < 0) {
-
-  const parentElement = target.closest('.react-pdf__Page__textContent');
-  if (!parentElement) return;
-
-  const spans = Array.from(parentElement.querySelectorAll('span'));
-  const clickedIndex = spans.indexOf(target);
-  const contextWindow = 3;
-  const startIndex = Math.max(0, clickedIndex - contextWindow);
-  const endIndex = Math.min(spans.length - 1, clickedIndex + contextWindow);
-  const contextText = spans
-    .slice(startIndex, endIndex + 1)
-    .map((span) => span.textContent)
-    .join(' ')
-    .trim();
-
-  if (!contextText?.trim()) return;
-
-  const cleanContext = contextText.trim().replace(/\s+/g, ' ');
-
-  // Fast path when highlight overlays are disabled:
-  // avoid expensive span-level fuzzy matching and just map
-  // the clicked context to a sentence using cheap string checks.
-  if (!enableHighlight) {
-    const sentences = processTextToSentences(pdfText);
-    const idx = sentences.findIndex((sentence) => {
-      const cleanSentence = sentence.trim().replace(/\s+/g, ' ');
-      return (
-        cleanSentence.includes(cleanContext) ||
-        cleanContext.includes(cleanSentence)
-      );
-    });
-
-    if (idx !== -1) {
-      stopAndPlayFromIndex(idx);
-    }
     return;
   }
 
-  const allText = Array.from(parentElement.querySelectorAll('span')).map((node) => ({
+  const words = alignment.words || [];
-    element: node as HTMLElement,
+  if (!words.length || wordIndex >= words.length) return;
-    text: (node.textContent || '').trim(),
-  })).filter((node) => node.text.length > 0);
 
-  const bestMatch = findBestTextMatch(allText, cleanContext, cleanContext.length * 2);
+  const container = containerRef.current;
-  const similarityThreshold = bestMatch.lengthDiff < cleanContext.length * 0.3 ? 0.3 : 0.5;
+  if (!container) return;
+  if (!lastSentenceTokenWindow) return;
+  if (!lastTokens.length || !lastSpanNodes.length) return;
 
-  if (bestMatch.rating >= similarityThreshold) {
+  const cleanSentence =
-    const matchText = bestMatch.text;
+    sentence && sentence.trim()
-    // Use the same sentence processing logic as TTSContext for consistency
+      ? sentence.trim().replace(/\s+/g, ' ')
-    const sentences = processTextToSentences(pdfText);
+      : null;
-    console.log("sentences inside handleTextClick: %d", sentences.length)
+  if (!cleanSentence || !lastSentencePattern) return;
-    let bestSentenceMatch = { sentence: '', rating: 0 };
+  if (cleanSentence !== lastSentencePattern) return;
 
-    for (const sentence of sentences) {
+  const start = lastSentenceTokenWindow.start;
-      const rating = cmp.compare(matchText, sentence);
+  const end = lastSentenceTokenWindow.end;
-      if (rating > bestSentenceMatch.rating) {
+  if (end < start) return;
-        bestSentenceMatch = { sentence, rating };
+
-      }
+  // Lazily build or refresh the mapping from alignment word
+  // indices to PDF token indices for this sentence window.
+  if (
+    !lastSentenceWordToTokenMap ||
+    lastSentenceWordToTokenMap.length !== words.length
+  ) {
+    const pdfFiltered: { tokenIndex: number; norm: string }[] = [];
+    for (let i = start; i <= end; i++) {
+      const norm = normalizeWordForMatch(lastTokens[i].text);
+      if (!norm) continue;
+      pdfFiltered.push({ tokenIndex: i, norm });
     }
 
-    if (bestSentenceMatch.rating >= 0.5) {
+    const ttsFiltered: { wordIndex: number; norm: string }[] = [];
-      const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence);
+    for (let i = 0; i < words.length; i++) {
-      if (sentenceIndex !== -1) {
+      const norm = normalizeWordForMatch(words[i].text);
-        stopAndPlayFromIndex(sentenceIndex);
+      if (!norm) continue;
-        if (enableHighlight) {
+      ttsFiltered.push({ wordIndex: i, norm });
-          highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef);
+    }
+
+    const wordToToken = new Array<number>(words.length).fill(-1);
+
+    const m = pdfFiltered.length;
+    const n = ttsFiltered.length;
+
+    if (m && n) {
+      const dp: number[][] = Array.from({ length: m + 1 }, () =>
+        new Array<number>(n + 1).fill(Number.POSITIVE_INFINITY)
+      );
+      const bt: number[][] = Array.from({ length: m + 1 }, () =>
+        new Array<number>(n + 1).fill(0)
+      ); // 0=diag,1=up,2=left
+
+      dp[0][0] = 0;
+      const GAP_COST = 0.7;
+
+      for (let i = 0; i <= m; i++) {
+        for (let j = 0; j <= n; j++) {
+          if (i > 0 && j > 0) {
+            const a = pdfFiltered[i - 1].norm;
+            const b = ttsFiltered[j - 1].norm;
+            const sim = cmp.compare(a, b);
+            const subCost = 1 - sim;
+            const cand = dp[i - 1][j - 1] + subCost;
+            if (cand < dp[i][j]) {
+              dp[i][j] = cand;
+              bt[i][j] = 0;
+            }
+          }
+          if (i > 0) {
+            const cand = dp[i - 1][j] + GAP_COST;
+            if (cand < dp[i][j]) {
+              dp[i][j] = cand;
+              bt[i][j] = 1;
+            }
+          }
+          if (j > 0) {
+            const cand = dp[i][j - 1] + GAP_COST;
+            if (cand < dp[i][j]) {
+              dp[i][j] = cand;
+              bt[i][j] = 2;
+            }
+          }
+        }
+      }
+
+      let i = m;
+      let j = n;
+      while (i > 0 || j > 0) {
+        const move = bt[i][j];
+        if (i > 0 && j > 0 && move === 0) {
+          const pdfIdx = pdfFiltered[i - 1].tokenIndex;
+          const ttsIdx = ttsFiltered[j - 1].wordIndex;
+          if (wordToToken[ttsIdx] === -1) {
+            wordToToken[ttsIdx] = pdfIdx;
+          }
+          i -= 1;
+          j -= 1;
+        } else if (i > 0 && (move === 1 || j === 0)) {
+          i -= 1;
+        } else if (j > 0 && (move === 2 || i === 0)) {
+          j -= 1;
+        } else {
+          break;
+        }
+      }
+
+      // Propagate nearest known mapping to fill gaps
+      let lastSeen = -1;
+      for (let k = 0; k < wordToToken.length; k++) {
+        if (wordToToken[k] !== -1) {
+          lastSeen = wordToToken[k];
+        } else if (lastSeen !== -1) {
+          wordToToken[k] = lastSeen;
+        }
+      }
+      let nextSeen = -1;
+      for (let k = wordToToken.length - 1; k >= 0; k--) {
+        if (wordToToken[k] !== -1) {
+          nextSeen = wordToToken[k];
+        } else if (nextSeen !== -1) {
+          wordToToken[k] = nextSeen;
         }
       }
     }
+
+    lastSentenceWordToTokenMap = wordToToken;
+  }
+
+  const mappedIndex =
+    lastSentenceWordToTokenMap && wordIndex < lastSentenceWordToTokenMap.length
+      ? lastSentenceWordToTokenMap[wordIndex]
+      : -1;
+
+  if (mappedIndex === -1) return;
+
+  const chosenTokenIndex = mappedIndex;
+
+  const token = lastTokens[chosenTokenIndex];
+  const span = lastSpanNodes[token.spanIndex];
+  if (!span) return;
+
+  const node = token.textNode;
+  if (!node || node.nodeType !== Node.TEXT_NODE) return;
+
+  try {
+    const range = document.createRange();
+    range.setStart(node, token.startOffset);
+    range.setEnd(node, token.endOffset);
+
+    const pageLayer = span.closest(
+      '.react-pdf__Page__textContent'
+    ) as HTMLElement | null;
+    if (!pageLayer) return;
+
+    const pageRect = pageLayer.getBoundingClientRect();
+    const rects = Array.from(range.getClientRects());
+
+    rects.forEach((rect) => {
+      const highlight = document.createElement('div');
+      highlight.className = 'pdf-word-highlight-overlay';
+      highlight.style.position = 'absolute';
+      highlight.style.backgroundColor = 'var(--accent)';
+      highlight.style.opacity = '0.4';
+      highlight.style.pointerEvents = 'none';
+      highlight.style.left = `${rect.left - pageRect.left}px`;
+      highlight.style.top = `${rect.top - pageRect.top}px`;
+      highlight.style.width = `${rect.width}px`;
+      highlight.style.height = `${rect.height}px`;
+      highlight.style.zIndex = '2';
+      pageLayer.appendChild(highlight);
+    });
+  } catch {
+    // Ignore range errors
   }
 }
 

src/types/config.ts

@@ -23,7 +23,9 @@ export interface AppConfigValues {
   savedVoices: SavedVoices;
   smartSentenceSplitting: boolean;
   pdfHighlightEnabled: boolean;
+  pdfWordHighlightEnabled: boolean;
   epubHighlightEnabled: boolean;
+  epubWordHighlightEnabled: boolean;
   firstVisit: boolean;
   documentListState: DocumentListState;
 }
@@ -47,7 +49,9 @@ export const APP_CONFIG_DEFAULTS: AppConfigValues = {
   savedVoices: {},
   smartSentenceSplitting: true,
   pdfHighlightEnabled: true,
+  pdfWordHighlightEnabled: true,
   epubHighlightEnabled: true,
+  epubWordHighlightEnabled: true,
   firstVisit: false,
   documentListState: {
     sortBy: 'name',

src/types/tts.ts

@@ -63,6 +63,22 @@ export interface TTSPageTurnEstimate {
   fraction: number;
 }
 
+// Word-level alignment within a single spoken sentence/block
+export interface TTSSentenceWord {
+  text: string;
+  startSec: number;
+  endSec: number;
+  charStart: number;
+  charEnd: number;
+}
+
+// Alignment metadata for a single TTS sentence/block
+export interface TTSSentenceAlignment {
+  sentence: string;
+  sentenceIndex: number;
+  words: TTSSentenceWord[];
+}
+
 // Metadata for an audiobook chapter
 export interface TTSAudiobookChapter {
   index: number;
@@ -71,4 +87,4 @@ export interface TTSAudiobookChapter {
   status: 'pending' | 'generating' | 'completed' | 'error';
   bookId?: string;
   format?: 'mp3' | 'm4b';
-}
+}