danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #110 from richardr1126/fix/epub-text-highlight

Browse source 
fix(epub/text): native offset word-by-word highlighting
This commit is contained in:
Richard R 2026-06-09 09:19:52 -06:00 committed by GitHub
commit 6c12a47f74
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 298 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
compute/core/src/whisper/alignment-map.ts
View file

@ -1,9 +1,14 @@
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
// Keep byte-for-byte in lock-step with `preprocessSentenceForAudio` in
// `src/lib/shared/nlp.ts` (and the position-preserving copy in
// `src/lib/client/highlight-char-map.ts`). The word `charStart`/`charEnd`
// offsets this module emits are consumed against text normalized by those
// client functions, so any divergence shifts viewer highlights off-word.
function preprocessSentenceForAudio(text: string): string { function preprocessSentenceForAudio(text: string): string {
return text return text
.replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -')
.replace(/(\w+)-\s+(\w+)/g, '$1$2') .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2')
.replace(/\*/g, '') .replace(/\*/g, '')
.replace(/\s+/g, ' ') .replace(/\s+/g, ' ')
.trim(); .trim();

4
src/app/(app)/epub/[id]/useEpubDocument.ts
View file

@ -36,7 +36,6 @@ import {
} from '@/lib/client/epub/epub-rendered-text-maps'; } from '@/lib/client/epub/epub-rendered-text-maps';
import { import {
useEPUBHighlighting, useEPUBHighlighting,
type EpubWordHighlightMapCache,
} from '@/hooks/epub/useEPUBHighlighting'; } from '@/hooks/epub/useEPUBHighlighting';
import { useEPUBLocationController } from '@/hooks/epub/useEPUBLocationController'; import { useEPUBLocationController } from '@/hooks/epub/useEPUBLocationController';
import { useEPUBAudiobook } from '@/hooks/epub/useEPUBAudiobook'; import { useEPUBAudiobook } from '@/hooks/epub/useEPUBAudiobook';
@ -148,7 +147,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
const currentHighlightCfi = useRef<string | null>(null); const currentHighlightCfi = useRef<string | null>(null);
const currentWordHighlightCfi = useRef<string | null>(null); const currentWordHighlightCfi = useRef<string | null>(null);
const renderedTextMapsRef = useRef<EpubRenderedTextMap[]>([]); const renderedTextMapsRef = useRef<EpubRenderedTextMap[]>([]);
const wordHighlightMapCacheRef = useRef<EpubWordHighlightMapCache | null>(null);
const renderedLocationCloneManagerRef = useRef<EpubRenderedLocationCloneManager>( const renderedLocationCloneManagerRef = useRef<EpubRenderedLocationCloneManager>(
new EpubRenderedLocationCloneManager(), new EpubRenderedLocationCloneManager(),
); );
@ -165,8 +163,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
currentHighlightCfiRef: currentHighlightCfi, currentHighlightCfiRef: currentHighlightCfi,
currentWordHighlightCfiRef: currentWordHighlightCfi, currentWordHighlightCfiRef: currentWordHighlightCfi,
renderedTextMapsRef, renderedTextMapsRef,
wordHighlightMapCacheRef,
language: resolvedLanguage,
}); });
useEffect(() => () => { useEffect(() => () => {

7
src/contexts/TTSContext.tsx
View file

@ -2485,6 +2485,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const pos = activeHowl.seek() as number; const pos = activeHowl.seek() as number;
if (typeof pos === 'number' && Number.isFinite(pos)) { if (typeof pos === 'number' && Number.isFinite(pos)) {
const words = currentSentenceAlignment.words; const words = currentSentenceAlignment.words;
// Find the word whose timing window contains `pos`. When `pos` falls
// in a gap between words (or past the last word), snap to the last
// word that has already started instead of freezing on a stale index
// — monotonic, so it never jumps to a later word that hasn't started.
let idx = -1; let idx = -1;
for (let i = 0; i < words.length; i++) { for (let i = 0; i < words.length; i++) {
const w = words[i]; const w = words[i];
@ -2492,6 +2496,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
idx = i; idx = i;
break; break;
} }
if (pos >= w.startSec) {
idx = i;
}
} }
if (idx !== -1) { if (idx !== -1) {
setCurrentWordIndex((prev) => (prev === idx ? prev : idx)); setCurrentWordIndex((prev) => (prev === idx ? prev : idx));

96
src/hooks/epub/useEPUBHighlighting.ts
View file

@ -4,15 +4,8 @@ import { useCallback, useEffect, type MutableRefObject, type RefObject } from 'r
import type { Rendition } from 'epubjs'; import type { Rendition } from 'epubjs';
import { import {
buildWordHighlightCacheKey,
resolveAlignmentWordSourceRange, resolveAlignmentWordSourceRange,
tokenizeCanonicalSegment,
type EpubCanonicalWordToken,
} from '@/lib/client/epub/epub-word-highlight'; } from '@/lib/client/epub/epub-word-highlight';
import {
buildAlignmentTokenRanges,
type HighlightTokenRange,
} from '@/lib/client/highlight-token-alignment';
import { import {
createRangeFromMappedOffsets, createRangeFromMappedOffsets,
resolveVisibleSegmentRange, resolveVisibleSegmentRange,
@ -21,20 +14,12 @@ import {
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
import type { TTSSentenceAlignment } from '@/types/tts'; import type { TTSSentenceAlignment } from '@/types/tts';
export type EpubWordHighlightMapCache = {
key: string;
wordToTokenRange: Array<HighlightTokenRange | null>;
tokens: EpubCanonicalWordToken[];
};
type UseEpubHighlightingParams = { type UseEpubHighlightingParams = {
renditionRef: RefObject<Rendition | undefined>; renditionRef: RefObject<Rendition | undefined>;
epubHighlightEnabled: boolean; epubHighlightEnabled: boolean;
currentHighlightCfiRef: MutableRefObject<string | null>; currentHighlightCfiRef: MutableRefObject<string | null>;
currentWordHighlightCfiRef: MutableRefObject<string | null>; currentWordHighlightCfiRef: MutableRefObject<string | null>;
renderedTextMapsRef: MutableRefObject<EpubRenderedTextMap[]>; renderedTextMapsRef: MutableRefObject<EpubRenderedTextMap[]>;
wordHighlightMapCacheRef: MutableRefObject<EpubWordHighlightMapCache | null>;
language?: string;
}; };
type UseEpubHighlightingResult = { type UseEpubHighlightingResult = {
@ -56,8 +41,6 @@ export function useEPUBHighlighting({
currentHighlightCfiRef, currentHighlightCfiRef,
currentWordHighlightCfiRef, currentWordHighlightCfiRef,
renderedTextMapsRef, renderedTextMapsRef,
wordHighlightMapCacheRef,
language,
}: UseEpubHighlightingParams): UseEpubHighlightingResult { }: UseEpubHighlightingParams): UseEpubHighlightingResult {
const clearWordHighlights = useCallback(() => { const clearWordHighlights = useCallback(() => {
if (!renditionRef.current) return; if (!renditionRef.current) return;
@ -122,68 +105,19 @@ export function useEPUBHighlighting({
const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment); const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment);
if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return; if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return;
// Native path only: the alignment's char offsets are authoritative for the
// spoken word (they live in the same canonical space as the segment text and
// the rendered char map). Clamp to the portion of the segment that is
// actually visible in this map so a word straddling a page/spread boundary
// still highlights its visible part instead of dropping the word entirely.
const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]); const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]);
if ( if (!alignmentRange) return;
alignmentRange
&& alignmentRange.sourceStart >= resolved.startOffset
&& alignmentRange.sourceEnd <= resolved.endOffset
) {
const wordRange = createRangeFromMappedOffsets(
resolved.map,
alignmentRange.sourceStart,
alignmentRange.sourceEnd,
);
if (wordRange) {
try {
const wordCfi = resolved.map.content.cfiFromRange(wordRange);
currentWordHighlightCfiRef.current = wordCfi;
renditionRef.current.annotations.add(
'highlight',
wordCfi,
{},
() => { },
'',
{
fill: 'var(--accent)',
'fill-opacity': '0.4',
'mix-blend-mode': 'multiply',
}
);
return;
} catch (error) {
console.error('Error highlighting EPUB word from alignment offsets:', error);
}
}
}
const cacheKey = buildWordHighlightCacheKey(segment, alignment, language); const clampedStart = Math.max(alignmentRange.sourceStart, resolved.startOffset);
if (wordHighlightMapCacheRef.current?.key !== cacheKey) { const clampedEnd = Math.min(alignmentRange.sourceEnd, resolved.endOffset);
const tokens = tokenizeCanonicalSegment(segment, language); if (clampedEnd <= clampedStart) return;
wordHighlightMapCacheRef.current = {
key: cacheKey,
tokens,
wordToTokenRange: buildAlignmentTokenRanges(
words,
tokens.map((token) => token.norm),
{ minimumSimilarity: 0.8 },
),
};
}
const cached = wordHighlightMapCacheRef.current; const wordRange = createRangeFromMappedOffsets(resolved.map, clampedStart, clampedEnd);
const tokenRange = cached.wordToTokenRange[wordIndex];
if (!tokenRange) return;
const firstToken = cached.tokens[tokenRange.start];
const lastToken = cached.tokens[tokenRange.end];
if (!firstToken || !lastToken) return;
if (firstToken.sourceStart < resolved.startOffset || lastToken.sourceEnd > resolved.endOffset) return;
const wordRange = createRangeFromMappedOffsets(
resolved.map,
firstToken.sourceStart,
lastToken.sourceEnd,
);
if (!wordRange) return; if (!wordRange) return;
try { try {
@ -202,7 +136,7 @@ export function useEPUBHighlighting({
} }
); );
} catch (error) { } catch (error) {
console.error('Error highlighting EPUB word:', error); console.error('Error highlighting EPUB word from alignment offsets:', error);
} }
}, [ }, [
clearWordHighlights, clearWordHighlights,
@ -210,20 +144,16 @@ export function useEPUBHighlighting({
epubHighlightEnabled, epubHighlightEnabled,
renderedTextMapsRef, renderedTextMapsRef,
renditionRef, renditionRef,
wordHighlightMapCacheRef,
language,
]); ]);
const setRenderedTextMaps = useCallback((maps: EpubRenderedTextMap[]) => { const setRenderedTextMaps = useCallback((maps: EpubRenderedTextMap[]) => {
renderedTextMapsRef.current = maps; renderedTextMapsRef.current = maps;
wordHighlightMapCacheRef.current = null; }, [renderedTextMapsRef]);
}, [renderedTextMapsRef, wordHighlightMapCacheRef]);
const resetHighlightState = useCallback(() => { const resetHighlightState = useCallback(() => {
renderedTextMapsRef.current = []; renderedTextMapsRef.current = [];
wordHighlightMapCacheRef.current = null;
clearHighlights(); clearHighlights();
}, [clearHighlights, renderedTextMapsRef, wordHighlightMapCacheRef]); }, [clearHighlights, renderedTextMapsRef]);
// Clear any highlight annotations when feature is disabled. // Clear any highlight annotations when feature is disabled.
useEffect(() => { useEffect(() => {

100
src/lib/client/epub/epub-rendered-text-maps.ts
View file

@ -2,16 +2,14 @@
import type { Rendition } from 'epubjs'; import type { Rendition } from 'epubjs';
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map';
type EpubMappedPosition = { type EpubMappedPosition = {
node: Text; node: Text;
offset: number; offset: number;
}; };
type EpubMappedChar = { type EpubMappedChar = MappedChar<EpubMappedPosition>;
char: string;
position: EpubMappedPosition;
};
export type EpubRenderedTextMap = { export type EpubRenderedTextMap = {
sourceKey: string; sourceKey: string;
@ -21,94 +19,6 @@ export type EpubRenderedTextMap = {
}; };
}; };
const cloneMappedChar = (char: string, source: EpubMappedChar): EpubMappedChar => ({
char,
position: source.position,
});
const replaceMappedUrls = (tokens: EpubMappedChar[]): EpubMappedChar[] => {
const text = tokens.map((token) => token.char).join('');
const urlPattern = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi;
const replaced: EpubMappedChar[] = [];
let cursor = 0;
let match: RegExpExecArray | null;
while ((match = urlPattern.exec(text)) !== null) {
const start = match.index;
const end = start + match[0].length;
replaced.push(...tokens.slice(cursor, start));
const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)];
if (anchor) {
const replacement = `- (link to ${match[1]}) -`;
for (const char of replacement) {
replaced.push(cloneMappedChar(char, anchor));
}
}
cursor = end;
}
replaced.push(...tokens.slice(cursor));
return replaced;
};
const removeMappedHyphenation = (tokens: EpubMappedChar[]): EpubMappedChar[] => {
const text = tokens.map((token) => token.char).join('');
const hyphenPattern = /(\w+)-\s+(\w+)/g;
const replaced: EpubMappedChar[] = [];
let cursor = 0;
let match: RegExpExecArray | null;
while ((match = hyphenPattern.exec(text)) !== null) {
const start = match.index;
const full = match[0];
const first = match[1];
const second = match[2];
const secondOffset = full.lastIndexOf(second);
replaced.push(...tokens.slice(cursor, start));
replaced.push(...tokens.slice(start, start + first.length));
replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length));
cursor = start + full.length;
}
replaced.push(...tokens.slice(cursor));
return replaced;
};
const normalizeMappedTokensForTts = (tokens: EpubMappedChar[]): EpubMappedChar[] => {
const withoutLinks = replaceMappedUrls(tokens);
const withoutHyphenation = removeMappedHyphenation(withoutLinks);
const normalized: EpubMappedChar[] = [];
let pendingWhitespace: EpubMappedChar | null = null;
const flushWhitespace = () => {
if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') {
pendingWhitespace = null;
return;
}
normalized.push(cloneMappedChar(' ', pendingWhitespace));
pendingWhitespace = null;
};
for (const token of withoutHyphenation) {
if (token.char === '*') continue;
if (/\s/.test(token.char)) {
pendingWhitespace ??= token;
continue;
}
flushWhitespace();
normalized.push(token);
}
if (normalized[normalized.length - 1]?.char === ' ') {
normalized.pop();
}
return normalized;
};
const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => { const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => {
const root = range.commonAncestorContainer; const root = range.commonAncestorContainer;
const doc = range.startContainer.ownerDocument ?? (range.startContainer as Document); const doc = range.startContainer.ownerDocument ?? (range.startContainer as Document);
@ -121,7 +31,7 @@ const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => {
for (let offset = safeStart; offset < safeEnd; offset += 1) { for (let offset = safeStart; offset < safeEnd; offset += 1) {
mapped.push({ mapped.push({
char: text[offset], char: text[offset],
position: { node: textNode, offset }, pos: { node: textNode, offset },
}); });
} }
}; };
@ -178,12 +88,12 @@ export const buildRenderedTextMaps = (
const range = content.range(rangeCfi); const range = content.range(rangeCfi);
if (!range) continue; if (!range) continue;
const normalized = normalizeMappedTokensForTts(collectMappedTextFromRange(range)); const normalized = normalizeMappedChars(collectMappedTextFromRange(range));
if (!normalized.length) continue; if (!normalized.length) continue;
maps.push({ maps.push({
sourceKey, sourceKey,
chars: normalized.map((token) => token.position), chars: normalized.map((token) => token.pos),
content, content,
}); });
} catch { } catch {

44
src/lib/client/epub/epub-word-highlight.ts
View file

@ -1,17 +1,12 @@
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; import type { TTSSentenceWord } from '@/types/tts';
import { segmentWords } from '@/lib/shared/language';
import { normalizeHighlightToken } from '@/lib/client/highlight-token-alignment';
export type EpubCanonicalWordToken = {
norm: string;
sourceStart: number;
sourceEnd: number;
};
export const normalizeWordForHighlight = (text: string): string =>
normalizeHighlightToken(text);
/**
* Resolve a spoken word's char offsets (from the Whisper alignment) into the
* canonical source-offset space the rendered text map is keyed by. The
* alignment's `charStart`/`charEnd` are offsets into the segment text, which is
* in the same normalized space as `segment.startAnchor.offset`.
*/
export const resolveAlignmentWordSourceRange = ( export const resolveAlignmentWordSourceRange = (
segment: CanonicalTtsSegment, segment: CanonicalTtsSegment,
word: TTSSentenceWord, word: TTSSentenceWord,
@ -25,28 +20,3 @@ export const resolveAlignmentWordSourceRange = (
sourceEnd: segment.startAnchor.offset + charEnd, sourceEnd: segment.startAnchor.offset + charEnd,
}; };
}; };
export const tokenizeCanonicalSegment = (
segment: CanonicalTtsSegment,
language?: string,
): EpubCanonicalWordToken[] =>
segmentWords(segment.text, language)
.map((token) => ({
norm: normalizeWordForHighlight(token.text),
sourceStart: segment.startAnchor.offset + token.start,
sourceEnd: segment.startAnchor.offset + token.end,
}))
.filter((token) => Boolean(token.norm));
export const buildWordHighlightCacheKey = (
segment: CanonicalTtsSegment,
alignment: TTSSentenceAlignment,
language?: string,
): string =>
[
segment.key,
segment.text.length,
language || '',
alignment.words.length,
alignment.words.map((word) => normalizeWordForHighlight(word.text)).join('|'),
].join('::');

124
src/lib/client/highlight-char-map.ts Normal file
View file

@ -0,0 +1,124 @@
/**
* Shared, position-preserving text normalizer for viewer highlighting.
*
* TTS word offsets (`charStart`/`charEnd`) are computed against the canonical
* "audio" form of a sentence — URLs rewritten, line-break hyphenation joined,
* `*` stripped, whitespace collapsed. To map those offsets back onto the
* rendered DOM we have to normalize the DOM text the SAME way while remembering
* which DOM position each surviving character came from.
*
* This module operates on an opaque position type so both the EPUB renderer
* (position = `{ node, offset }` in an iframe) and the HTML/TXT renderer
* (position = a `Text` node + offset in the main document) share one identical
* normalization. Keep the transforms here in lock-step with
* `preprocessSentenceForAudio` in `src/lib/shared/nlp.ts` and the copy in
* `compute/core/src/whisper/alignment-map.ts`.
*/
export interface MappedChar<TPos> {
char: string;
pos: TPos;
}
const URL_PATTERN = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi;
// Unicode-aware to match preprocessSentenceForAudio (nlp.ts / alignment-map.ts).
const HYPHENATION_PATTERN = /([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu;
const cloneMappedChar = <TPos>(char: string, source: MappedChar<TPos>): MappedChar<TPos> => ({
char,
pos: source.pos,
});
const replaceMappedUrls = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[] => {
const text = tokens.map((token) => token.char).join('');
const replaced: MappedChar<TPos>[] = [];
let cursor = 0;
let match: RegExpExecArray | null;
URL_PATTERN.lastIndex = 0;
while ((match = URL_PATTERN.exec(text)) !== null) {
const start = match.index;
const end = start + match[0].length;
replaced.push(...tokens.slice(cursor, start));
const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)];
if (anchor) {
const replacement = `- (link to ${match[1]}) -`;
// Spread the replacement characters across the original URL span so the
// mapped positions keep their positional spread instead of collapsing the
// whole URL onto a single DOM anchor.
const originalLength = Math.max(1, end - start);
for (let i = 0; i < replacement.length; i += 1) {
const sourceIndex = Math.min(end - 1, start + Math.floor((i * originalLength) / replacement.length));
const source = tokens[sourceIndex] ?? anchor;
replaced.push(cloneMappedChar(replacement[i], source));
}
}
cursor = end;
}
replaced.push(...tokens.slice(cursor));
return replaced;
};
const removeMappedHyphenation = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[] => {
const text = tokens.map((token) => token.char).join('');
const replaced: MappedChar<TPos>[] = [];
let cursor = 0;
let match: RegExpExecArray | null;
HYPHENATION_PATTERN.lastIndex = 0;
while ((match = HYPHENATION_PATTERN.exec(text)) !== null) {
const start = match.index;
const full = match[0];
const first = match[1];
const second = match[2];
const secondOffset = full.lastIndexOf(second);
replaced.push(...tokens.slice(cursor, start));
replaced.push(...tokens.slice(start, start + first.length));
replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length));
cursor = start + full.length;
}
replaced.push(...tokens.slice(cursor));
return replaced;
};
/**
* Normalize a position-tagged character stream into the canonical TTS form,
* dropping/rewriting characters while preserving the source position of every
* surviving character.
*/
export const normalizeMappedChars = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[] => {
const withoutLinks = replaceMappedUrls(tokens);
const withoutHyphenation = removeMappedHyphenation(withoutLinks);
const normalized: MappedChar<TPos>[] = [];
let pendingWhitespace: MappedChar<TPos> | null = null;
const flushWhitespace = () => {
if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') {
pendingWhitespace = null;
return;
}
normalized.push(cloneMappedChar(' ', pendingWhitespace));
pendingWhitespace = null;
};
for (const token of withoutHyphenation) {
if (token.char === '*') continue;
if (/\s/.test(token.char)) {
pendingWhitespace ??= token;
continue;
}
flushWhitespace();
normalized.push(token);
}
if (normalized[normalized.length - 1]?.char === ' ') {
normalized.pop();
}
return normalized;
};

158
src/lib/client/html/highlight.ts
View file

@ -11,18 +11,19 @@
* sentence * sentence
* - `WORD` — saturated background on the currently-spoken word * - `WORD` — saturated background on the currently-spoken word
* *
* Word-to-DOM alignment uses the shared viewer token-range mapper so DOM token * Word-to-DOM alignment is native: the Whisper alignment gives authoritative
* counts that diverge from the timed alignment still produce a smooth, * `charStart`/`charEnd` offsets into the sentence text, and we map those offsets
* monotonic highlight across languages. * directly onto a normalized char→DOM map of the located sentence wrap. No fuzzy
* token matching for words — that path used to jump the highlight to a later
* word in the sentence.
*/ */
import type { TTSSentenceAlignment } from '@/types/tts'; import type { TTSSentenceAlignment } from '@/types/tts';
import { segmentWords } from '@/lib/shared/language'; import { segmentWords } from '@/lib/shared/language';
import { import {
buildAlignmentTokenRanges,
findBestHighlightTokenMatch, findBestHighlightTokenMatch,
normalizeHighlightToken, normalizeHighlightToken,
type HighlightTokenRange,
} from '@/lib/client/highlight-token-alignment'; } from '@/lib/client/highlight-token-alignment';
import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map';
export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence'; export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence';
export const HTML_WORD_CLASS = 'openreader-html-highlight-word'; export const HTML_WORD_CLASS = 'openreader-html-highlight-word';
@ -34,23 +35,33 @@ interface DomToken {
norm: string; norm: string;
} }
interface CharPosition {
node: Text;
offset: number;
}
let sentenceWraps: HTMLSpanElement[] = []; let sentenceWraps: HTMLSpanElement[] = [];
let wordWraps: HTMLSpanElement[] = []; let wordWraps: HTMLSpanElement[] = [];
/** /**
* Per-sentence state used by the word highlighter. Built once when the * Per-sentence state used by the word highlighter. Built once when the
* sentence wrap is applied and then read by every word-advance event, so we * sentence wrap is applied and then read by every word-advance event, so we
* don't re-walk the DOM or re-run the DP on every whisper tick. * don't re-walk the DOM on every whisper tick.
*/ */
interface SentenceState { interface SentenceState {
sentence: string; sentence: string;
// DOM tokens inside the wrapped sentence, captured AFTER the sentence wrap // Normalized char→DOM map of the sentence wrap. `chars[i]` is the DOM position
// is in place. Stable across word wrap/unwrap cycles because clear() calls // of the i-th character of `text`. Built AFTER the sentence wrap is in place;
// stable across word wrap/unwrap cycles because clear() calls
// `parent.normalize()` which restores the original text-node structure. // `parent.normalize()` which restores the original text-node structure.
wordTokens: DomToken[]; chars: CharPosition[];
// For an alignment we've already seen, the cached wordIndex → token range map. // Normalized text of the wrap (chars joined), used to locate the sentence so
// alignment char offsets can be rebased onto the wrap.
text: string;
// For an alignment we've already seen: the offset of `sentence` inside `text`
// (the wrap window may be slightly wider than the sentence). null = not found.
alignment: TTSSentenceAlignment | null; alignment: TTSSentenceAlignment | null;
wordToTokenRange: Array<HighlightTokenRange | null> | null; base: number | null;
} }
let sentenceState: SentenceState | null = null; let sentenceState: SentenceState | null = null;
@ -144,32 +155,31 @@ function collectDomTokens(
} }
/** /**
* Walk only inside the current sentence wrap spans (used after the sentence * Walk inside the current sentence wrap spans and build a normalized char→DOM
* wrap is applied; lets us index just the words *within* the highlighted * map. Every surviving character of the normalized text remembers the exact
* sentence rather than the whole document). * Text node + offset it came from, so alignment char offsets map straight to a
* DOM range. Normalization matches `preprocessSentenceForAudio` (the canonical
* space the alignment offsets live in).
*/ */
function collectTokensInsideWraps(wraps: HTMLSpanElement[], language?: string): DomToken[] { function collectWrapCharMap(wraps: HTMLSpanElement[]): { chars: CharPosition[]; text: string } {
const tokens: DomToken[] = []; const raw: MappedChar<CharPosition>[] = [];
for (const wrap of wraps) { for (const wrap of wraps) {
const walker = document.createTreeWalker(wrap, NodeFilter.SHOW_TEXT); const walker = document.createTreeWalker(wrap, NodeFilter.SHOW_TEXT);
let current: Node | null = walker.nextNode(); let current: Node | null = walker.nextNode();
while (current) { while (current) {
const t = current as Text; const t = current as Text;
const text = t.nodeValue || ''; const value = t.nodeValue || '';
for (const token of segmentWords(text, language)) { for (let offset = 0; offset < value.length; offset += 1) {
const norm = normalizeWord(token.text); raw.push({ char: value[offset], pos: { node: t, offset } });
if (!norm) continue;
tokens.push({
textNode: t,
startOffset: token.start,
endOffset: token.end,
norm,
});
} }
current = walker.nextNode(); current = walker.nextNode();
} }
} }
return tokens; const normalized = normalizeMappedChars(raw);
return {
chars: normalized.map((entry) => entry.pos),
text: normalized.map((entry) => entry.char).join(''),
};
} }
function findBestWindow(tokens: DomToken[], patternTokens: string[]): { start: number; end: number } | null { function findBestWindow(tokens: DomToken[], patternTokens: string[]): { start: number; end: number } | null {
@ -216,6 +226,50 @@ function wrapTokenRange(tokens: DomToken[], start: number, end: number, classNam
return wraps; return wraps;
} }
/**
* Wrap a half-open char range [start, end) of the sentence char map. Groups the
* covered characters by Text node (min/max offset per node) and splits/wraps
* each, mirroring `wrapTokenRange` but driven by per-character DOM positions.
*/
function wrapCharRange(chars: CharPosition[], start: number, end: number, className: string): HTMLSpanElement[] {
const perNode = new Map<Text, { start: number; end: number }>();
for (let i = start; i < end; i += 1) {
const pos = chars[i];
if (!pos) continue;
const existing = perNode.get(pos.node);
if (existing) {
existing.start = Math.min(existing.start, pos.offset);
existing.end = Math.max(existing.end, pos.offset + 1);
} else {
perNode.set(pos.node, { start: pos.offset, end: pos.offset + 1 });
}
}
const wraps: HTMLSpanElement[] = [];
for (const [textNode, { start: s, end: e }] of perNode) {
const parent = textNode.parentNode;
if (!parent) continue;
try {
let target: Text = textNode;
if (s > 0) {
target = target.splitText(s);
}
const innerLen = e - s;
if (innerLen < target.length) {
target.splitText(innerLen);
}
const span = document.createElement('span');
span.className = className;
parent.insertBefore(span, target);
span.appendChild(target);
wraps.push(span);
} catch {
// skip any text node that can't be split (already wrapped, detached, etc.)
}
}
return wraps;
}
export function highlightHtmlSentence( export function highlightHtmlSentence(
container: HTMLElement | null | undefined, container: HTMLElement | null | undefined,
sentence: string | null | undefined, sentence: string | null | undefined,
@ -236,13 +290,15 @@ export function highlightHtmlSentence(
sentenceWraps = wrapTokenRange(domTokens, win.start, win.end, HTML_SENTENCE_CLASS); sentenceWraps = wrapTokenRange(domTokens, win.start, win.end, HTML_SENTENCE_CLASS);
if (!sentenceWraps.length) return false; if (!sentenceWraps.length) return false;
// Capture the per-token DOM map AFTER the sentence wrap is in place so we // Capture the normalized char→DOM map AFTER the sentence wrap is in place so
// can look up individual word tokens without re-walking the doc. // we can resolve individual word offsets without re-walking the doc.
const { chars, text } = collectWrapCharMap(sentenceWraps);
sentenceState = { sentenceState = {
sentence, sentence,
wordTokens: collectTokensInsideWraps(sentenceWraps, language), chars,
text,
alignment: null, alignment: null,
wordToTokenRange: null, base: null,
}; };
return true; return true;
} }
@ -254,32 +310,46 @@ export function highlightHtmlWord(
): boolean { ): boolean {
// Always tear down the previous word wrap first. The `unwrap` call // Always tear down the previous word wrap first. The `unwrap` call
// normalizes the parent, restoring the post-sentence-wrap text-node // normalizes the parent, restoring the post-sentence-wrap text-node
// structure that `sentenceState.wordTokens` points at, so the cached map // structure that `sentenceState.chars` points at, so the cached map
// stays valid across consecutive word advances. // stays valid across consecutive word advances.
clearHtmlWordHighlight(); clearHtmlWordHighlight();
if (!container || !alignment) return false; if (!container || !alignment) return false;
if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return false; if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return false;
if (!sentenceState || !sentenceState.wordTokens.length) return false; if (!sentenceState || !sentenceState.chars.length) return false;
if (!sentenceWraps.length) return false; if (!sentenceWraps.length) return false;
const words = alignment.words || []; const words = alignment.words || [];
if (!words.length || wordIndex >= words.length) return false; if (!words.length || wordIndex >= words.length) return false;
// (Re)build the alignment map when this is a new alignment object. // Locate the sentence inside the wrap's normalized text once per alignment.
if (sentenceState.alignment !== alignment || !sentenceState.wordToTokenRange) { // The sentence wrap is found by fuzzy token windowing so it may be slightly
// wider than the spoken sentence; rebasing keeps the alignment char offsets
// accurate regardless.
if (sentenceState.alignment !== alignment || sentenceState.base === null) {
const found = sentenceState.text.indexOf(alignment.sentence);
if (found < 0) {
// Sentence not located in the wrap — fail closed (leave only the sentence
// highlight) rather than snapping the word highlight to offset 0. Don't
// commit the alignment so the next tick retries the lookup.
sentenceState.base = null;
return false;
}
sentenceState.alignment = alignment; sentenceState.alignment = alignment;
sentenceState.wordToTokenRange = buildAlignmentTokenRanges( sentenceState.base = found;
alignment.words,
sentenceState.wordTokens.map((token) => token.norm),
{ fillGaps: true },
);
} }
const tokenRange = sentenceState.wordToTokenRange[wordIndex]; const base = sentenceState.base;
if (!tokenRange) return false; if (base === null) return false;
if (tokenRange.start < 0 || tokenRange.end >= sentenceState.wordTokens.length) return false;
wordWraps = wrapTokenRange(sentenceState.wordTokens, tokenRange.start, tokenRange.end, HTML_WORD_CLASS); const word = words[wordIndex];
const { charStart, charEnd } = word;
if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charEnd <= charStart) return false;
const start = Math.max(0, base + charStart);
const end = Math.min(sentenceState.chars.length, base + charEnd);
if (end <= start) return false;
wordWraps = wrapCharRange(sentenceState.chars, start, end, HTML_WORD_CLASS);
return wordWraps.length > 0; return wordWraps.length > 0;
} }

7
src/lib/shared/nlp.ts
View file

@ -129,7 +129,12 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => {
}; };
/** /**
* Preprocesses text for audio generation by cleaning up various text artifacts * Preprocesses text for audio generation by cleaning up various text artifacts.
*
* Source of truth for the canonical "audio" text form. Keep byte-for-byte in
* lock-step with the copy in `compute/core/src/whisper/alignment-map.ts` (which
* computes word char offsets) and the position-preserving variant in
* `src/lib/client/highlight-char-map.ts` (which maps those offsets to the DOM).
* *
* @param {string} text - The text to preprocess * @param {string} text - The text to preprocess
* @returns {string} The cleaned text * @returns {string} The cleaned text

23
tests/unit/epub-word-highlight.vitest.spec.ts
View file

@ -2,7 +2,6 @@ import { describe, expect, test } from 'vitest';
import { import {
resolveAlignmentWordSourceRange, resolveAlignmentWordSourceRange,
tokenizeCanonicalSegment,
} from '../../src/lib/client/epub/epub-word-highlight'; } from '../../src/lib/client/epub/epub-word-highlight';
import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan'; import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan';
import type { TTSSentenceAlignment } from '../../src/types/tts'; import type { TTSSentenceAlignment } from '../../src/types/tts';
@ -19,16 +18,6 @@ const segment = (text: string, offset = 0): CanonicalTtsSegment => ({
}); });
describe('EPUB word highlight mapping', () => { describe('EPUB word highlight mapping', () => {
test('tokenizes Japanese and Chinese using locale-aware word boundaries', () => {
const japanese = tokenizeCanonicalSegment(segment('これは日本語です。', 5), 'ja');
expect(japanese.length).toBeGreaterThan(1);
expect(japanese.every((token) => token.norm.length > 0)).toBe(true);
const chinese = tokenizeCanonicalSegment(segment('这是中文。', 10), 'zh');
expect(chinese.length).toBeGreaterThan(1);
expect(chinese.map((token) => token.norm).join('')).toBe('这是中文');
});
test('resolves Japanese alignment chunks directly from character offsets', () => { test('resolves Japanese alignment chunks directly from character offsets', () => {
const japanese = segment('これは日本語です。', 25); const japanese = segment('これは日本語です。', 25);
const word: TTSSentenceAlignment['words'][number] = { const word: TTSSentenceAlignment['words'][number] = {
@ -45,7 +34,7 @@ describe('EPUB word highlight mapping', () => {
}); });
}); });
test('rejects invalid alignment character offsets so token mapping can be used', () => { test('rejects out-of-range alignment character offsets so the word is skipped', () => {
const japanese = segment('これは日本語です。', 25); const japanese = segment('これは日本語です。', 25);
const word: TTSSentenceAlignment['words'][number] = { const word: TTSSentenceAlignment['words'][number] = {
text: '範囲外', text: '範囲外',
@ -58,14 +47,4 @@ describe('EPUB word highlight mapping', () => {
expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull(); expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull();
}); });
test('tokenizes canonical segment words with source offsets', () => {
const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12));
expect(tokens).toEqual([
{ norm: 'hello', sourceStart: 13, sourceEnd: 18 },
{ norm: 'she', sourceStart: 21, sourceEnd: 24 },
{ norm: 'said', sourceStart: 25, sourceEnd: 29 },
]);
});
}); });

14
tests/unit/whisper-alignment-mapping.vitest.spec.ts
View file

@ -18,4 +18,18 @@ describe('whisper alignment mapping', () => {
expect(aligned.words[2].charStart).toBeGreaterThan(aligned.words[1].charEnd); expect(aligned.words[2].charStart).toBeGreaterThan(aligned.words[1].charEnd);
expect(aligned.words[2].charEnd).toBeLessThanOrEqual('Hello, world again.'.length); expect(aligned.words[2].charEnd).toBeLessThanOrEqual('Hello, world again.'.length);
}); });
test('joins line-break hyphenation across unicode letters', () => {
// "Über-\n mensch" must normalize to "Übermensch" so the offsets match the
// client char map. This only works with the unicode-aware hyphen regex that
// is kept in lock-step across nlp.ts / alignment-map.ts / highlight-char-map.ts.
const aligned = mapWordsToSentenceOffsets('Über-\n mensch walks', [
{ word: 'Übermensch', start: 0, end: 0.5 },
{ word: 'walks', start: 0.5, end: 1.0 },
]);
expect(aligned.words[0].charStart).toBe(0);
expect(aligned.words[0].charEnd).toBe('Übermensch'.length);
expect(aligned.words[1].charStart).toBeGreaterThanOrEqual(aligned.words[0].charEnd);
});
}); });