From ff5a6b834d9390a9589f65f9e6c254d13eadb118 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 08:20:53 -0600 Subject: [PATCH 1/2] fix(epub): unify TTS word highlight mapping and normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor EPUB and HTML word highlighting to use a shared, position-preserving text normalization pipeline for mapping TTS alignment offsets to DOM positions. Replace the previous fuzzy token-matching logic with a direct char→DOM map based on the canonical "audio" text form, ensuring highlights stay in sync across languages and with Unicode-aware hyphenation handling. - Add `highlight-char-map.ts` for shared normalization logic - Update normalization to handle Unicode hyphenation consistently across compute, client, and shared layers - Remove unused fuzzy token mapping and cache logic from EPUB highlighting - Update tests for new highlight mapping and normalization behavior BREAKING CHANGE: Removes legacy fuzzy token-based word highlight mapping in favor of direct char offset mapping; highlight cache and tokenization APIs are removed. --- compute/core/src/whisper/alignment-map.ts | 7 +- src/app/(app)/epub/[id]/useEpubDocument.ts | 4 - src/contexts/TTSContext.tsx | 7 + src/hooks/epub/useEPUBHighlighting.ts | 96 ++---------- .../client/epub/epub-rendered-text-maps.ts | 100 +----------- src/lib/client/epub/epub-word-highlight.ts | 44 +----- src/lib/client/highlight-char-map.ts | 118 ++++++++++++++ src/lib/client/html/highlight.ts | 148 ++++++++++++------ src/lib/shared/nlp.ts | 9 +- tests/unit/epub-word-highlight.vitest.spec.ts | 23 +-- .../whisper-alignment-mapping.vitest.spec.ts | 14 ++ 11 files changed, 282 insertions(+), 288 deletions(-) create mode 100644 src/lib/client/highlight-char-map.ts diff --git a/compute/core/src/whisper/alignment-map.ts b/compute/core/src/whisper/alignment-map.ts index b26ecc8..92b0303 100644 --- a/compute/core/src/whisper/alignment-map.ts +++ b/compute/core/src/whisper/alignment-map.ts @@ -1,9 +1,14 @@ import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; +// Keep byte-for-byte in lock-step with `preprocessSentenceForAudio` in +// `src/lib/shared/nlp.ts` (and the position-preserving copy in +// `src/lib/client/highlight-char-map.ts`). The word `charStart`/`charEnd` +// offsets this module emits are consumed against text normalized by those +// client functions, so any divergence shifts viewer highlights off-word. function preprocessSentenceForAudio(text: string): string { return text .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - .replace(/(\w+)-\s+(\w+)/g, '$1$2') + .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') .replace(/\*/g, '') .replace(/\s+/g, ' ') .trim(); diff --git a/src/app/(app)/epub/[id]/useEpubDocument.ts b/src/app/(app)/epub/[id]/useEpubDocument.ts index 5cc5427..ee5167f 100644 --- a/src/app/(app)/epub/[id]/useEpubDocument.ts +++ b/src/app/(app)/epub/[id]/useEpubDocument.ts @@ -36,7 +36,6 @@ import { } from '@/lib/client/epub/epub-rendered-text-maps'; import { useEPUBHighlighting, - type EpubWordHighlightMapCache, } from '@/hooks/epub/useEPUBHighlighting'; import { useEPUBLocationController } from '@/hooks/epub/useEPUBLocationController'; import { useEPUBAudiobook } from '@/hooks/epub/useEPUBAudiobook'; @@ -148,7 +147,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const currentHighlightCfi = useRef(null); const currentWordHighlightCfi = useRef(null); const renderedTextMapsRef = useRef([]); - const wordHighlightMapCacheRef = useRef(null); const renderedLocationCloneManagerRef = useRef( new EpubRenderedLocationCloneManager(), ); @@ -165,8 +163,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currentHighlightCfiRef: currentHighlightCfi, currentWordHighlightCfiRef: currentWordHighlightCfi, renderedTextMapsRef, - wordHighlightMapCacheRef, - language: resolvedLanguage, }); useEffect(() => () => { diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index de5aa24..e8431f4 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -2485,6 +2485,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const pos = activeHowl.seek() as number; if (typeof pos === 'number' && Number.isFinite(pos)) { const words = currentSentenceAlignment.words; + // Find the word whose timing window contains `pos`. When `pos` falls + // in a gap between words (or past the last word), snap to the last + // word that has already started instead of freezing on a stale index + // — monotonic, so it never jumps to a later word that hasn't started. let idx = -1; for (let i = 0; i < words.length; i++) { const w = words[i]; @@ -2492,6 +2496,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement idx = i; break; } + if (pos >= w.startSec) { + idx = i; + } } if (idx !== -1) { setCurrentWordIndex((prev) => (prev === idx ? prev : idx)); diff --git a/src/hooks/epub/useEPUBHighlighting.ts b/src/hooks/epub/useEPUBHighlighting.ts index 7362d3e..4e09753 100644 --- a/src/hooks/epub/useEPUBHighlighting.ts +++ b/src/hooks/epub/useEPUBHighlighting.ts @@ -4,15 +4,8 @@ import { useCallback, useEffect, type MutableRefObject, type RefObject } from 'r import type { Rendition } from 'epubjs'; import { - buildWordHighlightCacheKey, resolveAlignmentWordSourceRange, - tokenizeCanonicalSegment, - type EpubCanonicalWordToken, } from '@/lib/client/epub/epub-word-highlight'; -import { - buildAlignmentTokenRanges, - type HighlightTokenRange, -} from '@/lib/client/highlight-token-alignment'; import { createRangeFromMappedOffsets, resolveVisibleSegmentRange, @@ -21,20 +14,12 @@ import { import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { TTSSentenceAlignment } from '@/types/tts'; -export type EpubWordHighlightMapCache = { - key: string; - wordToTokenRange: Array; - tokens: EpubCanonicalWordToken[]; -}; - type UseEpubHighlightingParams = { renditionRef: RefObject; epubHighlightEnabled: boolean; currentHighlightCfiRef: MutableRefObject; currentWordHighlightCfiRef: MutableRefObject; renderedTextMapsRef: MutableRefObject; - wordHighlightMapCacheRef: MutableRefObject; - language?: string; }; type UseEpubHighlightingResult = { @@ -56,8 +41,6 @@ export function useEPUBHighlighting({ currentHighlightCfiRef, currentWordHighlightCfiRef, renderedTextMapsRef, - wordHighlightMapCacheRef, - language, }: UseEpubHighlightingParams): UseEpubHighlightingResult { const clearWordHighlights = useCallback(() => { if (!renditionRef.current) return; @@ -122,68 +105,19 @@ export function useEPUBHighlighting({ const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment); if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return; + // Native path only: the alignment's char offsets are authoritative for the + // spoken word (they live in the same canonical space as the segment text and + // the rendered char map). Clamp to the portion of the segment that is + // actually visible in this map so a word straddling a page/spread boundary + // still highlights its visible part instead of dropping the word entirely. const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]); - if ( - alignmentRange - && alignmentRange.sourceStart >= resolved.startOffset - && alignmentRange.sourceEnd <= resolved.endOffset - ) { - const wordRange = createRangeFromMappedOffsets( - resolved.map, - alignmentRange.sourceStart, - alignmentRange.sourceEnd, - ); - if (wordRange) { - try { - const wordCfi = resolved.map.content.cfiFromRange(wordRange); - currentWordHighlightCfiRef.current = wordCfi; - renditionRef.current.annotations.add( - 'highlight', - wordCfi, - {}, - () => { }, - '', - { - fill: 'var(--accent)', - 'fill-opacity': '0.4', - 'mix-blend-mode': 'multiply', - } - ); - return; - } catch (error) { - console.error('Error highlighting EPUB word from alignment offsets:', error); - } - } - } + if (!alignmentRange) return; - const cacheKey = buildWordHighlightCacheKey(segment, alignment, language); - if (wordHighlightMapCacheRef.current?.key !== cacheKey) { - const tokens = tokenizeCanonicalSegment(segment, language); - wordHighlightMapCacheRef.current = { - key: cacheKey, - tokens, - wordToTokenRange: buildAlignmentTokenRanges( - words, - tokens.map((token) => token.norm), - { minimumSimilarity: 0.8 }, - ), - }; - } + const clampedStart = Math.max(alignmentRange.sourceStart, resolved.startOffset); + const clampedEnd = Math.min(alignmentRange.sourceEnd, resolved.endOffset); + if (clampedEnd <= clampedStart) return; - const cached = wordHighlightMapCacheRef.current; - const tokenRange = cached.wordToTokenRange[wordIndex]; - if (!tokenRange) return; - - const firstToken = cached.tokens[tokenRange.start]; - const lastToken = cached.tokens[tokenRange.end]; - if (!firstToken || !lastToken) return; - if (firstToken.sourceStart < resolved.startOffset || lastToken.sourceEnd > resolved.endOffset) return; - - const wordRange = createRangeFromMappedOffsets( - resolved.map, - firstToken.sourceStart, - lastToken.sourceEnd, - ); + const wordRange = createRangeFromMappedOffsets(resolved.map, clampedStart, clampedEnd); if (!wordRange) return; try { @@ -202,7 +136,7 @@ export function useEPUBHighlighting({ } ); } catch (error) { - console.error('Error highlighting EPUB word:', error); + console.error('Error highlighting EPUB word from alignment offsets:', error); } }, [ clearWordHighlights, @@ -210,20 +144,16 @@ export function useEPUBHighlighting({ epubHighlightEnabled, renderedTextMapsRef, renditionRef, - wordHighlightMapCacheRef, - language, ]); const setRenderedTextMaps = useCallback((maps: EpubRenderedTextMap[]) => { renderedTextMapsRef.current = maps; - wordHighlightMapCacheRef.current = null; - }, [renderedTextMapsRef, wordHighlightMapCacheRef]); + }, [renderedTextMapsRef]); const resetHighlightState = useCallback(() => { renderedTextMapsRef.current = []; - wordHighlightMapCacheRef.current = null; clearHighlights(); - }, [clearHighlights, renderedTextMapsRef, wordHighlightMapCacheRef]); + }, [clearHighlights, renderedTextMapsRef]); // Clear any highlight annotations when feature is disabled. useEffect(() => { diff --git a/src/lib/client/epub/epub-rendered-text-maps.ts b/src/lib/client/epub/epub-rendered-text-maps.ts index 2c691cc..82f3072 100644 --- a/src/lib/client/epub/epub-rendered-text-maps.ts +++ b/src/lib/client/epub/epub-rendered-text-maps.ts @@ -2,16 +2,14 @@ import type { Rendition } from 'epubjs'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; +import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map'; type EpubMappedPosition = { node: Text; offset: number; }; -type EpubMappedChar = { - char: string; - position: EpubMappedPosition; -}; +type EpubMappedChar = MappedChar; export type EpubRenderedTextMap = { sourceKey: string; @@ -21,94 +19,6 @@ export type EpubRenderedTextMap = { }; }; -const cloneMappedChar = (char: string, source: EpubMappedChar): EpubMappedChar => ({ - char, - position: source.position, -}); - -const replaceMappedUrls = (tokens: EpubMappedChar[]): EpubMappedChar[] => { - const text = tokens.map((token) => token.char).join(''); - const urlPattern = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi; - const replaced: EpubMappedChar[] = []; - let cursor = 0; - let match: RegExpExecArray | null; - - while ((match = urlPattern.exec(text)) !== null) { - const start = match.index; - const end = start + match[0].length; - replaced.push(...tokens.slice(cursor, start)); - - const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)]; - if (anchor) { - const replacement = `- (link to ${match[1]}) -`; - for (const char of replacement) { - replaced.push(cloneMappedChar(char, anchor)); - } - } - cursor = end; - } - - replaced.push(...tokens.slice(cursor)); - return replaced; -}; - -const removeMappedHyphenation = (tokens: EpubMappedChar[]): EpubMappedChar[] => { - const text = tokens.map((token) => token.char).join(''); - const hyphenPattern = /(\w+)-\s+(\w+)/g; - const replaced: EpubMappedChar[] = []; - let cursor = 0; - let match: RegExpExecArray | null; - - while ((match = hyphenPattern.exec(text)) !== null) { - const start = match.index; - const full = match[0]; - const first = match[1]; - const second = match[2]; - const secondOffset = full.lastIndexOf(second); - - replaced.push(...tokens.slice(cursor, start)); - replaced.push(...tokens.slice(start, start + first.length)); - replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length)); - cursor = start + full.length; - } - - replaced.push(...tokens.slice(cursor)); - return replaced; -}; - -const normalizeMappedTokensForTts = (tokens: EpubMappedChar[]): EpubMappedChar[] => { - const withoutLinks = replaceMappedUrls(tokens); - const withoutHyphenation = removeMappedHyphenation(withoutLinks); - const normalized: EpubMappedChar[] = []; - let pendingWhitespace: EpubMappedChar | null = null; - - const flushWhitespace = () => { - if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') { - pendingWhitespace = null; - return; - } - normalized.push(cloneMappedChar(' ', pendingWhitespace)); - pendingWhitespace = null; - }; - - for (const token of withoutHyphenation) { - if (token.char === '*') continue; - if (/\s/.test(token.char)) { - pendingWhitespace ??= token; - continue; - } - - flushWhitespace(); - normalized.push(token); - } - - if (normalized[normalized.length - 1]?.char === ' ') { - normalized.pop(); - } - - return normalized; -}; - const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => { const root = range.commonAncestorContainer; const doc = range.startContainer.ownerDocument ?? (range.startContainer as Document); @@ -121,7 +31,7 @@ const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => { for (let offset = safeStart; offset < safeEnd; offset += 1) { mapped.push({ char: text[offset], - position: { node: textNode, offset }, + pos: { node: textNode, offset }, }); } }; @@ -178,12 +88,12 @@ export const buildRenderedTextMaps = ( const range = content.range(rangeCfi); if (!range) continue; - const normalized = normalizeMappedTokensForTts(collectMappedTextFromRange(range)); + const normalized = normalizeMappedChars(collectMappedTextFromRange(range)); if (!normalized.length) continue; maps.push({ sourceKey, - chars: normalized.map((token) => token.position), + chars: normalized.map((token) => token.pos), content, }); } catch { diff --git a/src/lib/client/epub/epub-word-highlight.ts b/src/lib/client/epub/epub-word-highlight.ts index de44021..ba5914b 100644 --- a/src/lib/client/epub/epub-word-highlight.ts +++ b/src/lib/client/epub/epub-word-highlight.ts @@ -1,17 +1,12 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; -import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; -import { segmentWords } from '@/lib/shared/language'; -import { normalizeHighlightToken } from '@/lib/client/highlight-token-alignment'; - -export type EpubCanonicalWordToken = { - norm: string; - sourceStart: number; - sourceEnd: number; -}; - -export const normalizeWordForHighlight = (text: string): string => - normalizeHighlightToken(text); +import type { TTSSentenceWord } from '@/types/tts'; +/** + * Resolve a spoken word's char offsets (from the Whisper alignment) into the + * canonical source-offset space the rendered text map is keyed by. The + * alignment's `charStart`/`charEnd` are offsets into the segment text, which is + * in the same normalized space as `segment.startAnchor.offset`. + */ export const resolveAlignmentWordSourceRange = ( segment: CanonicalTtsSegment, word: TTSSentenceWord, @@ -25,28 +20,3 @@ export const resolveAlignmentWordSourceRange = ( sourceEnd: segment.startAnchor.offset + charEnd, }; }; - -export const tokenizeCanonicalSegment = ( - segment: CanonicalTtsSegment, - language?: string, -): EpubCanonicalWordToken[] => - segmentWords(segment.text, language) - .map((token) => ({ - norm: normalizeWordForHighlight(token.text), - sourceStart: segment.startAnchor.offset + token.start, - sourceEnd: segment.startAnchor.offset + token.end, - })) - .filter((token) => Boolean(token.norm)); - -export const buildWordHighlightCacheKey = ( - segment: CanonicalTtsSegment, - alignment: TTSSentenceAlignment, - language?: string, -): string => - [ - segment.key, - segment.text.length, - language || '', - alignment.words.length, - alignment.words.map((word) => normalizeWordForHighlight(word.text)).join('|'), - ].join('::'); diff --git a/src/lib/client/highlight-char-map.ts b/src/lib/client/highlight-char-map.ts new file mode 100644 index 0000000..a5abcfe --- /dev/null +++ b/src/lib/client/highlight-char-map.ts @@ -0,0 +1,118 @@ +/** + * Shared, position-preserving text normalizer for viewer highlighting. + * + * TTS word offsets (`charStart`/`charEnd`) are computed against the canonical + * "audio" form of a sentence — URLs rewritten, line-break hyphenation joined, + * `*` stripped, whitespace collapsed. To map those offsets back onto the + * rendered DOM we have to normalize the DOM text the SAME way while remembering + * which DOM position each surviving character came from. + * + * This module operates on an opaque position type so both the EPUB renderer + * (position = `{ node, offset }` in an iframe) and the HTML/TXT renderer + * (position = a `Text` node + offset in the main document) share one identical + * normalization. Keep the transforms here in lock-step with + * `preprocessSentenceForAudio` in `src/lib/shared/nlp.ts` and the copy in + * `compute/core/src/whisper/alignment-map.ts`. + */ + +export interface MappedChar { + char: string; + pos: TPos; +} + +const URL_PATTERN = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi; +// Unicode-aware to match preprocessSentenceForAudio (nlp.ts / alignment-map.ts). +const HYPHENATION_PATTERN = /([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu; + +const cloneMappedChar = (char: string, source: MappedChar): MappedChar => ({ + char, + pos: source.pos, +}); + +const replaceMappedUrls = (tokens: MappedChar[]): MappedChar[] => { + const text = tokens.map((token) => token.char).join(''); + const replaced: MappedChar[] = []; + let cursor = 0; + let match: RegExpExecArray | null; + + URL_PATTERN.lastIndex = 0; + while ((match = URL_PATTERN.exec(text)) !== null) { + const start = match.index; + const end = start + match[0].length; + replaced.push(...tokens.slice(cursor, start)); + + const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)]; + if (anchor) { + const replacement = `- (link to ${match[1]}) -`; + for (const char of replacement) { + replaced.push(cloneMappedChar(char, anchor)); + } + } + cursor = end; + } + + replaced.push(...tokens.slice(cursor)); + return replaced; +}; + +const removeMappedHyphenation = (tokens: MappedChar[]): MappedChar[] => { + const text = tokens.map((token) => token.char).join(''); + const replaced: MappedChar[] = []; + let cursor = 0; + let match: RegExpExecArray | null; + + HYPHENATION_PATTERN.lastIndex = 0; + while ((match = HYPHENATION_PATTERN.exec(text)) !== null) { + const start = match.index; + const full = match[0]; + const first = match[1]; + const second = match[2]; + const secondOffset = full.lastIndexOf(second); + + replaced.push(...tokens.slice(cursor, start)); + replaced.push(...tokens.slice(start, start + first.length)); + replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length)); + cursor = start + full.length; + } + + replaced.push(...tokens.slice(cursor)); + return replaced; +}; + +/** + * Normalize a position-tagged character stream into the canonical TTS form, + * dropping/rewriting characters while preserving the source position of every + * surviving character. + */ +export const normalizeMappedChars = (tokens: MappedChar[]): MappedChar[] => { + const withoutLinks = replaceMappedUrls(tokens); + const withoutHyphenation = removeMappedHyphenation(withoutLinks); + const normalized: MappedChar[] = []; + let pendingWhitespace: MappedChar | null = null; + + const flushWhitespace = () => { + if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') { + pendingWhitespace = null; + return; + } + normalized.push(cloneMappedChar(' ', pendingWhitespace)); + pendingWhitespace = null; + }; + + for (const token of withoutHyphenation) { + if (token.char === '*') continue; + if (/\s/.test(token.char)) { + pendingWhitespace ??= token; + continue; + } + + flushWhitespace(); + normalized.push(token); + } + + if (normalized[normalized.length - 1]?.char === ' ') { + normalized.pop(); + } + + return normalized; +}; diff --git a/src/lib/client/html/highlight.ts b/src/lib/client/html/highlight.ts index a78a70a..baec7d8 100644 --- a/src/lib/client/html/highlight.ts +++ b/src/lib/client/html/highlight.ts @@ -11,18 +11,19 @@ * sentence * - `WORD` — saturated background on the currently-spoken word * - * Word-to-DOM alignment uses the shared viewer token-range mapper so DOM token - * counts that diverge from the timed alignment still produce a smooth, - * monotonic highlight across languages. + * Word-to-DOM alignment is native: the Whisper alignment gives authoritative + * `charStart`/`charEnd` offsets into the sentence text, and we map those offsets + * directly onto a normalized char→DOM map of the located sentence wrap. No fuzzy + * token matching for words — that path used to jump the highlight to a later + * word in the sentence. */ import type { TTSSentenceAlignment } from '@/types/tts'; import { segmentWords } from '@/lib/shared/language'; import { - buildAlignmentTokenRanges, findBestHighlightTokenMatch, normalizeHighlightToken, - type HighlightTokenRange, } from '@/lib/client/highlight-token-alignment'; +import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map'; export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence'; export const HTML_WORD_CLASS = 'openreader-html-highlight-word'; @@ -34,23 +35,33 @@ interface DomToken { norm: string; } +interface CharPosition { + node: Text; + offset: number; +} + let sentenceWraps: HTMLSpanElement[] = []; let wordWraps: HTMLSpanElement[] = []; /** * Per-sentence state used by the word highlighter. Built once when the * sentence wrap is applied and then read by every word-advance event, so we - * don't re-walk the DOM or re-run the DP on every whisper tick. + * don't re-walk the DOM on every whisper tick. */ interface SentenceState { sentence: string; - // DOM tokens inside the wrapped sentence, captured AFTER the sentence wrap - // is in place. Stable across word wrap/unwrap cycles because clear() calls + // Normalized char→DOM map of the sentence wrap. `chars[i]` is the DOM position + // of the i-th character of `text`. Built AFTER the sentence wrap is in place; + // stable across word wrap/unwrap cycles because clear() calls // `parent.normalize()` which restores the original text-node structure. - wordTokens: DomToken[]; - // For an alignment we've already seen, the cached wordIndex → token range map. + chars: CharPosition[]; + // Normalized text of the wrap (chars joined), used to locate the sentence so + // alignment char offsets can be rebased onto the wrap. + text: string; + // For an alignment we've already seen: the offset of `sentence` inside `text` + // (the wrap window may be slightly wider than the sentence). null = not found. alignment: TTSSentenceAlignment | null; - wordToTokenRange: Array | null; + base: number | null; } let sentenceState: SentenceState | null = null; @@ -144,32 +155,31 @@ function collectDomTokens( } /** - * Walk only inside the current sentence wrap spans (used after the sentence - * wrap is applied; lets us index just the words *within* the highlighted - * sentence rather than the whole document). + * Walk inside the current sentence wrap spans and build a normalized char→DOM + * map. Every surviving character of the normalized text remembers the exact + * Text node + offset it came from, so alignment char offsets map straight to a + * DOM range. Normalization matches `preprocessSentenceForAudio` (the canonical + * space the alignment offsets live in). */ -function collectTokensInsideWraps(wraps: HTMLSpanElement[], language?: string): DomToken[] { - const tokens: DomToken[] = []; +function collectWrapCharMap(wraps: HTMLSpanElement[]): { chars: CharPosition[]; text: string } { + const raw: MappedChar[] = []; for (const wrap of wraps) { const walker = document.createTreeWalker(wrap, NodeFilter.SHOW_TEXT); let current: Node | null = walker.nextNode(); while (current) { const t = current as Text; - const text = t.nodeValue || ''; - for (const token of segmentWords(text, language)) { - const norm = normalizeWord(token.text); - if (!norm) continue; - tokens.push({ - textNode: t, - startOffset: token.start, - endOffset: token.end, - norm, - }); + const value = t.nodeValue || ''; + for (let offset = 0; offset < value.length; offset += 1) { + raw.push({ char: value[offset], pos: { node: t, offset } }); } current = walker.nextNode(); } } - return tokens; + const normalized = normalizeMappedChars(raw); + return { + chars: normalized.map((entry) => entry.pos), + text: normalized.map((entry) => entry.char).join(''), + }; } function findBestWindow(tokens: DomToken[], patternTokens: string[]): { start: number; end: number } | null { @@ -216,6 +226,50 @@ function wrapTokenRange(tokens: DomToken[], start: number, end: number, classNam return wraps; } +/** + * Wrap a half-open char range [start, end) of the sentence char map. Groups the + * covered characters by Text node (min/max offset per node) and splits/wraps + * each, mirroring `wrapTokenRange` but driven by per-character DOM positions. + */ +function wrapCharRange(chars: CharPosition[], start: number, end: number, className: string): HTMLSpanElement[] { + const perNode = new Map(); + for (let i = start; i < end; i += 1) { + const pos = chars[i]; + if (!pos) continue; + const existing = perNode.get(pos.node); + if (existing) { + existing.start = Math.min(existing.start, pos.offset); + existing.end = Math.max(existing.end, pos.offset + 1); + } else { + perNode.set(pos.node, { start: pos.offset, end: pos.offset + 1 }); + } + } + + const wraps: HTMLSpanElement[] = []; + for (const [textNode, { start: s, end: e }] of perNode) { + const parent = textNode.parentNode; + if (!parent) continue; + try { + let target: Text = textNode; + if (s > 0) { + target = target.splitText(s); + } + const innerLen = e - s; + if (innerLen < target.length) { + target.splitText(innerLen); + } + const span = document.createElement('span'); + span.className = className; + parent.insertBefore(span, target); + span.appendChild(target); + wraps.push(span); + } catch { + // skip any text node that can't be split (already wrapped, detached, etc.) + } + } + return wraps; +} + export function highlightHtmlSentence( container: HTMLElement | null | undefined, sentence: string | null | undefined, @@ -236,13 +290,15 @@ export function highlightHtmlSentence( sentenceWraps = wrapTokenRange(domTokens, win.start, win.end, HTML_SENTENCE_CLASS); if (!sentenceWraps.length) return false; - // Capture the per-token DOM map AFTER the sentence wrap is in place so we - // can look up individual word tokens without re-walking the doc. + // Capture the normalized char→DOM map AFTER the sentence wrap is in place so + // we can resolve individual word offsets without re-walking the doc. + const { chars, text } = collectWrapCharMap(sentenceWraps); sentenceState = { sentence, - wordTokens: collectTokensInsideWraps(sentenceWraps, language), + chars, + text, alignment: null, - wordToTokenRange: null, + base: null, }; return true; } @@ -254,32 +310,36 @@ export function highlightHtmlWord( ): boolean { // Always tear down the previous word wrap first. The `unwrap` call // normalizes the parent, restoring the post-sentence-wrap text-node - // structure that `sentenceState.wordTokens` points at, so the cached map + // structure that `sentenceState.chars` points at, so the cached map // stays valid across consecutive word advances. clearHtmlWordHighlight(); if (!container || !alignment) return false; if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return false; - if (!sentenceState || !sentenceState.wordTokens.length) return false; + if (!sentenceState || !sentenceState.chars.length) return false; if (!sentenceWraps.length) return false; const words = alignment.words || []; if (!words.length || wordIndex >= words.length) return false; - // (Re)build the alignment map when this is a new alignment object. - if (sentenceState.alignment !== alignment || !sentenceState.wordToTokenRange) { + // Locate the sentence inside the wrap's normalized text once per alignment. + // The sentence wrap is found by fuzzy token windowing so it may be slightly + // wider than the spoken sentence; rebasing keeps the alignment char offsets + // accurate regardless. + if (sentenceState.alignment !== alignment || sentenceState.base === null) { sentenceState.alignment = alignment; - sentenceState.wordToTokenRange = buildAlignmentTokenRanges( - alignment.words, - sentenceState.wordTokens.map((token) => token.norm), - { fillGaps: true }, - ); + const found = sentenceState.text.indexOf(alignment.sentence); + sentenceState.base = found >= 0 ? found : 0; } - const tokenRange = sentenceState.wordToTokenRange[wordIndex]; - if (!tokenRange) return false; - if (tokenRange.start < 0 || tokenRange.end >= sentenceState.wordTokens.length) return false; + const word = words[wordIndex]; + const { charStart, charEnd } = word; + if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charEnd <= charStart) return false; - wordWraps = wrapTokenRange(sentenceState.wordTokens, tokenRange.start, tokenRange.end, HTML_WORD_CLASS); + const start = Math.max(0, sentenceState.base + charStart); + const end = Math.min(sentenceState.chars.length, sentenceState.base + charEnd); + if (end <= start) return false; + + wordWraps = wrapCharRange(sentenceState.chars, start, end, HTML_WORD_CLASS); return wordWraps.length > 0; } diff --git a/src/lib/shared/nlp.ts b/src/lib/shared/nlp.ts index 6712116..3a8468d 100644 --- a/src/lib/shared/nlp.ts +++ b/src/lib/shared/nlp.ts @@ -129,8 +129,13 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => { }; /** - * Preprocesses text for audio generation by cleaning up various text artifacts - * + * Preprocesses text for audio generation by cleaning up various text artifacts. + * + * Source of truth for the canonical "audio" text form. Keep byte-for-byte in + * lock-step with the copy in `compute/core/src/whisper/alignment-map.ts` (which + * computes word char offsets) and the position-preserving variant in + * `src/lib/client/highlight-char-map.ts` (which maps those offsets to the DOM). + * * @param {string} text - The text to preprocess * @returns {string} The cleaned text */ diff --git a/tests/unit/epub-word-highlight.vitest.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts index 6d0e3cc..23f6d77 100644 --- a/tests/unit/epub-word-highlight.vitest.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -2,7 +2,6 @@ import { describe, expect, test } from 'vitest'; import { resolveAlignmentWordSourceRange, - tokenizeCanonicalSegment, } from '../../src/lib/client/epub/epub-word-highlight'; import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan'; import type { TTSSentenceAlignment } from '../../src/types/tts'; @@ -19,16 +18,6 @@ const segment = (text: string, offset = 0): CanonicalTtsSegment => ({ }); describe('EPUB word highlight mapping', () => { - test('tokenizes Japanese and Chinese using locale-aware word boundaries', () => { - const japanese = tokenizeCanonicalSegment(segment('これは日本語です。', 5), 'ja'); - expect(japanese.length).toBeGreaterThan(1); - expect(japanese.every((token) => token.norm.length > 0)).toBe(true); - - const chinese = tokenizeCanonicalSegment(segment('这是中文。', 10), 'zh'); - expect(chinese.length).toBeGreaterThan(1); - expect(chinese.map((token) => token.norm).join('')).toBe('这是中文'); - }); - test('resolves Japanese alignment chunks directly from character offsets', () => { const japanese = segment('これは日本語です。', 25); const word: TTSSentenceAlignment['words'][number] = { @@ -45,7 +34,7 @@ describe('EPUB word highlight mapping', () => { }); }); - test('rejects invalid alignment character offsets so token mapping can be used', () => { + test('rejects out-of-range alignment character offsets so the word is skipped', () => { const japanese = segment('これは日本語です。', 25); const word: TTSSentenceAlignment['words'][number] = { text: '範囲外', @@ -58,14 +47,4 @@ describe('EPUB word highlight mapping', () => { expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull(); }); - test('tokenizes canonical segment words with source offsets', () => { - const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); - - expect(tokens).toEqual([ - { norm: 'hello', sourceStart: 13, sourceEnd: 18 }, - { norm: 'she', sourceStart: 21, sourceEnd: 24 }, - { norm: 'said', sourceStart: 25, sourceEnd: 29 }, - ]); - }); - }); diff --git a/tests/unit/whisper-alignment-mapping.vitest.spec.ts b/tests/unit/whisper-alignment-mapping.vitest.spec.ts index b53a4f6..e5c9575 100644 --- a/tests/unit/whisper-alignment-mapping.vitest.spec.ts +++ b/tests/unit/whisper-alignment-mapping.vitest.spec.ts @@ -18,4 +18,18 @@ describe('whisper alignment mapping', () => { expect(aligned.words[2].charStart).toBeGreaterThan(aligned.words[1].charEnd); expect(aligned.words[2].charEnd).toBeLessThanOrEqual('Hello, world again.'.length); }); + + test('joins line-break hyphenation across unicode letters', () => { + // "Über-\n mensch" must normalize to "Übermensch" so the offsets match the + // client char map. This only works with the unicode-aware hyphen regex that + // is kept in lock-step across nlp.ts / alignment-map.ts / highlight-char-map.ts. + const aligned = mapWordsToSentenceOffsets('Über-\n mensch walks', [ + { word: 'Übermensch', start: 0, end: 0.5 }, + { word: 'walks', start: 0.5, end: 1.0 }, + ]); + + expect(aligned.words[0].charStart).toBe(0); + expect(aligned.words[0].charEnd).toBe('Übermensch'.length); + expect(aligned.words[1].charStart).toBeGreaterThanOrEqual(aligned.words[0].charEnd); + }); }); From a2cdc575ef79df605ac8e9345be827591fd44ecd Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 9 Jun 2026 09:01:48 -0600 Subject: [PATCH 2/2] fix(highlight): improve URL replacement spread and sentence alignment fallback Distribute replacement characters across original URL spans to preserve positional mapping in highlights. In sentence alignment, add fallback to avoid misaligned word highlights when the sentence cannot be located, ensuring robustness in TTS word highlighting. --- src/lib/client/highlight-char-map.ts | 10 ++++++++-- src/lib/client/html/highlight.ts | 18 ++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/lib/client/highlight-char-map.ts b/src/lib/client/highlight-char-map.ts index a5abcfe..54e9814 100644 --- a/src/lib/client/highlight-char-map.ts +++ b/src/lib/client/highlight-char-map.ts @@ -44,8 +44,14 @@ const replaceMappedUrls = (tokens: MappedChar[]): MappedChar[] const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)]; if (anchor) { const replacement = `- (link to ${match[1]}) -`; - for (const char of replacement) { - replaced.push(cloneMappedChar(char, anchor)); + // Spread the replacement characters across the original URL span so the + // mapped positions keep their positional spread instead of collapsing the + // whole URL onto a single DOM anchor. + const originalLength = Math.max(1, end - start); + for (let i = 0; i < replacement.length; i += 1) { + const sourceIndex = Math.min(end - 1, start + Math.floor((i * originalLength) / replacement.length)); + const source = tokens[sourceIndex] ?? anchor; + replaced.push(cloneMappedChar(replacement[i], source)); } } cursor = end; diff --git a/src/lib/client/html/highlight.ts b/src/lib/client/html/highlight.ts index baec7d8..f9a35f6 100644 --- a/src/lib/client/html/highlight.ts +++ b/src/lib/client/html/highlight.ts @@ -326,17 +326,27 @@ export function highlightHtmlWord( // wider than the spoken sentence; rebasing keeps the alignment char offsets // accurate regardless. if (sentenceState.alignment !== alignment || sentenceState.base === null) { - sentenceState.alignment = alignment; const found = sentenceState.text.indexOf(alignment.sentence); - sentenceState.base = found >= 0 ? found : 0; + if (found < 0) { + // Sentence not located in the wrap — fail closed (leave only the sentence + // highlight) rather than snapping the word highlight to offset 0. Don't + // commit the alignment so the next tick retries the lookup. + sentenceState.base = null; + return false; + } + sentenceState.alignment = alignment; + sentenceState.base = found; } + const base = sentenceState.base; + if (base === null) return false; + const word = words[wordIndex]; const { charStart, charEnd } = word; if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charEnd <= charStart) return false; - const start = Math.max(0, sentenceState.base + charStart); - const end = Math.min(sentenceState.chars.length, sentenceState.base + charEnd); + const start = Math.max(0, base + charStart); + const end = Math.min(sentenceState.chars.length, base + charEnd); if (end <= start) return false; wordWraps = wrapCharRange(sentenceState.chars, start, end, HTML_WORD_CLASS);