diff --git a/compute/core/src/whisper/alignment-map.ts b/compute/core/src/whisper/alignment-map.ts index b26ecc8..92b0303 100644 --- a/compute/core/src/whisper/alignment-map.ts +++ b/compute/core/src/whisper/alignment-map.ts @@ -1,9 +1,14 @@ import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; +// Keep byte-for-byte in lock-step with `preprocessSentenceForAudio` in +// `src/lib/shared/nlp.ts` (and the position-preserving copy in +// `src/lib/client/highlight-char-map.ts`). The word `charStart`/`charEnd` +// offsets this module emits are consumed against text normalized by those +// client functions, so any divergence shifts viewer highlights off-word. function preprocessSentenceForAudio(text: string): string { return text .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - .replace(/(\w+)-\s+(\w+)/g, '$1$2') + .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') .replace(/\*/g, '') .replace(/\s+/g, ' ') .trim(); diff --git a/src/app/(app)/epub/[id]/useEpubDocument.ts b/src/app/(app)/epub/[id]/useEpubDocument.ts index 5cc5427..ee5167f 100644 --- a/src/app/(app)/epub/[id]/useEpubDocument.ts +++ b/src/app/(app)/epub/[id]/useEpubDocument.ts @@ -36,7 +36,6 @@ import { } from '@/lib/client/epub/epub-rendered-text-maps'; import { useEPUBHighlighting, - type EpubWordHighlightMapCache, } from '@/hooks/epub/useEPUBHighlighting'; import { useEPUBLocationController } from '@/hooks/epub/useEPUBLocationController'; import { useEPUBAudiobook } from '@/hooks/epub/useEPUBAudiobook'; @@ -148,7 +147,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const currentHighlightCfi = useRef(null); const currentWordHighlightCfi = useRef(null); const renderedTextMapsRef = useRef([]); - const wordHighlightMapCacheRef = useRef(null); const renderedLocationCloneManagerRef = useRef( new EpubRenderedLocationCloneManager(), ); @@ -165,8 +163,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currentHighlightCfiRef: currentHighlightCfi, currentWordHighlightCfiRef: currentWordHighlightCfi, renderedTextMapsRef, - wordHighlightMapCacheRef, - language: resolvedLanguage, }); useEffect(() => () => { diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index de5aa24..e8431f4 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -2485,6 +2485,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const pos = activeHowl.seek() as number; if (typeof pos === 'number' && Number.isFinite(pos)) { const words = currentSentenceAlignment.words; + // Find the word whose timing window contains `pos`. When `pos` falls + // in a gap between words (or past the last word), snap to the last + // word that has already started instead of freezing on a stale index + // — monotonic, so it never jumps to a later word that hasn't started. let idx = -1; for (let i = 0; i < words.length; i++) { const w = words[i]; @@ -2492,6 +2496,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement idx = i; break; } + if (pos >= w.startSec) { + idx = i; + } } if (idx !== -1) { setCurrentWordIndex((prev) => (prev === idx ? prev : idx)); diff --git a/src/hooks/epub/useEPUBHighlighting.ts b/src/hooks/epub/useEPUBHighlighting.ts index 7362d3e..4e09753 100644 --- a/src/hooks/epub/useEPUBHighlighting.ts +++ b/src/hooks/epub/useEPUBHighlighting.ts @@ -4,15 +4,8 @@ import { useCallback, useEffect, type MutableRefObject, type RefObject } from 'r import type { Rendition } from 'epubjs'; import { - buildWordHighlightCacheKey, resolveAlignmentWordSourceRange, - tokenizeCanonicalSegment, - type EpubCanonicalWordToken, } from '@/lib/client/epub/epub-word-highlight'; -import { - buildAlignmentTokenRanges, - type HighlightTokenRange, -} from '@/lib/client/highlight-token-alignment'; import { createRangeFromMappedOffsets, resolveVisibleSegmentRange, @@ -21,20 +14,12 @@ import { import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { TTSSentenceAlignment } from '@/types/tts'; -export type EpubWordHighlightMapCache = { - key: string; - wordToTokenRange: Array; - tokens: EpubCanonicalWordToken[]; -}; - type UseEpubHighlightingParams = { renditionRef: RefObject; epubHighlightEnabled: boolean; currentHighlightCfiRef: MutableRefObject; currentWordHighlightCfiRef: MutableRefObject; renderedTextMapsRef: MutableRefObject; - wordHighlightMapCacheRef: MutableRefObject; - language?: string; }; type UseEpubHighlightingResult = { @@ -56,8 +41,6 @@ export function useEPUBHighlighting({ currentHighlightCfiRef, currentWordHighlightCfiRef, renderedTextMapsRef, - wordHighlightMapCacheRef, - language, }: UseEpubHighlightingParams): UseEpubHighlightingResult { const clearWordHighlights = useCallback(() => { if (!renditionRef.current) return; @@ -122,68 +105,19 @@ export function useEPUBHighlighting({ const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment); if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return; + // Native path only: the alignment's char offsets are authoritative for the + // spoken word (they live in the same canonical space as the segment text and + // the rendered char map). Clamp to the portion of the segment that is + // actually visible in this map so a word straddling a page/spread boundary + // still highlights its visible part instead of dropping the word entirely. const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]); - if ( - alignmentRange - && alignmentRange.sourceStart >= resolved.startOffset - && alignmentRange.sourceEnd <= resolved.endOffset - ) { - const wordRange = createRangeFromMappedOffsets( - resolved.map, - alignmentRange.sourceStart, - alignmentRange.sourceEnd, - ); - if (wordRange) { - try { - const wordCfi = resolved.map.content.cfiFromRange(wordRange); - currentWordHighlightCfiRef.current = wordCfi; - renditionRef.current.annotations.add( - 'highlight', - wordCfi, - {}, - () => { }, - '', - { - fill: 'var(--accent)', - 'fill-opacity': '0.4', - 'mix-blend-mode': 'multiply', - } - ); - return; - } catch (error) { - console.error('Error highlighting EPUB word from alignment offsets:', error); - } - } - } + if (!alignmentRange) return; - const cacheKey = buildWordHighlightCacheKey(segment, alignment, language); - if (wordHighlightMapCacheRef.current?.key !== cacheKey) { - const tokens = tokenizeCanonicalSegment(segment, language); - wordHighlightMapCacheRef.current = { - key: cacheKey, - tokens, - wordToTokenRange: buildAlignmentTokenRanges( - words, - tokens.map((token) => token.norm), - { minimumSimilarity: 0.8 }, - ), - }; - } + const clampedStart = Math.max(alignmentRange.sourceStart, resolved.startOffset); + const clampedEnd = Math.min(alignmentRange.sourceEnd, resolved.endOffset); + if (clampedEnd <= clampedStart) return; - const cached = wordHighlightMapCacheRef.current; - const tokenRange = cached.wordToTokenRange[wordIndex]; - if (!tokenRange) return; - - const firstToken = cached.tokens[tokenRange.start]; - const lastToken = cached.tokens[tokenRange.end]; - if (!firstToken || !lastToken) return; - if (firstToken.sourceStart < resolved.startOffset || lastToken.sourceEnd > resolved.endOffset) return; - - const wordRange = createRangeFromMappedOffsets( - resolved.map, - firstToken.sourceStart, - lastToken.sourceEnd, - ); + const wordRange = createRangeFromMappedOffsets(resolved.map, clampedStart, clampedEnd); if (!wordRange) return; try { @@ -202,7 +136,7 @@ export function useEPUBHighlighting({ } ); } catch (error) { - console.error('Error highlighting EPUB word:', error); + console.error('Error highlighting EPUB word from alignment offsets:', error); } }, [ clearWordHighlights, @@ -210,20 +144,16 @@ export function useEPUBHighlighting({ epubHighlightEnabled, renderedTextMapsRef, renditionRef, - wordHighlightMapCacheRef, - language, ]); const setRenderedTextMaps = useCallback((maps: EpubRenderedTextMap[]) => { renderedTextMapsRef.current = maps; - wordHighlightMapCacheRef.current = null; - }, [renderedTextMapsRef, wordHighlightMapCacheRef]); + }, [renderedTextMapsRef]); const resetHighlightState = useCallback(() => { renderedTextMapsRef.current = []; - wordHighlightMapCacheRef.current = null; clearHighlights(); - }, [clearHighlights, renderedTextMapsRef, wordHighlightMapCacheRef]); + }, [clearHighlights, renderedTextMapsRef]); // Clear any highlight annotations when feature is disabled. useEffect(() => { diff --git a/src/lib/client/epub/epub-rendered-text-maps.ts b/src/lib/client/epub/epub-rendered-text-maps.ts index 2c691cc..82f3072 100644 --- a/src/lib/client/epub/epub-rendered-text-maps.ts +++ b/src/lib/client/epub/epub-rendered-text-maps.ts @@ -2,16 +2,14 @@ import type { Rendition } from 'epubjs'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; +import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map'; type EpubMappedPosition = { node: Text; offset: number; }; -type EpubMappedChar = { - char: string; - position: EpubMappedPosition; -}; +type EpubMappedChar = MappedChar; export type EpubRenderedTextMap = { sourceKey: string; @@ -21,94 +19,6 @@ export type EpubRenderedTextMap = { }; }; -const cloneMappedChar = (char: string, source: EpubMappedChar): EpubMappedChar => ({ - char, - position: source.position, -}); - -const replaceMappedUrls = (tokens: EpubMappedChar[]): EpubMappedChar[] => { - const text = tokens.map((token) => token.char).join(''); - const urlPattern = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi; - const replaced: EpubMappedChar[] = []; - let cursor = 0; - let match: RegExpExecArray | null; - - while ((match = urlPattern.exec(text)) !== null) { - const start = match.index; - const end = start + match[0].length; - replaced.push(...tokens.slice(cursor, start)); - - const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)]; - if (anchor) { - const replacement = `- (link to ${match[1]}) -`; - for (const char of replacement) { - replaced.push(cloneMappedChar(char, anchor)); - } - } - cursor = end; - } - - replaced.push(...tokens.slice(cursor)); - return replaced; -}; - -const removeMappedHyphenation = (tokens: EpubMappedChar[]): EpubMappedChar[] => { - const text = tokens.map((token) => token.char).join(''); - const hyphenPattern = /(\w+)-\s+(\w+)/g; - const replaced: EpubMappedChar[] = []; - let cursor = 0; - let match: RegExpExecArray | null; - - while ((match = hyphenPattern.exec(text)) !== null) { - const start = match.index; - const full = match[0]; - const first = match[1]; - const second = match[2]; - const secondOffset = full.lastIndexOf(second); - - replaced.push(...tokens.slice(cursor, start)); - replaced.push(...tokens.slice(start, start + first.length)); - replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length)); - cursor = start + full.length; - } - - replaced.push(...tokens.slice(cursor)); - return replaced; -}; - -const normalizeMappedTokensForTts = (tokens: EpubMappedChar[]): EpubMappedChar[] => { - const withoutLinks = replaceMappedUrls(tokens); - const withoutHyphenation = removeMappedHyphenation(withoutLinks); - const normalized: EpubMappedChar[] = []; - let pendingWhitespace: EpubMappedChar | null = null; - - const flushWhitespace = () => { - if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') { - pendingWhitespace = null; - return; - } - normalized.push(cloneMappedChar(' ', pendingWhitespace)); - pendingWhitespace = null; - }; - - for (const token of withoutHyphenation) { - if (token.char === '*') continue; - if (/\s/.test(token.char)) { - pendingWhitespace ??= token; - continue; - } - - flushWhitespace(); - normalized.push(token); - } - - if (normalized[normalized.length - 1]?.char === ' ') { - normalized.pop(); - } - - return normalized; -}; - const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => { const root = range.commonAncestorContainer; const doc = range.startContainer.ownerDocument ?? (range.startContainer as Document); @@ -121,7 +31,7 @@ const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => { for (let offset = safeStart; offset < safeEnd; offset += 1) { mapped.push({ char: text[offset], - position: { node: textNode, offset }, + pos: { node: textNode, offset }, }); } }; @@ -178,12 +88,12 @@ export const buildRenderedTextMaps = ( const range = content.range(rangeCfi); if (!range) continue; - const normalized = normalizeMappedTokensForTts(collectMappedTextFromRange(range)); + const normalized = normalizeMappedChars(collectMappedTextFromRange(range)); if (!normalized.length) continue; maps.push({ sourceKey, - chars: normalized.map((token) => token.position), + chars: normalized.map((token) => token.pos), content, }); } catch { diff --git a/src/lib/client/epub/epub-word-highlight.ts b/src/lib/client/epub/epub-word-highlight.ts index de44021..ba5914b 100644 --- a/src/lib/client/epub/epub-word-highlight.ts +++ b/src/lib/client/epub/epub-word-highlight.ts @@ -1,17 +1,12 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; -import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; -import { segmentWords } from '@/lib/shared/language'; -import { normalizeHighlightToken } from '@/lib/client/highlight-token-alignment'; - -export type EpubCanonicalWordToken = { - norm: string; - sourceStart: number; - sourceEnd: number; -}; - -export const normalizeWordForHighlight = (text: string): string => - normalizeHighlightToken(text); +import type { TTSSentenceWord } from '@/types/tts'; +/** + * Resolve a spoken word's char offsets (from the Whisper alignment) into the + * canonical source-offset space the rendered text map is keyed by. The + * alignment's `charStart`/`charEnd` are offsets into the segment text, which is + * in the same normalized space as `segment.startAnchor.offset`. + */ export const resolveAlignmentWordSourceRange = ( segment: CanonicalTtsSegment, word: TTSSentenceWord, @@ -25,28 +20,3 @@ export const resolveAlignmentWordSourceRange = ( sourceEnd: segment.startAnchor.offset + charEnd, }; }; - -export const tokenizeCanonicalSegment = ( - segment: CanonicalTtsSegment, - language?: string, -): EpubCanonicalWordToken[] => - segmentWords(segment.text, language) - .map((token) => ({ - norm: normalizeWordForHighlight(token.text), - sourceStart: segment.startAnchor.offset + token.start, - sourceEnd: segment.startAnchor.offset + token.end, - })) - .filter((token) => Boolean(token.norm)); - -export const buildWordHighlightCacheKey = ( - segment: CanonicalTtsSegment, - alignment: TTSSentenceAlignment, - language?: string, -): string => - [ - segment.key, - segment.text.length, - language || '', - alignment.words.length, - alignment.words.map((word) => normalizeWordForHighlight(word.text)).join('|'), - ].join('::'); diff --git a/src/lib/client/highlight-char-map.ts b/src/lib/client/highlight-char-map.ts new file mode 100644 index 0000000..54e9814 --- /dev/null +++ b/src/lib/client/highlight-char-map.ts @@ -0,0 +1,124 @@ +/** + * Shared, position-preserving text normalizer for viewer highlighting. + * + * TTS word offsets (`charStart`/`charEnd`) are computed against the canonical + * "audio" form of a sentence — URLs rewritten, line-break hyphenation joined, + * `*` stripped, whitespace collapsed. To map those offsets back onto the + * rendered DOM we have to normalize the DOM text the SAME way while remembering + * which DOM position each surviving character came from. + * + * This module operates on an opaque position type so both the EPUB renderer + * (position = `{ node, offset }` in an iframe) and the HTML/TXT renderer + * (position = a `Text` node + offset in the main document) share one identical + * normalization. Keep the transforms here in lock-step with + * `preprocessSentenceForAudio` in `src/lib/shared/nlp.ts` and the copy in + * `compute/core/src/whisper/alignment-map.ts`. + */ + +export interface MappedChar { + char: string; + pos: TPos; +} + +const URL_PATTERN = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi; +// Unicode-aware to match preprocessSentenceForAudio (nlp.ts / alignment-map.ts). +const HYPHENATION_PATTERN = /([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu; + +const cloneMappedChar = (char: string, source: MappedChar): MappedChar => ({ + char, + pos: source.pos, +}); + +const replaceMappedUrls = (tokens: MappedChar[]): MappedChar[] => { + const text = tokens.map((token) => token.char).join(''); + const replaced: MappedChar[] = []; + let cursor = 0; + let match: RegExpExecArray | null; + + URL_PATTERN.lastIndex = 0; + while ((match = URL_PATTERN.exec(text)) !== null) { + const start = match.index; + const end = start + match[0].length; + replaced.push(...tokens.slice(cursor, start)); + + const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)]; + if (anchor) { + const replacement = `- (link to ${match[1]}) -`; + // Spread the replacement characters across the original URL span so the + // mapped positions keep their positional spread instead of collapsing the + // whole URL onto a single DOM anchor. + const originalLength = Math.max(1, end - start); + for (let i = 0; i < replacement.length; i += 1) { + const sourceIndex = Math.min(end - 1, start + Math.floor((i * originalLength) / replacement.length)); + const source = tokens[sourceIndex] ?? anchor; + replaced.push(cloneMappedChar(replacement[i], source)); + } + } + cursor = end; + } + + replaced.push(...tokens.slice(cursor)); + return replaced; +}; + +const removeMappedHyphenation = (tokens: MappedChar[]): MappedChar[] => { + const text = tokens.map((token) => token.char).join(''); + const replaced: MappedChar[] = []; + let cursor = 0; + let match: RegExpExecArray | null; + + HYPHENATION_PATTERN.lastIndex = 0; + while ((match = HYPHENATION_PATTERN.exec(text)) !== null) { + const start = match.index; + const full = match[0]; + const first = match[1]; + const second = match[2]; + const secondOffset = full.lastIndexOf(second); + + replaced.push(...tokens.slice(cursor, start)); + replaced.push(...tokens.slice(start, start + first.length)); + replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length)); + cursor = start + full.length; + } + + replaced.push(...tokens.slice(cursor)); + return replaced; +}; + +/** + * Normalize a position-tagged character stream into the canonical TTS form, + * dropping/rewriting characters while preserving the source position of every + * surviving character. + */ +export const normalizeMappedChars = (tokens: MappedChar[]): MappedChar[] => { + const withoutLinks = replaceMappedUrls(tokens); + const withoutHyphenation = removeMappedHyphenation(withoutLinks); + const normalized: MappedChar[] = []; + let pendingWhitespace: MappedChar | null = null; + + const flushWhitespace = () => { + if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') { + pendingWhitespace = null; + return; + } + normalized.push(cloneMappedChar(' ', pendingWhitespace)); + pendingWhitespace = null; + }; + + for (const token of withoutHyphenation) { + if (token.char === '*') continue; + if (/\s/.test(token.char)) { + pendingWhitespace ??= token; + continue; + } + + flushWhitespace(); + normalized.push(token); + } + + if (normalized[normalized.length - 1]?.char === ' ') { + normalized.pop(); + } + + return normalized; +}; diff --git a/src/lib/client/html/highlight.ts b/src/lib/client/html/highlight.ts index a78a70a..f9a35f6 100644 --- a/src/lib/client/html/highlight.ts +++ b/src/lib/client/html/highlight.ts @@ -11,18 +11,19 @@ * sentence * - `WORD` — saturated background on the currently-spoken word * - * Word-to-DOM alignment uses the shared viewer token-range mapper so DOM token - * counts that diverge from the timed alignment still produce a smooth, - * monotonic highlight across languages. + * Word-to-DOM alignment is native: the Whisper alignment gives authoritative + * `charStart`/`charEnd` offsets into the sentence text, and we map those offsets + * directly onto a normalized char→DOM map of the located sentence wrap. No fuzzy + * token matching for words — that path used to jump the highlight to a later + * word in the sentence. */ import type { TTSSentenceAlignment } from '@/types/tts'; import { segmentWords } from '@/lib/shared/language'; import { - buildAlignmentTokenRanges, findBestHighlightTokenMatch, normalizeHighlightToken, - type HighlightTokenRange, } from '@/lib/client/highlight-token-alignment'; +import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map'; export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence'; export const HTML_WORD_CLASS = 'openreader-html-highlight-word'; @@ -34,23 +35,33 @@ interface DomToken { norm: string; } +interface CharPosition { + node: Text; + offset: number; +} + let sentenceWraps: HTMLSpanElement[] = []; let wordWraps: HTMLSpanElement[] = []; /** * Per-sentence state used by the word highlighter. Built once when the * sentence wrap is applied and then read by every word-advance event, so we - * don't re-walk the DOM or re-run the DP on every whisper tick. + * don't re-walk the DOM on every whisper tick. */ interface SentenceState { sentence: string; - // DOM tokens inside the wrapped sentence, captured AFTER the sentence wrap - // is in place. Stable across word wrap/unwrap cycles because clear() calls + // Normalized char→DOM map of the sentence wrap. `chars[i]` is the DOM position + // of the i-th character of `text`. Built AFTER the sentence wrap is in place; + // stable across word wrap/unwrap cycles because clear() calls // `parent.normalize()` which restores the original text-node structure. - wordTokens: DomToken[]; - // For an alignment we've already seen, the cached wordIndex → token range map. + chars: CharPosition[]; + // Normalized text of the wrap (chars joined), used to locate the sentence so + // alignment char offsets can be rebased onto the wrap. + text: string; + // For an alignment we've already seen: the offset of `sentence` inside `text` + // (the wrap window may be slightly wider than the sentence). null = not found. alignment: TTSSentenceAlignment | null; - wordToTokenRange: Array | null; + base: number | null; } let sentenceState: SentenceState | null = null; @@ -144,32 +155,31 @@ function collectDomTokens( } /** - * Walk only inside the current sentence wrap spans (used after the sentence - * wrap is applied; lets us index just the words *within* the highlighted - * sentence rather than the whole document). + * Walk inside the current sentence wrap spans and build a normalized char→DOM + * map. Every surviving character of the normalized text remembers the exact + * Text node + offset it came from, so alignment char offsets map straight to a + * DOM range. Normalization matches `preprocessSentenceForAudio` (the canonical + * space the alignment offsets live in). */ -function collectTokensInsideWraps(wraps: HTMLSpanElement[], language?: string): DomToken[] { - const tokens: DomToken[] = []; +function collectWrapCharMap(wraps: HTMLSpanElement[]): { chars: CharPosition[]; text: string } { + const raw: MappedChar[] = []; for (const wrap of wraps) { const walker = document.createTreeWalker(wrap, NodeFilter.SHOW_TEXT); let current: Node | null = walker.nextNode(); while (current) { const t = current as Text; - const text = t.nodeValue || ''; - for (const token of segmentWords(text, language)) { - const norm = normalizeWord(token.text); - if (!norm) continue; - tokens.push({ - textNode: t, - startOffset: token.start, - endOffset: token.end, - norm, - }); + const value = t.nodeValue || ''; + for (let offset = 0; offset < value.length; offset += 1) { + raw.push({ char: value[offset], pos: { node: t, offset } }); } current = walker.nextNode(); } } - return tokens; + const normalized = normalizeMappedChars(raw); + return { + chars: normalized.map((entry) => entry.pos), + text: normalized.map((entry) => entry.char).join(''), + }; } function findBestWindow(tokens: DomToken[], patternTokens: string[]): { start: number; end: number } | null { @@ -216,6 +226,50 @@ function wrapTokenRange(tokens: DomToken[], start: number, end: number, classNam return wraps; } +/** + * Wrap a half-open char range [start, end) of the sentence char map. Groups the + * covered characters by Text node (min/max offset per node) and splits/wraps + * each, mirroring `wrapTokenRange` but driven by per-character DOM positions. + */ +function wrapCharRange(chars: CharPosition[], start: number, end: number, className: string): HTMLSpanElement[] { + const perNode = new Map(); + for (let i = start; i < end; i += 1) { + const pos = chars[i]; + if (!pos) continue; + const existing = perNode.get(pos.node); + if (existing) { + existing.start = Math.min(existing.start, pos.offset); + existing.end = Math.max(existing.end, pos.offset + 1); + } else { + perNode.set(pos.node, { start: pos.offset, end: pos.offset + 1 }); + } + } + + const wraps: HTMLSpanElement[] = []; + for (const [textNode, { start: s, end: e }] of perNode) { + const parent = textNode.parentNode; + if (!parent) continue; + try { + let target: Text = textNode; + if (s > 0) { + target = target.splitText(s); + } + const innerLen = e - s; + if (innerLen < target.length) { + target.splitText(innerLen); + } + const span = document.createElement('span'); + span.className = className; + parent.insertBefore(span, target); + span.appendChild(target); + wraps.push(span); + } catch { + // skip any text node that can't be split (already wrapped, detached, etc.) + } + } + return wraps; +} + export function highlightHtmlSentence( container: HTMLElement | null | undefined, sentence: string | null | undefined, @@ -236,13 +290,15 @@ export function highlightHtmlSentence( sentenceWraps = wrapTokenRange(domTokens, win.start, win.end, HTML_SENTENCE_CLASS); if (!sentenceWraps.length) return false; - // Capture the per-token DOM map AFTER the sentence wrap is in place so we - // can look up individual word tokens without re-walking the doc. + // Capture the normalized char→DOM map AFTER the sentence wrap is in place so + // we can resolve individual word offsets without re-walking the doc. + const { chars, text } = collectWrapCharMap(sentenceWraps); sentenceState = { sentence, - wordTokens: collectTokensInsideWraps(sentenceWraps, language), + chars, + text, alignment: null, - wordToTokenRange: null, + base: null, }; return true; } @@ -254,32 +310,46 @@ export function highlightHtmlWord( ): boolean { // Always tear down the previous word wrap first. The `unwrap` call // normalizes the parent, restoring the post-sentence-wrap text-node - // structure that `sentenceState.wordTokens` points at, so the cached map + // structure that `sentenceState.chars` points at, so the cached map // stays valid across consecutive word advances. clearHtmlWordHighlight(); if (!container || !alignment) return false; if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return false; - if (!sentenceState || !sentenceState.wordTokens.length) return false; + if (!sentenceState || !sentenceState.chars.length) return false; if (!sentenceWraps.length) return false; const words = alignment.words || []; if (!words.length || wordIndex >= words.length) return false; - // (Re)build the alignment map when this is a new alignment object. - if (sentenceState.alignment !== alignment || !sentenceState.wordToTokenRange) { + // Locate the sentence inside the wrap's normalized text once per alignment. + // The sentence wrap is found by fuzzy token windowing so it may be slightly + // wider than the spoken sentence; rebasing keeps the alignment char offsets + // accurate regardless. + if (sentenceState.alignment !== alignment || sentenceState.base === null) { + const found = sentenceState.text.indexOf(alignment.sentence); + if (found < 0) { + // Sentence not located in the wrap — fail closed (leave only the sentence + // highlight) rather than snapping the word highlight to offset 0. Don't + // commit the alignment so the next tick retries the lookup. + sentenceState.base = null; + return false; + } sentenceState.alignment = alignment; - sentenceState.wordToTokenRange = buildAlignmentTokenRanges( - alignment.words, - sentenceState.wordTokens.map((token) => token.norm), - { fillGaps: true }, - ); + sentenceState.base = found; } - const tokenRange = sentenceState.wordToTokenRange[wordIndex]; - if (!tokenRange) return false; - if (tokenRange.start < 0 || tokenRange.end >= sentenceState.wordTokens.length) return false; + const base = sentenceState.base; + if (base === null) return false; - wordWraps = wrapTokenRange(sentenceState.wordTokens, tokenRange.start, tokenRange.end, HTML_WORD_CLASS); + const word = words[wordIndex]; + const { charStart, charEnd } = word; + if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charEnd <= charStart) return false; + + const start = Math.max(0, base + charStart); + const end = Math.min(sentenceState.chars.length, base + charEnd); + if (end <= start) return false; + + wordWraps = wrapCharRange(sentenceState.chars, start, end, HTML_WORD_CLASS); return wordWraps.length > 0; } diff --git a/src/lib/shared/nlp.ts b/src/lib/shared/nlp.ts index 6712116..3a8468d 100644 --- a/src/lib/shared/nlp.ts +++ b/src/lib/shared/nlp.ts @@ -129,8 +129,13 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => { }; /** - * Preprocesses text for audio generation by cleaning up various text artifacts - * + * Preprocesses text for audio generation by cleaning up various text artifacts. + * + * Source of truth for the canonical "audio" text form. Keep byte-for-byte in + * lock-step with the copy in `compute/core/src/whisper/alignment-map.ts` (which + * computes word char offsets) and the position-preserving variant in + * `src/lib/client/highlight-char-map.ts` (which maps those offsets to the DOM). + * * @param {string} text - The text to preprocess * @returns {string} The cleaned text */ diff --git a/tests/unit/epub-word-highlight.vitest.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts index 6d0e3cc..23f6d77 100644 --- a/tests/unit/epub-word-highlight.vitest.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -2,7 +2,6 @@ import { describe, expect, test } from 'vitest'; import { resolveAlignmentWordSourceRange, - tokenizeCanonicalSegment, } from '../../src/lib/client/epub/epub-word-highlight'; import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan'; import type { TTSSentenceAlignment } from '../../src/types/tts'; @@ -19,16 +18,6 @@ const segment = (text: string, offset = 0): CanonicalTtsSegment => ({ }); describe('EPUB word highlight mapping', () => { - test('tokenizes Japanese and Chinese using locale-aware word boundaries', () => { - const japanese = tokenizeCanonicalSegment(segment('これは日本語です。', 5), 'ja'); - expect(japanese.length).toBeGreaterThan(1); - expect(japanese.every((token) => token.norm.length > 0)).toBe(true); - - const chinese = tokenizeCanonicalSegment(segment('这是中文。', 10), 'zh'); - expect(chinese.length).toBeGreaterThan(1); - expect(chinese.map((token) => token.norm).join('')).toBe('这是中文'); - }); - test('resolves Japanese alignment chunks directly from character offsets', () => { const japanese = segment('これは日本語です。', 25); const word: TTSSentenceAlignment['words'][number] = { @@ -45,7 +34,7 @@ describe('EPUB word highlight mapping', () => { }); }); - test('rejects invalid alignment character offsets so token mapping can be used', () => { + test('rejects out-of-range alignment character offsets so the word is skipped', () => { const japanese = segment('これは日本語です。', 25); const word: TTSSentenceAlignment['words'][number] = { text: '範囲外', @@ -58,14 +47,4 @@ describe('EPUB word highlight mapping', () => { expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull(); }); - test('tokenizes canonical segment words with source offsets', () => { - const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); - - expect(tokens).toEqual([ - { norm: 'hello', sourceStart: 13, sourceEnd: 18 }, - { norm: 'she', sourceStart: 21, sourceEnd: 24 }, - { norm: 'said', sourceStart: 25, sourceEnd: 29 }, - ]); - }); - }); diff --git a/tests/unit/whisper-alignment-mapping.vitest.spec.ts b/tests/unit/whisper-alignment-mapping.vitest.spec.ts index b53a4f6..e5c9575 100644 --- a/tests/unit/whisper-alignment-mapping.vitest.spec.ts +++ b/tests/unit/whisper-alignment-mapping.vitest.spec.ts @@ -18,4 +18,18 @@ describe('whisper alignment mapping', () => { expect(aligned.words[2].charStart).toBeGreaterThan(aligned.words[1].charEnd); expect(aligned.words[2].charEnd).toBeLessThanOrEqual('Hello, world again.'.length); }); + + test('joins line-break hyphenation across unicode letters', () => { + // "Über-\n mensch" must normalize to "Übermensch" so the offsets match the + // client char map. This only works with the unicode-aware hyphen regex that + // is kept in lock-step across nlp.ts / alignment-map.ts / highlight-char-map.ts. + const aligned = mapWordsToSentenceOffsets('Über-\n mensch walks', [ + { word: 'Übermensch', start: 0, end: 0.5 }, + { word: 'walks', start: 0.5, end: 1.0 }, + ]); + + expect(aligned.words[0].charStart).toBe(0); + expect(aligned.words[0].charEnd).toBe('Übermensch'.length); + expect(aligned.words[1].charStart).toBeGreaterThanOrEqual(aligned.words[0].charEnd); + }); });