Merge pull request #110 from richardr1126/fix/epub-text-highlight
fix(epub/text): native offset word-by-word highlighting
This commit is contained in:
Richard R
GitHub
commit
6c12a47f74
11 changed files with 298 additions and 288 deletions
|
|
@ -1,9 +1,14 @@
|
|||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
||||
|
||||
// Keep byte-for-byte in lock-step with `preprocessSentenceForAudio` in
|
||||
// `src/lib/shared/nlp.ts` (and the position-preserving copy in
|
||||
// `src/lib/client/highlight-char-map.ts`). The word `charStart`/`charEnd`
|
||||
// offsets this module emits are consumed against text normalized by those
|
||||
// client functions, so any divergence shifts viewer highlights off-word.
|
||||
function preprocessSentenceForAudio(text: string): string {
|
||||
return text
|
||||
.replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -')
|
||||
.replace(/(\w+)-\s+(\w+)/g, '$1$2')
|
||||
.replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2')
|
||||
.replace(/\*/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ import {
|
|||
} from '@/lib/client/epub/epub-rendered-text-maps';
|
||||
import {
|
||||
useEPUBHighlighting,
|
||||
type EpubWordHighlightMapCache,
|
||||
} from '@/hooks/epub/useEPUBHighlighting';
|
||||
import { useEPUBLocationController } from '@/hooks/epub/useEPUBLocationController';
|
||||
import { useEPUBAudiobook } from '@/hooks/epub/useEPUBAudiobook';
|
||||
|
|
@ -148,7 +147,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
const currentHighlightCfi = useRef<string | null>(null);
|
||||
const currentWordHighlightCfi = useRef<string | null>(null);
|
||||
const renderedTextMapsRef = useRef<EpubRenderedTextMap[]>([]);
|
||||
const wordHighlightMapCacheRef = useRef<EpubWordHighlightMapCache | null>(null);
|
||||
const renderedLocationCloneManagerRef = useRef<EpubRenderedLocationCloneManager>(
|
||||
new EpubRenderedLocationCloneManager(),
|
||||
);
|
||||
|
|
@ -165,8 +163,6 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
currentHighlightCfiRef: currentHighlightCfi,
|
||||
currentWordHighlightCfiRef: currentWordHighlightCfi,
|
||||
renderedTextMapsRef,
|
||||
wordHighlightMapCacheRef,
|
||||
language: resolvedLanguage,
|
||||
});
|
||||
|
||||
useEffect(() => () => {
|
||||
|
|
|
|||
|
|
@ -2485,6 +2485,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const pos = activeHowl.seek() as number;
|
||||
if (typeof pos === 'number' && Number.isFinite(pos)) {
|
||||
const words = currentSentenceAlignment.words;
|
||||
// Find the word whose timing window contains `pos`. When `pos` falls
|
||||
// in a gap between words (or past the last word), snap to the last
|
||||
// word that has already started instead of freezing on a stale index
|
||||
// — monotonic, so it never jumps to a later word that hasn't started.
|
||||
let idx = -1;
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const w = words[i];
|
||||
|
|
@ -2492,6 +2496,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
idx = i;
|
||||
break;
|
||||
}
|
||||
if (pos >= w.startSec) {
|
||||
idx = i;
|
||||
}
|
||||
}
|
||||
if (idx !== -1) {
|
||||
setCurrentWordIndex((prev) => (prev === idx ? prev : idx));
|
||||
|
|
|
|||
|
|
@ -4,15 +4,8 @@ import { useCallback, useEffect, type MutableRefObject, type RefObject } from 'r
|
|||
import type { Rendition } from 'epubjs';
|
||||
|
||||
import {
|
||||
buildWordHighlightCacheKey,
|
||||
resolveAlignmentWordSourceRange,
|
||||
tokenizeCanonicalSegment,
|
||||
type EpubCanonicalWordToken,
|
||||
} from '@/lib/client/epub/epub-word-highlight';
|
||||
import {
|
||||
buildAlignmentTokenRanges,
|
||||
type HighlightTokenRange,
|
||||
} from '@/lib/client/highlight-token-alignment';
|
||||
import {
|
||||
createRangeFromMappedOffsets,
|
||||
resolveVisibleSegmentRange,
|
||||
|
|
@ -21,20 +14,12 @@ import {
|
|||
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
|
||||
import type { TTSSentenceAlignment } from '@/types/tts';
|
||||
|
||||
export type EpubWordHighlightMapCache = {
|
||||
key: string;
|
||||
wordToTokenRange: Array<HighlightTokenRange | null>;
|
||||
tokens: EpubCanonicalWordToken[];
|
||||
};
|
||||
|
||||
type UseEpubHighlightingParams = {
|
||||
renditionRef: RefObject<Rendition | undefined>;
|
||||
epubHighlightEnabled: boolean;
|
||||
currentHighlightCfiRef: MutableRefObject<string | null>;
|
||||
currentWordHighlightCfiRef: MutableRefObject<string | null>;
|
||||
renderedTextMapsRef: MutableRefObject<EpubRenderedTextMap[]>;
|
||||
wordHighlightMapCacheRef: MutableRefObject<EpubWordHighlightMapCache | null>;
|
||||
language?: string;
|
||||
};
|
||||
|
||||
type UseEpubHighlightingResult = {
|
||||
|
|
@ -56,8 +41,6 @@ export function useEPUBHighlighting({
|
|||
currentHighlightCfiRef,
|
||||
currentWordHighlightCfiRef,
|
||||
renderedTextMapsRef,
|
||||
wordHighlightMapCacheRef,
|
||||
language,
|
||||
}: UseEpubHighlightingParams): UseEpubHighlightingResult {
|
||||
const clearWordHighlights = useCallback(() => {
|
||||
if (!renditionRef.current) return;
|
||||
|
|
@ -122,68 +105,19 @@ export function useEPUBHighlighting({
|
|||
const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment);
|
||||
if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return;
|
||||
|
||||
// Native path only: the alignment's char offsets are authoritative for the
|
||||
// spoken word (they live in the same canonical space as the segment text and
|
||||
// the rendered char map). Clamp to the portion of the segment that is
|
||||
// actually visible in this map so a word straddling a page/spread boundary
|
||||
// still highlights its visible part instead of dropping the word entirely.
|
||||
const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]);
|
||||
if (
|
||||
alignmentRange
|
||||
&& alignmentRange.sourceStart >= resolved.startOffset
|
||||
&& alignmentRange.sourceEnd <= resolved.endOffset
|
||||
) {
|
||||
const wordRange = createRangeFromMappedOffsets(
|
||||
resolved.map,
|
||||
alignmentRange.sourceStart,
|
||||
alignmentRange.sourceEnd,
|
||||
);
|
||||
if (wordRange) {
|
||||
try {
|
||||
const wordCfi = resolved.map.content.cfiFromRange(wordRange);
|
||||
currentWordHighlightCfiRef.current = wordCfi;
|
||||
renditionRef.current.annotations.add(
|
||||
'highlight',
|
||||
wordCfi,
|
||||
{},
|
||||
() => { },
|
||||
'',
|
||||
{
|
||||
fill: 'var(--accent)',
|
||||
'fill-opacity': '0.4',
|
||||
'mix-blend-mode': 'multiply',
|
||||
}
|
||||
);
|
||||
return;
|
||||
} catch (error) {
|
||||
console.error('Error highlighting EPUB word from alignment offsets:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!alignmentRange) return;
|
||||
|
||||
const cacheKey = buildWordHighlightCacheKey(segment, alignment, language);
|
||||
if (wordHighlightMapCacheRef.current?.key !== cacheKey) {
|
||||
const tokens = tokenizeCanonicalSegment(segment, language);
|
||||
wordHighlightMapCacheRef.current = {
|
||||
key: cacheKey,
|
||||
tokens,
|
||||
wordToTokenRange: buildAlignmentTokenRanges(
|
||||
words,
|
||||
tokens.map((token) => token.norm),
|
||||
{ minimumSimilarity: 0.8 },
|
||||
),
|
||||
};
|
||||
}
|
||||
const clampedStart = Math.max(alignmentRange.sourceStart, resolved.startOffset);
|
||||
const clampedEnd = Math.min(alignmentRange.sourceEnd, resolved.endOffset);
|
||||
if (clampedEnd <= clampedStart) return;
|
||||
|
||||
const cached = wordHighlightMapCacheRef.current;
|
||||
const tokenRange = cached.wordToTokenRange[wordIndex];
|
||||
if (!tokenRange) return;
|
||||
|
||||
const firstToken = cached.tokens[tokenRange.start];
|
||||
const lastToken = cached.tokens[tokenRange.end];
|
||||
if (!firstToken || !lastToken) return;
|
||||
if (firstToken.sourceStart < resolved.startOffset || lastToken.sourceEnd > resolved.endOffset) return;
|
||||
|
||||
const wordRange = createRangeFromMappedOffsets(
|
||||
resolved.map,
|
||||
firstToken.sourceStart,
|
||||
lastToken.sourceEnd,
|
||||
);
|
||||
const wordRange = createRangeFromMappedOffsets(resolved.map, clampedStart, clampedEnd);
|
||||
if (!wordRange) return;
|
||||
|
||||
try {
|
||||
|
|
@ -202,7 +136,7 @@ export function useEPUBHighlighting({
|
|||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Error highlighting EPUB word:', error);
|
||||
console.error('Error highlighting EPUB word from alignment offsets:', error);
|
||||
}
|
||||
}, [
|
||||
clearWordHighlights,
|
||||
|
|
@ -210,20 +144,16 @@ export function useEPUBHighlighting({
|
|||
epubHighlightEnabled,
|
||||
renderedTextMapsRef,
|
||||
renditionRef,
|
||||
wordHighlightMapCacheRef,
|
||||
language,
|
||||
]);
|
||||
|
||||
const setRenderedTextMaps = useCallback((maps: EpubRenderedTextMap[]) => {
|
||||
renderedTextMapsRef.current = maps;
|
||||
wordHighlightMapCacheRef.current = null;
|
||||
}, [renderedTextMapsRef, wordHighlightMapCacheRef]);
|
||||
}, [renderedTextMapsRef]);
|
||||
|
||||
const resetHighlightState = useCallback(() => {
|
||||
renderedTextMapsRef.current = [];
|
||||
wordHighlightMapCacheRef.current = null;
|
||||
clearHighlights();
|
||||
}, [clearHighlights, renderedTextMapsRef, wordHighlightMapCacheRef]);
|
||||
}, [clearHighlights, renderedTextMapsRef]);
|
||||
|
||||
// Clear any highlight annotations when feature is disabled.
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -2,16 +2,14 @@
|
|||
|
||||
import type { Rendition } from 'epubjs';
|
||||
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
|
||||
import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map';
|
||||
|
||||
type EpubMappedPosition = {
|
||||
node: Text;
|
||||
offset: number;
|
||||
};
|
||||
|
||||
type EpubMappedChar = {
|
||||
char: string;
|
||||
position: EpubMappedPosition;
|
||||
};
|
||||
type EpubMappedChar = MappedChar<EpubMappedPosition>;
|
||||
|
||||
export type EpubRenderedTextMap = {
|
||||
sourceKey: string;
|
||||
|
|
@ -21,94 +19,6 @@ export type EpubRenderedTextMap = {
|
|||
};
|
||||
};
|
||||
|
||||
const cloneMappedChar = (char: string, source: EpubMappedChar): EpubMappedChar => ({
|
||||
char,
|
||||
position: source.position,
|
||||
});
|
||||
|
||||
const replaceMappedUrls = (tokens: EpubMappedChar[]): EpubMappedChar[] => {
|
||||
const text = tokens.map((token) => token.char).join('');
|
||||
const urlPattern = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi;
|
||||
const replaced: EpubMappedChar[] = [];
|
||||
let cursor = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = urlPattern.exec(text)) !== null) {
|
||||
const start = match.index;
|
||||
const end = start + match[0].length;
|
||||
replaced.push(...tokens.slice(cursor, start));
|
||||
|
||||
const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)];
|
||||
if (anchor) {
|
||||
const replacement = `- (link to ${match[1]}) -`;
|
||||
for (const char of replacement) {
|
||||
replaced.push(cloneMappedChar(char, anchor));
|
||||
}
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
|
||||
replaced.push(...tokens.slice(cursor));
|
||||
return replaced;
|
||||
};
|
||||
|
||||
const removeMappedHyphenation = (tokens: EpubMappedChar[]): EpubMappedChar[] => {
|
||||
const text = tokens.map((token) => token.char).join('');
|
||||
const hyphenPattern = /(\w+)-\s+(\w+)/g;
|
||||
const replaced: EpubMappedChar[] = [];
|
||||
let cursor = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = hyphenPattern.exec(text)) !== null) {
|
||||
const start = match.index;
|
||||
const full = match[0];
|
||||
const first = match[1];
|
||||
const second = match[2];
|
||||
const secondOffset = full.lastIndexOf(second);
|
||||
|
||||
replaced.push(...tokens.slice(cursor, start));
|
||||
replaced.push(...tokens.slice(start, start + first.length));
|
||||
replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length));
|
||||
cursor = start + full.length;
|
||||
}
|
||||
|
||||
replaced.push(...tokens.slice(cursor));
|
||||
return replaced;
|
||||
};
|
||||
|
||||
const normalizeMappedTokensForTts = (tokens: EpubMappedChar[]): EpubMappedChar[] => {
|
||||
const withoutLinks = replaceMappedUrls(tokens);
|
||||
const withoutHyphenation = removeMappedHyphenation(withoutLinks);
|
||||
const normalized: EpubMappedChar[] = [];
|
||||
let pendingWhitespace: EpubMappedChar | null = null;
|
||||
|
||||
const flushWhitespace = () => {
|
||||
if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') {
|
||||
pendingWhitespace = null;
|
||||
return;
|
||||
}
|
||||
normalized.push(cloneMappedChar(' ', pendingWhitespace));
|
||||
pendingWhitespace = null;
|
||||
};
|
||||
|
||||
for (const token of withoutHyphenation) {
|
||||
if (token.char === '*') continue;
|
||||
if (/\s/.test(token.char)) {
|
||||
pendingWhitespace ??= token;
|
||||
continue;
|
||||
}
|
||||
|
||||
flushWhitespace();
|
||||
normalized.push(token);
|
||||
}
|
||||
|
||||
if (normalized[normalized.length - 1]?.char === ' ') {
|
||||
normalized.pop();
|
||||
}
|
||||
|
||||
return normalized;
|
||||
};
|
||||
|
||||
const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => {
|
||||
const root = range.commonAncestorContainer;
|
||||
const doc = range.startContainer.ownerDocument ?? (range.startContainer as Document);
|
||||
|
|
@ -121,7 +31,7 @@ const collectMappedTextFromRange = (range: Range): EpubMappedChar[] => {
|
|||
for (let offset = safeStart; offset < safeEnd; offset += 1) {
|
||||
mapped.push({
|
||||
char: text[offset],
|
||||
position: { node: textNode, offset },
|
||||
pos: { node: textNode, offset },
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -178,12 +88,12 @@ export const buildRenderedTextMaps = (
|
|||
const range = content.range(rangeCfi);
|
||||
if (!range) continue;
|
||||
|
||||
const normalized = normalizeMappedTokensForTts(collectMappedTextFromRange(range));
|
||||
const normalized = normalizeMappedChars(collectMappedTextFromRange(range));
|
||||
if (!normalized.length) continue;
|
||||
|
||||
maps.push({
|
||||
sourceKey,
|
||||
chars: normalized.map((token) => token.position),
|
||||
chars: normalized.map((token) => token.pos),
|
||||
content,
|
||||
});
|
||||
} catch {
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
|
||||
import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts';
|
||||
import { segmentWords } from '@/lib/shared/language';
|
||||
import { normalizeHighlightToken } from '@/lib/client/highlight-token-alignment';
|
||||
|
||||
export type EpubCanonicalWordToken = {
|
||||
norm: string;
|
||||
sourceStart: number;
|
||||
sourceEnd: number;
|
||||
};
|
||||
|
||||
export const normalizeWordForHighlight = (text: string): string =>
|
||||
normalizeHighlightToken(text);
|
||||
import type { TTSSentenceWord } from '@/types/tts';
|
||||
|
||||
/**
|
||||
* Resolve a spoken word's char offsets (from the Whisper alignment) into the
|
||||
* canonical source-offset space the rendered text map is keyed by. The
|
||||
* alignment's `charStart`/`charEnd` are offsets into the segment text, which is
|
||||
* in the same normalized space as `segment.startAnchor.offset`.
|
||||
*/
|
||||
export const resolveAlignmentWordSourceRange = (
|
||||
segment: CanonicalTtsSegment,
|
||||
word: TTSSentenceWord,
|
||||
|
|
@ -25,28 +20,3 @@ export const resolveAlignmentWordSourceRange = (
|
|||
sourceEnd: segment.startAnchor.offset + charEnd,
|
||||
};
|
||||
};
|
||||
|
||||
export const tokenizeCanonicalSegment = (
|
||||
segment: CanonicalTtsSegment,
|
||||
language?: string,
|
||||
): EpubCanonicalWordToken[] =>
|
||||
segmentWords(segment.text, language)
|
||||
.map((token) => ({
|
||||
norm: normalizeWordForHighlight(token.text),
|
||||
sourceStart: segment.startAnchor.offset + token.start,
|
||||
sourceEnd: segment.startAnchor.offset + token.end,
|
||||
}))
|
||||
.filter((token) => Boolean(token.norm));
|
||||
|
||||
export const buildWordHighlightCacheKey = (
|
||||
segment: CanonicalTtsSegment,
|
||||
alignment: TTSSentenceAlignment,
|
||||
language?: string,
|
||||
): string =>
|
||||
[
|
||||
segment.key,
|
||||
segment.text.length,
|
||||
language || '',
|
||||
alignment.words.length,
|
||||
alignment.words.map((word) => normalizeWordForHighlight(word.text)).join('|'),
|
||||
].join('::');
|
||||
|
|
|
|||
124
src/lib/client/highlight-char-map.ts
Normal file
124
src/lib/client/highlight-char-map.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
/**
|
||||
* Shared, position-preserving text normalizer for viewer highlighting.
|
||||
*
|
||||
* TTS word offsets (`charStart`/`charEnd`) are computed against the canonical
|
||||
* "audio" form of a sentence — URLs rewritten, line-break hyphenation joined,
|
||||
* `*` stripped, whitespace collapsed. To map those offsets back onto the
|
||||
* rendered DOM we have to normalize the DOM text the SAME way while remembering
|
||||
* which DOM position each surviving character came from.
|
||||
*
|
||||
* This module operates on an opaque position type so both the EPUB renderer
|
||||
* (position = `{ node, offset }` in an iframe) and the HTML/TXT renderer
|
||||
* (position = a `Text` node + offset in the main document) share one identical
|
||||
* normalization. Keep the transforms here in lock-step with
|
||||
* `preprocessSentenceForAudio` in `src/lib/shared/nlp.ts` and the copy in
|
||||
* `compute/core/src/whisper/alignment-map.ts`.
|
||||
*/
|
||||
|
||||
export interface MappedChar<TPos> {
|
||||
char: string;
|
||||
pos: TPos;
|
||||
}
|
||||
|
||||
const URL_PATTERN = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi;
|
||||
// Unicode-aware to match preprocessSentenceForAudio (nlp.ts / alignment-map.ts).
|
||||
const HYPHENATION_PATTERN = /([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu;
|
||||
|
||||
const cloneMappedChar = <TPos>(char: string, source: MappedChar<TPos>): MappedChar<TPos> => ({
|
||||
char,
|
||||
pos: source.pos,
|
||||
});
|
||||
|
||||
const replaceMappedUrls = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[] => {
|
||||
const text = tokens.map((token) => token.char).join('');
|
||||
const replaced: MappedChar<TPos>[] = [];
|
||||
let cursor = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
URL_PATTERN.lastIndex = 0;
|
||||
while ((match = URL_PATTERN.exec(text)) !== null) {
|
||||
const start = match.index;
|
||||
const end = start + match[0].length;
|
||||
replaced.push(...tokens.slice(cursor, start));
|
||||
|
||||
const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)];
|
||||
if (anchor) {
|
||||
const replacement = `- (link to ${match[1]}) -`;
|
||||
// Spread the replacement characters across the original URL span so the
|
||||
// mapped positions keep their positional spread instead of collapsing the
|
||||
// whole URL onto a single DOM anchor.
|
||||
const originalLength = Math.max(1, end - start);
|
||||
for (let i = 0; i < replacement.length; i += 1) {
|
||||
const sourceIndex = Math.min(end - 1, start + Math.floor((i * originalLength) / replacement.length));
|
||||
const source = tokens[sourceIndex] ?? anchor;
|
||||
replaced.push(cloneMappedChar(replacement[i], source));
|
||||
}
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
|
||||
replaced.push(...tokens.slice(cursor));
|
||||
return replaced;
|
||||
};
|
||||
|
||||
const removeMappedHyphenation = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[] => {
|
||||
const text = tokens.map((token) => token.char).join('');
|
||||
const replaced: MappedChar<TPos>[] = [];
|
||||
let cursor = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
HYPHENATION_PATTERN.lastIndex = 0;
|
||||
while ((match = HYPHENATION_PATTERN.exec(text)) !== null) {
|
||||
const start = match.index;
|
||||
const full = match[0];
|
||||
const first = match[1];
|
||||
const second = match[2];
|
||||
const secondOffset = full.lastIndexOf(second);
|
||||
|
||||
replaced.push(...tokens.slice(cursor, start));
|
||||
replaced.push(...tokens.slice(start, start + first.length));
|
||||
replaced.push(...tokens.slice(start + secondOffset, start + secondOffset + second.length));
|
||||
cursor = start + full.length;
|
||||
}
|
||||
|
||||
replaced.push(...tokens.slice(cursor));
|
||||
return replaced;
|
||||
};
|
||||
|
||||
/**
|
||||
* Normalize a position-tagged character stream into the canonical TTS form,
|
||||
* dropping/rewriting characters while preserving the source position of every
|
||||
* surviving character.
|
||||
*/
|
||||
export const normalizeMappedChars = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[] => {
|
||||
const withoutLinks = replaceMappedUrls(tokens);
|
||||
const withoutHyphenation = removeMappedHyphenation(withoutLinks);
|
||||
const normalized: MappedChar<TPos>[] = [];
|
||||
let pendingWhitespace: MappedChar<TPos> | null = null;
|
||||
|
||||
const flushWhitespace = () => {
|
||||
if (!pendingWhitespace || normalized.length === 0 || normalized[normalized.length - 1].char === ' ') {
|
||||
pendingWhitespace = null;
|
||||
return;
|
||||
}
|
||||
normalized.push(cloneMappedChar(' ', pendingWhitespace));
|
||||
pendingWhitespace = null;
|
||||
};
|
||||
|
||||
for (const token of withoutHyphenation) {
|
||||
if (token.char === '*') continue;
|
||||
if (/\s/.test(token.char)) {
|
||||
pendingWhitespace ??= token;
|
||||
continue;
|
||||
}
|
||||
|
||||
flushWhitespace();
|
||||
normalized.push(token);
|
||||
}
|
||||
|
||||
if (normalized[normalized.length - 1]?.char === ' ') {
|
||||
normalized.pop();
|
||||
}
|
||||
|
||||
return normalized;
|
||||
};
|
||||
|
|
@ -11,18 +11,19 @@
|
|||
* sentence
|
||||
* - `WORD` — saturated background on the currently-spoken word
|
||||
*
|
||||
* Word-to-DOM alignment uses the shared viewer token-range mapper so DOM token
|
||||
* counts that diverge from the timed alignment still produce a smooth,
|
||||
* monotonic highlight across languages.
|
||||
* Word-to-DOM alignment is native: the Whisper alignment gives authoritative
|
||||
* `charStart`/`charEnd` offsets into the sentence text, and we map those offsets
|
||||
* directly onto a normalized char→DOM map of the located sentence wrap. No fuzzy
|
||||
* token matching for words — that path used to jump the highlight to a later
|
||||
* word in the sentence.
|
||||
*/
|
||||
import type { TTSSentenceAlignment } from '@/types/tts';
|
||||
import { segmentWords } from '@/lib/shared/language';
|
||||
import {
|
||||
buildAlignmentTokenRanges,
|
||||
findBestHighlightTokenMatch,
|
||||
normalizeHighlightToken,
|
||||
type HighlightTokenRange,
|
||||
} from '@/lib/client/highlight-token-alignment';
|
||||
import { normalizeMappedChars, type MappedChar } from '@/lib/client/highlight-char-map';
|
||||
|
||||
export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence';
|
||||
export const HTML_WORD_CLASS = 'openreader-html-highlight-word';
|
||||
|
|
@ -34,23 +35,33 @@ interface DomToken {
|
|||
norm: string;
|
||||
}
|
||||
|
||||
interface CharPosition {
|
||||
node: Text;
|
||||
offset: number;
|
||||
}
|
||||
|
||||
let sentenceWraps: HTMLSpanElement[] = [];
|
||||
let wordWraps: HTMLSpanElement[] = [];
|
||||
|
||||
/**
|
||||
* Per-sentence state used by the word highlighter. Built once when the
|
||||
* sentence wrap is applied and then read by every word-advance event, so we
|
||||
* don't re-walk the DOM or re-run the DP on every whisper tick.
|
||||
* don't re-walk the DOM on every whisper tick.
|
||||
*/
|
||||
interface SentenceState {
|
||||
sentence: string;
|
||||
// DOM tokens inside the wrapped sentence, captured AFTER the sentence wrap
|
||||
// is in place. Stable across word wrap/unwrap cycles because clear() calls
|
||||
// Normalized char→DOM map of the sentence wrap. `chars[i]` is the DOM position
|
||||
// of the i-th character of `text`. Built AFTER the sentence wrap is in place;
|
||||
// stable across word wrap/unwrap cycles because clear() calls
|
||||
// `parent.normalize()` which restores the original text-node structure.
|
||||
wordTokens: DomToken[];
|
||||
// For an alignment we've already seen, the cached wordIndex → token range map.
|
||||
chars: CharPosition[];
|
||||
// Normalized text of the wrap (chars joined), used to locate the sentence so
|
||||
// alignment char offsets can be rebased onto the wrap.
|
||||
text: string;
|
||||
// For an alignment we've already seen: the offset of `sentence` inside `text`
|
||||
// (the wrap window may be slightly wider than the sentence). null = not found.
|
||||
alignment: TTSSentenceAlignment | null;
|
||||
wordToTokenRange: Array<HighlightTokenRange | null> | null;
|
||||
base: number | null;
|
||||
}
|
||||
|
||||
let sentenceState: SentenceState | null = null;
|
||||
|
|
@ -144,32 +155,31 @@ function collectDomTokens(
|
|||
}
|
||||
|
||||
/**
|
||||
* Walk only inside the current sentence wrap spans (used after the sentence
|
||||
* wrap is applied; lets us index just the words *within* the highlighted
|
||||
* sentence rather than the whole document).
|
||||
* Walk inside the current sentence wrap spans and build a normalized char→DOM
|
||||
* map. Every surviving character of the normalized text remembers the exact
|
||||
* Text node + offset it came from, so alignment char offsets map straight to a
|
||||
* DOM range. Normalization matches `preprocessSentenceForAudio` (the canonical
|
||||
* space the alignment offsets live in).
|
||||
*/
|
||||
function collectTokensInsideWraps(wraps: HTMLSpanElement[], language?: string): DomToken[] {
|
||||
const tokens: DomToken[] = [];
|
||||
function collectWrapCharMap(wraps: HTMLSpanElement[]): { chars: CharPosition[]; text: string } {
|
||||
const raw: MappedChar<CharPosition>[] = [];
|
||||
for (const wrap of wraps) {
|
||||
const walker = document.createTreeWalker(wrap, NodeFilter.SHOW_TEXT);
|
||||
let current: Node | null = walker.nextNode();
|
||||
while (current) {
|
||||
const t = current as Text;
|
||||
const text = t.nodeValue || '';
|
||||
for (const token of segmentWords(text, language)) {
|
||||
const norm = normalizeWord(token.text);
|
||||
if (!norm) continue;
|
||||
tokens.push({
|
||||
textNode: t,
|
||||
startOffset: token.start,
|
||||
endOffset: token.end,
|
||||
norm,
|
||||
});
|
||||
const value = t.nodeValue || '';
|
||||
for (let offset = 0; offset < value.length; offset += 1) {
|
||||
raw.push({ char: value[offset], pos: { node: t, offset } });
|
||||
}
|
||||
current = walker.nextNode();
|
||||
}
|
||||
}
|
||||
return tokens;
|
||||
const normalized = normalizeMappedChars(raw);
|
||||
return {
|
||||
chars: normalized.map((entry) => entry.pos),
|
||||
text: normalized.map((entry) => entry.char).join(''),
|
||||
};
|
||||
}
|
||||
|
||||
function findBestWindow(tokens: DomToken[], patternTokens: string[]): { start: number; end: number } | null {
|
||||
|
|
@ -216,6 +226,50 @@ function wrapTokenRange(tokens: DomToken[], start: number, end: number, classNam
|
|||
return wraps;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a half-open char range [start, end) of the sentence char map. Groups the
|
||||
* covered characters by Text node (min/max offset per node) and splits/wraps
|
||||
* each, mirroring `wrapTokenRange` but driven by per-character DOM positions.
|
||||
*/
|
||||
function wrapCharRange(chars: CharPosition[], start: number, end: number, className: string): HTMLSpanElement[] {
|
||||
const perNode = new Map<Text, { start: number; end: number }>();
|
||||
for (let i = start; i < end; i += 1) {
|
||||
const pos = chars[i];
|
||||
if (!pos) continue;
|
||||
const existing = perNode.get(pos.node);
|
||||
if (existing) {
|
||||
existing.start = Math.min(existing.start, pos.offset);
|
||||
existing.end = Math.max(existing.end, pos.offset + 1);
|
||||
} else {
|
||||
perNode.set(pos.node, { start: pos.offset, end: pos.offset + 1 });
|
||||
}
|
||||
}
|
||||
|
||||
const wraps: HTMLSpanElement[] = [];
|
||||
for (const [textNode, { start: s, end: e }] of perNode) {
|
||||
const parent = textNode.parentNode;
|
||||
if (!parent) continue;
|
||||
try {
|
||||
let target: Text = textNode;
|
||||
if (s > 0) {
|
||||
target = target.splitText(s);
|
||||
}
|
||||
const innerLen = e - s;
|
||||
if (innerLen < target.length) {
|
||||
target.splitText(innerLen);
|
||||
}
|
||||
const span = document.createElement('span');
|
||||
span.className = className;
|
||||
parent.insertBefore(span, target);
|
||||
span.appendChild(target);
|
||||
wraps.push(span);
|
||||
} catch {
|
||||
// skip any text node that can't be split (already wrapped, detached, etc.)
|
||||
}
|
||||
}
|
||||
return wraps;
|
||||
}
|
||||
|
||||
export function highlightHtmlSentence(
|
||||
container: HTMLElement | null | undefined,
|
||||
sentence: string | null | undefined,
|
||||
|
|
@ -236,13 +290,15 @@ export function highlightHtmlSentence(
|
|||
sentenceWraps = wrapTokenRange(domTokens, win.start, win.end, HTML_SENTENCE_CLASS);
|
||||
if (!sentenceWraps.length) return false;
|
||||
|
||||
// Capture the per-token DOM map AFTER the sentence wrap is in place so we
|
||||
// can look up individual word tokens without re-walking the doc.
|
||||
// Capture the normalized char→DOM map AFTER the sentence wrap is in place so
|
||||
// we can resolve individual word offsets without re-walking the doc.
|
||||
const { chars, text } = collectWrapCharMap(sentenceWraps);
|
||||
sentenceState = {
|
||||
sentence,
|
||||
wordTokens: collectTokensInsideWraps(sentenceWraps, language),
|
||||
chars,
|
||||
text,
|
||||
alignment: null,
|
||||
wordToTokenRange: null,
|
||||
base: null,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
|
@ -254,32 +310,46 @@ export function highlightHtmlWord(
|
|||
): boolean {
|
||||
// Always tear down the previous word wrap first. The `unwrap` call
|
||||
// normalizes the parent, restoring the post-sentence-wrap text-node
|
||||
// structure that `sentenceState.wordTokens` points at, so the cached map
|
||||
// structure that `sentenceState.chars` points at, so the cached map
|
||||
// stays valid across consecutive word advances.
|
||||
clearHtmlWordHighlight();
|
||||
if (!container || !alignment) return false;
|
||||
if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return false;
|
||||
if (!sentenceState || !sentenceState.wordTokens.length) return false;
|
||||
if (!sentenceState || !sentenceState.chars.length) return false;
|
||||
if (!sentenceWraps.length) return false;
|
||||
|
||||
const words = alignment.words || [];
|
||||
if (!words.length || wordIndex >= words.length) return false;
|
||||
|
||||
// (Re)build the alignment map when this is a new alignment object.
|
||||
if (sentenceState.alignment !== alignment || !sentenceState.wordToTokenRange) {
|
||||
// Locate the sentence inside the wrap's normalized text once per alignment.
|
||||
// The sentence wrap is found by fuzzy token windowing so it may be slightly
|
||||
// wider than the spoken sentence; rebasing keeps the alignment char offsets
|
||||
// accurate regardless.
|
||||
if (sentenceState.alignment !== alignment || sentenceState.base === null) {
|
||||
const found = sentenceState.text.indexOf(alignment.sentence);
|
||||
if (found < 0) {
|
||||
// Sentence not located in the wrap — fail closed (leave only the sentence
|
||||
// highlight) rather than snapping the word highlight to offset 0. Don't
|
||||
// commit the alignment so the next tick retries the lookup.
|
||||
sentenceState.base = null;
|
||||
return false;
|
||||
}
|
||||
sentenceState.alignment = alignment;
|
||||
sentenceState.wordToTokenRange = buildAlignmentTokenRanges(
|
||||
alignment.words,
|
||||
sentenceState.wordTokens.map((token) => token.norm),
|
||||
{ fillGaps: true },
|
||||
);
|
||||
sentenceState.base = found;
|
||||
}
|
||||
|
||||
const tokenRange = sentenceState.wordToTokenRange[wordIndex];
|
||||
if (!tokenRange) return false;
|
||||
if (tokenRange.start < 0 || tokenRange.end >= sentenceState.wordTokens.length) return false;
|
||||
const base = sentenceState.base;
|
||||
if (base === null) return false;
|
||||
|
||||
wordWraps = wrapTokenRange(sentenceState.wordTokens, tokenRange.start, tokenRange.end, HTML_WORD_CLASS);
|
||||
const word = words[wordIndex];
|
||||
const { charStart, charEnd } = word;
|
||||
if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charEnd <= charStart) return false;
|
||||
|
||||
const start = Math.max(0, base + charStart);
|
||||
const end = Math.min(sentenceState.chars.length, base + charEnd);
|
||||
if (end <= start) return false;
|
||||
|
||||
wordWraps = wrapCharRange(sentenceState.chars, start, end, HTML_WORD_CLASS);
|
||||
return wordWraps.length > 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -129,8 +129,13 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => {
|
|||
};
|
||||
|
||||
/**
|
||||
* Preprocesses text for audio generation by cleaning up various text artifacts
|
||||
*
|
||||
* Preprocesses text for audio generation by cleaning up various text artifacts.
|
||||
*
|
||||
* Source of truth for the canonical "audio" text form. Keep byte-for-byte in
|
||||
* lock-step with the copy in `compute/core/src/whisper/alignment-map.ts` (which
|
||||
* computes word char offsets) and the position-preserving variant in
|
||||
* `src/lib/client/highlight-char-map.ts` (which maps those offsets to the DOM).
|
||||
*
|
||||
* @param {string} text - The text to preprocess
|
||||
* @returns {string} The cleaned text
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ import { describe, expect, test } from 'vitest';
|
|||
|
||||
import {
|
||||
resolveAlignmentWordSourceRange,
|
||||
tokenizeCanonicalSegment,
|
||||
} from '../../src/lib/client/epub/epub-word-highlight';
|
||||
import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan';
|
||||
import type { TTSSentenceAlignment } from '../../src/types/tts';
|
||||
|
|
@ -19,16 +18,6 @@ const segment = (text: string, offset = 0): CanonicalTtsSegment => ({
|
|||
});
|
||||
|
||||
describe('EPUB word highlight mapping', () => {
|
||||
test('tokenizes Japanese and Chinese using locale-aware word boundaries', () => {
|
||||
const japanese = tokenizeCanonicalSegment(segment('これは日本語です。', 5), 'ja');
|
||||
expect(japanese.length).toBeGreaterThan(1);
|
||||
expect(japanese.every((token) => token.norm.length > 0)).toBe(true);
|
||||
|
||||
const chinese = tokenizeCanonicalSegment(segment('这是中文。', 10), 'zh');
|
||||
expect(chinese.length).toBeGreaterThan(1);
|
||||
expect(chinese.map((token) => token.norm).join('')).toBe('这是中文');
|
||||
});
|
||||
|
||||
test('resolves Japanese alignment chunks directly from character offsets', () => {
|
||||
const japanese = segment('これは日本語です。', 25);
|
||||
const word: TTSSentenceAlignment['words'][number] = {
|
||||
|
|
@ -45,7 +34,7 @@ describe('EPUB word highlight mapping', () => {
|
|||
});
|
||||
});
|
||||
|
||||
test('rejects invalid alignment character offsets so token mapping can be used', () => {
|
||||
test('rejects out-of-range alignment character offsets so the word is skipped', () => {
|
||||
const japanese = segment('これは日本語です。', 25);
|
||||
const word: TTSSentenceAlignment['words'][number] = {
|
||||
text: '範囲外',
|
||||
|
|
@ -58,14 +47,4 @@ describe('EPUB word highlight mapping', () => {
|
|||
expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull();
|
||||
});
|
||||
|
||||
test('tokenizes canonical segment words with source offsets', () => {
|
||||
const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12));
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{ norm: 'hello', sourceStart: 13, sourceEnd: 18 },
|
||||
{ norm: 'she', sourceStart: 21, sourceEnd: 24 },
|
||||
{ norm: 'said', sourceStart: 25, sourceEnd: 29 },
|
||||
]);
|
||||
});
|
||||
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,4 +18,18 @@ describe('whisper alignment mapping', () => {
|
|||
expect(aligned.words[2].charStart).toBeGreaterThan(aligned.words[1].charEnd);
|
||||
expect(aligned.words[2].charEnd).toBeLessThanOrEqual('Hello, world again.'.length);
|
||||
});
|
||||
|
||||
test('joins line-break hyphenation across unicode letters', () => {
|
||||
// "Über-\n mensch" must normalize to "Übermensch" so the offsets match the
|
||||
// client char map. This only works with the unicode-aware hyphen regex that
|
||||
// is kept in lock-step across nlp.ts / alignment-map.ts / highlight-char-map.ts.
|
||||
const aligned = mapWordsToSentenceOffsets('Über-\n mensch walks', [
|
||||
{ word: 'Übermensch', start: 0, end: 0.5 },
|
||||
{ word: 'walks', start: 0.5, end: 1.0 },
|
||||
]);
|
||||
|
||||
expect(aligned.words[0].charStart).toBe(0);
|
||||
expect(aligned.words[0].charEnd).toBe('Übermensch'.length);
|
||||
expect(aligned.words[1].charStart).toBeGreaterThanOrEqual(aligned.words[0].charEnd);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue