From 3f15636e0e948325dc06e089c316688cecc902e3 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 10:37:55 -0600 Subject: [PATCH] fix(highlight): support Japanese token matching --- src/hooks/epub/useEPUBHighlighting.ts | 35 ++++++ src/lib/client/epub/epub-word-highlight.ts | 16 ++- src/lib/client/pdf-highlight-match.ts | 101 +++++++++++++++ src/lib/client/pdf-highlight-worker.ts | 116 ++---------------- src/lib/client/pdf.ts | 9 +- tests/unit/epub-word-highlight.vitest.spec.ts | 30 +++++ tests/unit/pdf-highlight-match.vitest.spec.ts | 29 +++++ 7 files changed, 223 insertions(+), 113 deletions(-) create mode 100644 src/lib/client/pdf-highlight-match.ts create mode 100644 tests/unit/pdf-highlight-match.vitest.spec.ts diff --git a/src/hooks/epub/useEPUBHighlighting.ts b/src/hooks/epub/useEPUBHighlighting.ts index 8c31d34..83beafa 100644 --- a/src/hooks/epub/useEPUBHighlighting.ts +++ b/src/hooks/epub/useEPUBHighlighting.ts @@ -6,6 +6,7 @@ import type { Rendition } from 'epubjs'; import { buildMonotonicWordToTokenMap, buildWordHighlightCacheKey, + resolveAlignmentWordSourceRange, tokenizeCanonicalSegment, type EpubCanonicalWordToken, } from '@/lib/client/epub/epub-word-highlight'; @@ -118,6 +119,40 @@ export function useEPUBHighlighting({ const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment); if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return; + const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]); + if ( + alignmentRange + && alignmentRange.sourceStart >= resolved.startOffset + && alignmentRange.sourceEnd <= resolved.endOffset + ) { + const wordRange = createRangeFromMappedOffsets( + resolved.map, + alignmentRange.sourceStart, + alignmentRange.sourceEnd, + ); + if (wordRange) { + try { + const wordCfi = resolved.map.content.cfiFromRange(wordRange); + currentWordHighlightCfiRef.current = wordCfi; + renditionRef.current.annotations.add( + 'highlight', + wordCfi, + {}, + () => { }, + '', + { + fill: 'var(--accent)', + 'fill-opacity': '0.4', + 'mix-blend-mode': 'multiply', + } + ); + return; + } catch (error) { + console.error('Error highlighting EPUB word from alignment offsets:', error); + } + } + } + const cacheKey = buildWordHighlightCacheKey(segment, alignment, language); if (wordHighlightMapCacheRef.current?.key !== cacheKey) { const tokens = tokenizeCanonicalSegment(segment, language); diff --git a/src/lib/client/epub/epub-word-highlight.ts b/src/lib/client/epub/epub-word-highlight.ts index 5e90cda..7007ce5 100644 --- a/src/lib/client/epub/epub-word-highlight.ts +++ b/src/lib/client/epub/epub-word-highlight.ts @@ -1,5 +1,5 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; -import type { TTSSentenceAlignment } from '@/types/tts'; +import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; export type EpubCanonicalWordToken = { @@ -11,6 +11,20 @@ export type EpubCanonicalWordToken = { export const normalizeWordForHighlight = (text: string): string => normalizeUnicodeToken(text); +export const resolveAlignmentWordSourceRange = ( + segment: CanonicalTtsSegment, + word: TTSSentenceWord, +): { sourceStart: number; sourceEnd: number } | null => { + const { charStart, charEnd } = word; + if (!Number.isInteger(charStart) || !Number.isInteger(charEnd)) return null; + if (charStart < 0 || charEnd <= charStart || charEnd > segment.text.length) return null; + + return { + sourceStart: segment.startAnchor.offset + charStart, + sourceEnd: segment.startAnchor.offset + charEnd, + }; +}; + export const tokenizeCanonicalSegment = ( segment: CanonicalTtsSegment, language?: string, diff --git a/src/lib/client/pdf-highlight-match.ts b/src/lib/client/pdf-highlight-match.ts new file mode 100644 index 0000000..b16ef5d --- /dev/null +++ b/src/lib/client/pdf-highlight-match.ts @@ -0,0 +1,101 @@ +import { CmpStr } from 'cmpstr'; + +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); + +export interface HighlightTokenMatchResult { + bestStart: number; + bestEnd: number; + rating: number; + lengthDiff: number; +} + +export function findBestHighlightTokenMatch( + patternTokens: string[], + tokenTexts: string[], +): HighlightTokenMatchResult { + const cleanPatternTokens = patternTokens.map((token) => token.trim()).filter(Boolean); + const cleanPattern = cleanPatternTokens.join(''); + const patternLen = cleanPattern.length; + const responseBase: HighlightTokenMatchResult = { + bestStart: -1, + bestEnd: -1, + rating: 0, + lengthDiff: Number.POSITIVE_INFINITY, + }; + + if (!patternLen || !tokenTexts.length) return responseBase; + + const patternTokenCount = cleanPatternTokens.length; + const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6)); + const maxWindowTokens = Math.max( + minWindowTokens, + Math.ceil(patternTokenCount * 1.4), + ); + + let bestStart = -1; + let bestEnd = -1; + let bestRating = 0; + let bestLengthDiff = Number.POSITIVE_INFINITY; + + for (let start = 0; start < tokenTexts.length; start += 1) { + let combined = ''; + + for ( + let offset = 0; + offset < maxWindowTokens && start + offset < tokenTexts.length; + offset += 1 + ) { + const token = tokenTexts[start + offset]; + combined += token; + + const windowSize = offset + 1; + if (windowSize < minWindowTokens) continue; + if (combined.length > patternLen * 2) break; + + const similarity = cmp.compare(combined, cleanPattern); + const lengthDiff = Math.abs(combined.length - patternLen); + const lengthPenalty = lengthDiff / patternLen; + const adjustedRating = similarity * (1 - lengthPenalty * 0.3); + + let boostedRating = adjustedRating; + const windowTokens = tokenTexts.slice(start, start + windowSize); + const maxPrefixCheck = Math.min( + windowTokens.length, + cleanPatternTokens.length, + 5, + ); + + let prefixMatches = 0; + for (let i = 0; i < maxPrefixCheck; i += 1) { + const tokenSim = cmp.compare(windowTokens[i], cleanPatternTokens[i]); + if (tokenSim < 0.8) break; + prefixMatches += 1; + } + + if (prefixMatches > 0) { + const prefixRatio = prefixMatches / maxPrefixCheck; + boostedRating = adjustedRating * (1 + prefixRatio * 0.25); + } + + if ( + boostedRating > bestRating + || ( + Math.abs(boostedRating - bestRating) < 1e-3 + && lengthDiff < bestLengthDiff + ) + ) { + bestRating = boostedRating; + bestLengthDiff = lengthDiff; + bestStart = start; + bestEnd = start + offset; + } + } + } + + return { + bestStart, + bestEnd, + rating: bestRating, + lengthDiff: bestLengthDiff, + }; +} diff --git a/src/lib/client/pdf-highlight-worker.ts b/src/lib/client/pdf-highlight-worker.ts index 5b41d45..6b396af 100644 --- a/src/lib/client/pdf-highlight-worker.ts +++ b/src/lib/client/pdf-highlight-worker.ts @@ -1,13 +1,11 @@ /// -import { CmpStr } from 'cmpstr'; - -const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); +import { findBestHighlightTokenMatch } from './pdf-highlight-match'; interface TokenMatchRequest { id: string; type: 'tokenMatch'; - pattern: string; + patternTokens: string[]; tokenTexts: string[]; } @@ -39,112 +37,14 @@ self.onmessage = (event: MessageEvent) => { const data = event.data; if (!data || data.type !== 'tokenMatch') return; - const { id, pattern, tokenTexts } = data; - - const cleanPattern = pattern.trim().replace(/\s+/g, ' '); - const patternLen = cleanPattern.length; - - const responseBase: TokenMatchResponse = { - id, - type: 'tokenMatchResult', - bestStart: -1, - bestEnd: -1, - rating: 0, - lengthDiff: Number.POSITIVE_INFINITY, - }; - - if (!patternLen || !tokenTexts.length) { - (self as unknown as DedicatedWorkerGlobalScope).postMessage(responseBase); - return; - } - - const patternTokens = cleanPattern.split(' ').filter(Boolean); - const patternTokenCount = patternTokens.length || 1; - - const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6)); - const maxWindowTokens = Math.max( - minWindowTokens, - Math.ceil(patternTokenCount * 1.4) - ); - - let bestStart = -1; - let bestEnd = -1; - let bestRating = 0; - let bestLengthDiff = Number.POSITIVE_INFINITY; - - for (let start = 0; start < tokenTexts.length; start++) { - let combined = ''; - - for ( - let offset = 0; - offset < maxWindowTokens && start + offset < tokenTexts.length; - offset++ - ) { - const token = tokenTexts[start + offset]; - combined = combined ? `${combined} ${token}` : token; - - const windowSize = offset + 1; - if (windowSize < minWindowTokens) continue; - if (combined.length > patternLen * 2) break; - - const similarity = cmp.compare(combined, cleanPattern); - const lengthDiff = Math.abs(combined.length - patternLen); - const lengthPenalty = lengthDiff / patternLen; - const adjustedRating = similarity * (1 - lengthPenalty * 0.3); - - // Prefix-alignment boost: - // Favour windows whose first few tokens closely match the beginning - // of the pattern, so we are less likely to cut off the first 1–2 words. - let boostedRating = adjustedRating; - if (patternTokens.length > 0) { - const windowTokens = tokenTexts.slice(start, start + windowSize); - const maxPrefixCheck = Math.min( - windowTokens.length, - patternTokens.length, - 5 - ); - - let prefixMatches = 0; - for (let i = 0; i < maxPrefixCheck; i++) { - const tokenText = windowTokens[i]; - const patternToken = patternTokens[i]; - // Require reasonably strong similarity for a prefix match - const tokenSim = cmp.compare(tokenText, patternToken); - if (tokenSim >= 0.8) { - prefixMatches++; - } else { - // Stop at the first non-matching leading token - break; - } - } - - if (prefixMatches > 0) { - const prefixRatio = prefixMatches / maxPrefixCheck; - const PREFIX_BOOST_FACTOR = 0.25; // up to +25% boost - boostedRating = adjustedRating * (1 + prefixRatio * PREFIX_BOOST_FACTOR); - } - } - - if ( - boostedRating > bestRating || - (Math.abs(boostedRating - bestRating) < 1e-3 && - lengthDiff < bestLengthDiff) - ) { - bestRating = boostedRating; - bestLengthDiff = lengthDiff; - bestStart = start; - bestEnd = start + offset; - } - } - } + const { id, patternTokens, tokenTexts } = data; + const result = findBestHighlightTokenMatch(patternTokens, tokenTexts); const response: TokenMatchResponse = { - ...responseBase, - bestStart, - bestEnd, - rating: bestRating, - lengthDiff: bestLengthDiff, + id, + type: 'tokenMatchResult', + ...result, }; (self as unknown as DedicatedWorkerGlobalScope).postMessage(response); -}; \ No newline at end of file +}; diff --git a/src/lib/client/pdf.ts b/src/lib/client/pdf.ts index 32a1a7e..b81234d 100644 --- a/src/lib/client/pdf.ts +++ b/src/lib/client/pdf.ts @@ -13,7 +13,7 @@ const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); interface HighlightTokenMatchRequest { id: string; type: 'tokenMatch'; - pattern: string; + patternTokens: string[]; tokenTexts: string[]; } @@ -46,7 +46,7 @@ function getHighlightWorker(): Worker | null { } function runHighlightTokenMatch( - pattern: string, + patternTokens: string[], tokenTexts: string[] ): Promise { const worker = getHighlightWorker(); @@ -71,7 +71,7 @@ function runHighlightTokenMatch( const message: HighlightTokenMatchRequest = { id, type: 'tokenMatch', - pattern, + patternTokens, tokenTexts, }; worker.postMessage(message); @@ -612,9 +612,10 @@ export function highlightPattern( }; const tokenTexts = tokens.map((t) => t.text); + const patternTokens = segmentWords(cleanPattern, options?.language).map((token) => token.text); // Fire-and-forget async worker call; UI thread returns immediately - runHighlightTokenMatch(cleanPattern, tokenTexts) + runHighlightTokenMatch(patternTokens, tokenTexts) .then((result) => { if (seq !== highlightPatternSeq) return; if (!result || result.bestStart === -1) { diff --git a/tests/unit/epub-word-highlight.vitest.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts index ad1f53c..20f0266 100644 --- a/tests/unit/epub-word-highlight.vitest.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'; import { buildMonotonicWordToTokenMap, + resolveAlignmentWordSourceRange, tokenizeCanonicalSegment, } from '../../src/lib/client/epub/epub-word-highlight'; import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan'; @@ -38,6 +39,35 @@ describe('EPUB word highlight mapping', () => { expect(chinese.map((token) => token.norm).join('')).toBe('这是中文'); }); + test('resolves Japanese alignment chunks directly from character offsets', () => { + const japanese = segment('これは日本語です。', 25); + const word: TTSSentenceAlignment['words'][number] = { + text: 'これは', + startSec: 0, + endSec: 0.5, + charStart: 0, + charEnd: 3, + }; + + expect(resolveAlignmentWordSourceRange(japanese, word)).toEqual({ + sourceStart: 25, + sourceEnd: 28, + }); + }); + + test('rejects invalid alignment character offsets so token mapping can be used', () => { + const japanese = segment('これは日本語です。', 25); + const word: TTSSentenceAlignment['words'][number] = { + text: '範囲外', + startSec: 0, + endSec: 0.5, + charStart: 20, + charEnd: 23, + }; + + expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull(); + }); + test('tokenizes canonical segment words with source offsets', () => { const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); diff --git a/tests/unit/pdf-highlight-match.vitest.spec.ts b/tests/unit/pdf-highlight-match.vitest.spec.ts new file mode 100644 index 0000000..716fc26 --- /dev/null +++ b/tests/unit/pdf-highlight-match.vitest.spec.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from 'vitest'; + +import { findBestHighlightTokenMatch } from '../../src/lib/client/pdf-highlight-match'; +import { segmentWords } from '../../src/lib/shared/language'; + +describe('PDF highlight token matching', () => { + test('matches a complete Japanese sentence using locale-aware token count', () => { + const sentence = 'これは日本語です。'; + const patternTokens = segmentWords(sentence, 'ja').map((token) => token.text); + const tokenTexts = ['前文', ...patternTokens, '次文']; + + expect(findBestHighlightTokenMatch(patternTokens, tokenTexts)).toMatchObject({ + bestStart: 1, + bestEnd: patternTokens.length, + lengthDiff: 0, + }); + }); + + test('matches spaced Latin text without relying on whitespace tokens', () => { + expect(findBestHighlightTokenMatch( + ['hello', 'world'], + ['before', 'hello', 'world', 'after'], + )).toMatchObject({ + bestStart: 1, + bestEnd: 2, + lengthDiff: 0, + }); + }); +});