feat(pdf): modernize text highlighting and parsing for fluidity
- Offload computationally intensive text matching for real-time highlighting to a dedicated Web Worker, ensuring the main thread remains responsive during playback. - Implement a new overlay-based highlighting system that renders independently of the PDF's text layer, providing smoother and more reliable visual feedback without interfering with document rendering. - Introduce a new setting allowing users to enable or disable real-time text highlighting in PDFs, offering personalized control over the reading interface. - Upgrade the underlying text comparison algorithm to Dice similarity for more accurate and context-aware matching of spoken words to on-screen text, improving synchronization precision. - Improve sentence boundary detection, especially for quoted dialogue and complex structures, by enhancing the NLP processing logic, leading to a more natural audio-text flow.
This commit is contained in:
parent
2f39e8f014
commit
071d967f8f
7 changed files with 545 additions and 114 deletions
|
|
@ -32,7 +32,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
||||||
footerMargin,
|
footerMargin,
|
||||||
leftMargin,
|
leftMargin,
|
||||||
rightMargin,
|
rightMargin,
|
||||||
updateConfigKey
|
updateConfigKey,
|
||||||
|
pdfHighlightEnabled,
|
||||||
} = useConfig();
|
} = useConfig();
|
||||||
const { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter } = useEPUB();
|
const { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter } = useEPUB();
|
||||||
const { createFullAudioBook: createPDFAudioBook, regenerateChapter: regeneratePDFChapter } = usePDF();
|
const { createFullAudioBook: createPDFAudioBook, regenerateChapter: regeneratePDFChapter } = usePDF();
|
||||||
|
|
@ -327,6 +328,22 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
{!epub && !html && (
|
||||||
|
<div className="space-y-1">
|
||||||
|
<label className="flex items-center space-x-2">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={pdfHighlightEnabled}
|
||||||
|
onChange={(e) => updateConfigKey('pdfHighlightEnabled', e.target.checked)}
|
||||||
|
className="form-checkbox h-4 w-4 text-accent rounded border-muted"
|
||||||
|
/>
|
||||||
|
<span className="text-sm font-medium text-foreground">Highlight text during playback</span>
|
||||||
|
</label>
|
||||||
|
<p className="text-sm text-muted pl-6">
|
||||||
|
Show visual highlighting in the PDF viewer while TTS is reading.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{epub && (
|
{epub && (
|
||||||
<div className="space-y-1">
|
<div className="space-y-1">
|
||||||
<label className="flex items-center space-x-2">
|
<label className="flex items-center space-x-2">
|
||||||
|
|
|
||||||
|
|
@ -25,13 +25,11 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
const { containerWidth } = usePDFResize(containerRef);
|
const { containerWidth } = usePDFResize(containerRef);
|
||||||
|
|
||||||
// Config context
|
// Config context
|
||||||
const { viewType } = useConfig();
|
const { viewType, pdfHighlightEnabled } = useConfig();
|
||||||
|
|
||||||
// TTS context
|
// TTS context
|
||||||
const {
|
const {
|
||||||
currentSentence,
|
currentSentence,
|
||||||
stopAndPlayFromIndex,
|
|
||||||
isProcessing,
|
|
||||||
skipToLocation,
|
skipToLocation,
|
||||||
} = useTTS();
|
} = useTTS();
|
||||||
|
|
||||||
|
|
@ -39,7 +37,6 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
const {
|
const {
|
||||||
highlightPattern,
|
highlightPattern,
|
||||||
clearHighlights,
|
clearHighlights,
|
||||||
handleTextClick,
|
|
||||||
onDocumentLoadSuccess,
|
onDocumentLoadSuccess,
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocPages,
|
currDocPages,
|
||||||
|
|
@ -47,51 +44,6 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
currDocPage,
|
currDocPage,
|
||||||
} = usePDF();
|
} = usePDF();
|
||||||
|
|
||||||
// Add static styles once during component mount
|
|
||||||
useEffect(() => {
|
|
||||||
const styleElement = document.createElement('style');
|
|
||||||
styleElement.textContent = `
|
|
||||||
.react-pdf__Page__textContent span {
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background-color 0.2s ease;
|
|
||||||
}
|
|
||||||
.react-pdf__Page__textContent span:hover {
|
|
||||||
background-color: rgba(255, 255, 0, 0.2) !important;
|
|
||||||
}
|
|
||||||
`;
|
|
||||||
document.head.appendChild(styleElement);
|
|
||||||
return () => {
|
|
||||||
styleElement.remove();
|
|
||||||
};
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
/*
|
|
||||||
* Sets up click event listeners for text selection in the PDF.
|
|
||||||
* Cleans up by removing the event listener when component unmounts.
|
|
||||||
*
|
|
||||||
* Dependencies:
|
|
||||||
* - pdfText: Re-run when the extracted text content changes
|
|
||||||
* - handleTextClick: Function from context that could change
|
|
||||||
* - stopAndPlayFromIndex: Function from context that could change
|
|
||||||
*/
|
|
||||||
const container = containerRef.current;
|
|
||||||
if (!container) return;
|
|
||||||
if (!currDocText) return;
|
|
||||||
|
|
||||||
const handleClick = (event: MouseEvent) => handleTextClick(
|
|
||||||
event,
|
|
||||||
currDocText,
|
|
||||||
containerRef as RefObject<HTMLDivElement>,
|
|
||||||
stopAndPlayFromIndex,
|
|
||||||
isProcessing
|
|
||||||
);
|
|
||||||
container.addEventListener('click', handleClick);
|
|
||||||
return () => {
|
|
||||||
container.removeEventListener('click', handleClick);
|
|
||||||
};
|
|
||||||
}, [currDocText, handleTextClick, stopAndPlayFromIndex, isProcessing]);
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
/*
|
/*
|
||||||
* Handles highlighting the current sentence being read by TTS.
|
* Handles highlighting the current sentence being read by TTS.
|
||||||
|
|
@ -103,7 +55,11 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
* - highlightPattern: Function from context that could change
|
* - highlightPattern: Function from context that could change
|
||||||
* - clearHighlights: Function from context that could change
|
* - clearHighlights: Function from context that could change
|
||||||
*/
|
*/
|
||||||
if (!currDocText) return;
|
|
||||||
|
if (!currDocText || !pdfHighlightEnabled) {
|
||||||
|
clearHighlights();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const highlightTimeout = setTimeout(() => {
|
const highlightTimeout = setTimeout(() => {
|
||||||
if (containerRef.current) {
|
if (containerRef.current) {
|
||||||
|
|
@ -115,7 +71,7 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
clearTimeout(highlightTimeout);
|
clearTimeout(highlightTimeout);
|
||||||
clearHighlights();
|
clearHighlights();
|
||||||
};
|
};
|
||||||
}, [currDocText, currentSentence, highlightPattern, clearHighlights]);
|
}, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]);
|
||||||
|
|
||||||
// Add page dimensions state
|
// Add page dimensions state
|
||||||
const [pageWidth, setPageWidth] = useState<number>(595); // default A4 width
|
const [pageWidth, setPageWidth] = useState<number>(595); // default A4 width
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ type ConfigValues = {
|
||||||
ttsInstructions: string;
|
ttsInstructions: string;
|
||||||
savedVoices: SavedVoices;
|
savedVoices: SavedVoices;
|
||||||
smartSentenceSplitting: boolean;
|
smartSentenceSplitting: boolean;
|
||||||
|
pdfHighlightEnabled: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Interface defining the configuration context shape and functionality */
|
/** Interface defining the configuration context shape and functionality */
|
||||||
|
|
@ -55,6 +56,7 @@ interface ConfigContextType {
|
||||||
updateConfigKey: <K extends keyof ConfigValues>(key: K, value: ConfigValues[K]) => Promise<void>;
|
updateConfigKey: <K extends keyof ConfigValues>(key: K, value: ConfigValues[K]) => Promise<void>;
|
||||||
isLoading: boolean;
|
isLoading: boolean;
|
||||||
isDBReady: boolean;
|
isDBReady: boolean;
|
||||||
|
pdfHighlightEnabled: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ConfigContext = createContext<ConfigContextType | undefined>(undefined);
|
const ConfigContext = createContext<ConfigContextType | undefined>(undefined);
|
||||||
|
|
@ -76,14 +78,15 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
const [skipBlank, setSkipBlank] = useState<boolean>(true);
|
const [skipBlank, setSkipBlank] = useState<boolean>(true);
|
||||||
const [epubTheme, setEpubTheme] = useState<boolean>(false);
|
const [epubTheme, setEpubTheme] = useState<boolean>(false);
|
||||||
const [smartSentenceSplitting, setSmartSentenceSplitting] = useState<boolean>(true);
|
const [smartSentenceSplitting, setSmartSentenceSplitting] = useState<boolean>(true);
|
||||||
const [headerMargin, setHeaderMargin] = useState<number>(0.07);
|
const [headerMargin, setHeaderMargin] = useState<number>(0.0);
|
||||||
const [footerMargin, setFooterMargin] = useState<number>(0.07);
|
const [footerMargin, setFooterMargin] = useState<number>(0.0);
|
||||||
const [leftMargin, setLeftMargin] = useState<number>(0.07);
|
const [leftMargin, setLeftMargin] = useState<number>(0.0);
|
||||||
const [rightMargin, setRightMargin] = useState<number>(0.07);
|
const [rightMargin, setRightMargin] = useState<number>(0.0);
|
||||||
const [ttsProvider, setTTSProvider] = useState<string>('custom-openai');
|
const [ttsProvider, setTTSProvider] = useState<string>('custom-openai');
|
||||||
const [ttsModel, setTTSModel] = useState<string>('kokoro');
|
const [ttsModel, setTTSModel] = useState<string>('kokoro');
|
||||||
const [ttsInstructions, setTTSInstructions] = useState<string>('');
|
const [ttsInstructions, setTTSInstructions] = useState<string>('');
|
||||||
const [savedVoices, setSavedVoices] = useState<SavedVoices>({});
|
const [savedVoices, setSavedVoices] = useState<SavedVoices>({});
|
||||||
|
const [pdfHighlightEnabled, setPdfHighlightEnabled] = useState<boolean>(true);
|
||||||
|
|
||||||
const [isLoading, setIsLoading] = useState(true);
|
const [isLoading, setIsLoading] = useState(true);
|
||||||
const [isDBReady, setIsDBReady] = useState(false);
|
const [isDBReady, setIsDBReady] = useState(false);
|
||||||
|
|
@ -115,6 +118,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
const cachedTTSModel = await getItem('ttsModel');
|
const cachedTTSModel = await getItem('ttsModel');
|
||||||
const cachedTTSInstructions = await getItem('ttsInstructions');
|
const cachedTTSInstructions = await getItem('ttsInstructions');
|
||||||
const cachedSavedVoices = await getItem('savedVoices');
|
const cachedSavedVoices = await getItem('savedVoices');
|
||||||
|
const cachedPdfHighlightEnabled = await getItem('pdfHighlightEnabled');
|
||||||
|
|
||||||
// Migration logic: infer provider and baseUrl for returning users
|
// Migration logic: infer provider and baseUrl for returning users
|
||||||
let inferredProvider = cachedTTSProvider || '';
|
let inferredProvider = cachedTTSProvider || '';
|
||||||
|
|
@ -200,6 +204,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
const finalModel = cachedTTSModel || (inferredProvider === 'openai' ? 'tts-1' : inferredProvider === 'deepinfra' ? 'hexgrad/Kokoro-82M' : 'kokoro');
|
const finalModel = cachedTTSModel || (inferredProvider === 'openai' ? 'tts-1' : inferredProvider === 'deepinfra' ? 'hexgrad/Kokoro-82M' : 'kokoro');
|
||||||
setTTSModel(finalModel);
|
setTTSModel(finalModel);
|
||||||
setTTSInstructions(cachedTTSInstructions || '');
|
setTTSInstructions(cachedTTSInstructions || '');
|
||||||
|
setPdfHighlightEnabled(cachedPdfHighlightEnabled === 'false' ? false : true);
|
||||||
|
|
||||||
// Restore voice for current provider-model if available in savedVoices
|
// Restore voice for current provider-model if available in savedVoices
|
||||||
const voiceKey = getVoiceKey(inferredProvider || 'custom-openai', finalModel);
|
const voiceKey = getVoiceKey(inferredProvider || 'custom-openai', finalModel);
|
||||||
|
|
@ -219,8 +224,8 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
if (cachedSmartSentenceSplitting === null) {
|
if (cachedSmartSentenceSplitting === null) {
|
||||||
await setItem('smartSentenceSplitting', 'true');
|
await setItem('smartSentenceSplitting', 'true');
|
||||||
}
|
}
|
||||||
if (cachedHeaderMargin === null) await setItem('headerMargin', '0.07');
|
if (cachedHeaderMargin === null) await setItem('headerMargin', '0.0');
|
||||||
if (cachedFooterMargin === null) await setItem('footerMargin', '0.07');
|
if (cachedFooterMargin === null) await setItem('footerMargin', '0.0');
|
||||||
if (cachedLeftMargin === null) await setItem('leftMargin', '0.0');
|
if (cachedLeftMargin === null) await setItem('leftMargin', '0.0');
|
||||||
if (cachedRightMargin === null) await setItem('rightMargin', '0.0');
|
if (cachedRightMargin === null) await setItem('rightMargin', '0.0');
|
||||||
if (cachedTTSProvider === null && inferredProvider) {
|
if (cachedTTSProvider === null && inferredProvider) {
|
||||||
|
|
@ -247,6 +252,10 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
// Always ensure voice is not stored standalone - only in savedVoices
|
// Always ensure voice is not stored standalone - only in savedVoices
|
||||||
await removeItem('voice');
|
await removeItem('voice');
|
||||||
|
|
||||||
|
if (cachedPdfHighlightEnabled === null) {
|
||||||
|
await setItem('pdfHighlightEnabled', 'true');
|
||||||
|
}
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error initializing:', error);
|
console.error('Error initializing:', error);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -379,6 +388,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
case 'ttsInstructions':
|
case 'ttsInstructions':
|
||||||
setTTSInstructions(value as string);
|
setTTSInstructions(value as string);
|
||||||
break;
|
break;
|
||||||
|
case 'pdfHighlightEnabled':
|
||||||
|
setPdfHighlightEnabled(value as boolean);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -411,7 +423,8 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
||||||
updateConfig,
|
updateConfig,
|
||||||
updateConfigKey,
|
updateConfigKey,
|
||||||
isLoading,
|
isLoading,
|
||||||
isDBReady
|
isDBReady,
|
||||||
|
pdfHighlightEnabled
|
||||||
}}>
|
}}>
|
||||||
{children}
|
{children}
|
||||||
</ConfigContext.Provider>
|
</ConfigContext.Provider>
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,8 @@ interface PDFContextType {
|
||||||
pdfText: string,
|
pdfText: string,
|
||||||
containerRef: RefObject<HTMLDivElement>,
|
containerRef: RefObject<HTMLDivElement>,
|
||||||
stopAndPlayFromIndex: (index: number) => void,
|
stopAndPlayFromIndex: (index: number) => void,
|
||||||
isProcessing: boolean
|
isProcessing: boolean,
|
||||||
|
enableHighlight?: boolean
|
||||||
) => void;
|
) => void;
|
||||||
createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise<string>;
|
createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise<string>;
|
||||||
regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', onProgress: (progress: number) => void, signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>;
|
regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', onProgress: (progress: number) => void, signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>;
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
|
|
||||||
import nlp from 'compromise';
|
import nlp from 'compromise';
|
||||||
|
|
||||||
const MAX_BLOCK_LENGTH = 300;
|
const MAX_BLOCK_LENGTH = 450;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Preprocesses text for audio generation by cleaning up various text artifacts
|
* Preprocesses text for audio generation by cleaning up various text artifacts
|
||||||
|
|
@ -79,13 +79,8 @@ export const processTextToSentences = (text: string): string[] => {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
if (text.length <= MAX_BLOCK_LENGTH) {
|
// Always use the full splitting logic so we consistently respect
|
||||||
// Single sentence preprocessing
|
// sentence boundaries and quoted dialogue, even for shorter texts.
|
||||||
const cleanedText = preprocessSentenceForAudio(text);
|
|
||||||
return [cleanedText];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Full text splitting into sentences
|
|
||||||
return splitIntoSentences(text);
|
return splitIntoSentences(text);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -160,20 +155,18 @@ const countDoubleQuotes = (s: string): number => {
|
||||||
return matches ? matches.length : 0;
|
return matches ? matches.length : 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
const countCurlySingleQuotes = (s: string): number => {
|
// Replace the old curly single-quote counter and standalone-straight counter with a unified, context-aware counter
|
||||||
const matches = s.match(/[‘’]/g);
|
const countNonApostropheSingleQuotes = (s: string): number => {
|
||||||
return matches ? matches.length : 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
const countStandaloneStraightSingles = (s: string): number => {
|
|
||||||
let count = 0;
|
let count = 0;
|
||||||
for (let i = 0; i < s.length; i++) {
|
for (let i = 0; i < s.length; i++) {
|
||||||
if (s[i] === "'") {
|
const ch = s[i];
|
||||||
|
if (ch === "'" || ch === '‘' || ch === '’') {
|
||||||
const prev = i > 0 ? s[i - 1] : '';
|
const prev = i > 0 ? s[i - 1] : '';
|
||||||
const next = i + 1 < s.length ? s[i + 1] : '';
|
const next = i + 1 < s.length ? s[i + 1] : '';
|
||||||
const isPrevAlphaNum = /[A-Za-z0-9]/.test(prev);
|
const isPrevAlphaNum = /[A-Za-z0-9]/.test(prev);
|
||||||
const isNextAlphaNum = /[A-Za-z0-9]/.test(next);
|
const isNextAlphaNum = /[A-Za-z0-9]/.test(next);
|
||||||
// Count only when not clearly an apostrophe inside a word (e.g., don't)
|
// Treat as a real quote mark only when it's not clearly an apostrophe
|
||||||
|
// between two alphanumeric characters (e.g., don't, WizardLM’s).
|
||||||
if (!(isPrevAlphaNum && isNextAlphaNum)) {
|
if (!(isPrevAlphaNum && isNextAlphaNum)) {
|
||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
|
|
@ -191,7 +184,10 @@ const mergeQuotedDialogue = (rawSentences: string[]): string[] => {
|
||||||
for (const s of rawSentences) {
|
for (const s of rawSentences) {
|
||||||
const t = s.trim();
|
const t = s.trim();
|
||||||
const dblCount = countDoubleQuotes(t);
|
const dblCount = countDoubleQuotes(t);
|
||||||
const singleCount = countCurlySingleQuotes(t) + countStandaloneStraightSingles(t);
|
// Use the new context-aware single-quote counter so curly apostrophes
|
||||||
|
// inside words don't incorrectly toggle quote state and merge large
|
||||||
|
// regions of plain prose into one block.
|
||||||
|
const singleCount = countNonApostropheSingleQuotes(t);
|
||||||
|
|
||||||
if (insideDouble || insideSingle) {
|
if (insideDouble || insideSingle) {
|
||||||
buffer = buffer ? `${buffer} ${t}` : t;
|
buffer = buffer ? `${buffer} ${t}` : t;
|
||||||
|
|
|
||||||
387
src/utils/pdf.ts
387
src/utils/pdf.ts
|
|
@ -5,7 +5,76 @@ import "core-js/proposals/promise-with-resolvers";
|
||||||
import { processTextToSentences } from '@/utils/nlp';
|
import { processTextToSentences } from '@/utils/nlp';
|
||||||
import { CmpStr } from 'cmpstr';
|
import { CmpStr } from 'cmpstr';
|
||||||
|
|
||||||
const cmp = CmpStr.create().setMetric( 'levenshtein' ).setFlags( 'i' );
|
const cmp = CmpStr.create().setMetric( 'dice' ).setFlags( 'itw' );
|
||||||
|
|
||||||
|
// Worker coordination for offloading highlight token matching
|
||||||
|
interface HighlightTokenMatchRequest {
|
||||||
|
id: string;
|
||||||
|
type: 'tokenMatch';
|
||||||
|
pattern: string;
|
||||||
|
tokenTexts: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HighlightTokenMatchResponse {
|
||||||
|
id: string;
|
||||||
|
type: 'tokenMatchResult';
|
||||||
|
bestStart: number;
|
||||||
|
bestEnd: number;
|
||||||
|
rating: number;
|
||||||
|
lengthDiff: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
let highlightWorker: Worker | null = null;
|
||||||
|
|
||||||
|
function getHighlightWorker(): Worker | null {
|
||||||
|
if (typeof window === 'undefined') return null;
|
||||||
|
if (highlightWorker) return highlightWorker;
|
||||||
|
|
||||||
|
try {
|
||||||
|
highlightWorker = new Worker(
|
||||||
|
new URL('pdfHighlightWorker.ts', import.meta.url),
|
||||||
|
{ type: 'module' }
|
||||||
|
);
|
||||||
|
return highlightWorker;
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to initialize PDF highlight worker:', e);
|
||||||
|
highlightWorker = null;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function runHighlightTokenMatch(
|
||||||
|
pattern: string,
|
||||||
|
tokenTexts: string[]
|
||||||
|
): Promise<HighlightTokenMatchResponse | null> {
|
||||||
|
const worker = getHighlightWorker();
|
||||||
|
if (!worker) {
|
||||||
|
return Promise.resolve(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
const id = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||||
|
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const handleMessage = (event: MessageEvent) => {
|
||||||
|
const data = event.data as HighlightTokenMatchResponse;
|
||||||
|
if (!data || data.id !== id || data.type !== 'tokenMatchResult') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
worker.removeEventListener('message', handleMessage as EventListener);
|
||||||
|
resolve(data);
|
||||||
|
};
|
||||||
|
|
||||||
|
worker.addEventListener('message', handleMessage as EventListener);
|
||||||
|
|
||||||
|
const message: HighlightTokenMatchRequest = {
|
||||||
|
id,
|
||||||
|
type: 'tokenMatch',
|
||||||
|
pattern,
|
||||||
|
tokenTexts,
|
||||||
|
};
|
||||||
|
worker.postMessage(message);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Function to detect if we need to use legacy build
|
// Function to detect if we need to use legacy build
|
||||||
function shouldUseLegacyBuild() {
|
function shouldUseLegacyBuild() {
|
||||||
|
|
@ -202,6 +271,14 @@ export function clearHighlights() {
|
||||||
element.style.backgroundColor = '';
|
element.style.backgroundColor = '';
|
||||||
element.style.opacity = '1';
|
element.style.opacity = '1';
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const overlays = document.querySelectorAll('.pdf-text-highlight-overlay');
|
||||||
|
overlays.forEach((node) => {
|
||||||
|
const element = node as HTMLElement;
|
||||||
|
if (element.parentElement) {
|
||||||
|
element.parentElement.removeChild(element);
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function findBestTextMatch(
|
export function findBestTextMatch(
|
||||||
|
|
@ -256,55 +333,268 @@ export function highlightPattern(
|
||||||
clearHighlights();
|
clearHighlights();
|
||||||
|
|
||||||
if (!pattern?.trim()) return;
|
if (!pattern?.trim()) return;
|
||||||
|
|
||||||
const cleanPattern = pattern.trim().replace(/\s+/g, ' ');
|
|
||||||
const container = containerRef.current;
|
const container = containerRef.current;
|
||||||
if (!container) return;
|
if (!container) return;
|
||||||
|
|
||||||
const textNodes = container.querySelectorAll('.react-pdf__Page__textContent span');
|
const cleanPattern = pattern.trim().replace(/\s+/g, ' ');
|
||||||
const allText = Array.from(textNodes).map((node) => ({
|
if (!cleanPattern) return;
|
||||||
element: node as HTMLElement,
|
|
||||||
text: (node.textContent || '').trim(),
|
|
||||||
})).filter((node) => node.text.length > 0);
|
|
||||||
|
|
||||||
const containerRect = container.getBoundingClientRect();
|
const spanNodes = Array.from(
|
||||||
const visibleTop = container.scrollTop;
|
container.querySelectorAll('.react-pdf__Page__textContent span')
|
||||||
const visibleBottom = visibleTop + containerRect.height;
|
) as HTMLElement[];
|
||||||
const bufferSize = containerRect.height;
|
|
||||||
|
|
||||||
const visibleNodes = allText.filter(({ element }) => {
|
if (!spanNodes.length) return;
|
||||||
const rect = element.getBoundingClientRect();
|
|
||||||
const elementTop = rect.top - containerRect.top + container.scrollTop;
|
type Token = {
|
||||||
return elementTop >= (visibleTop - bufferSize) && elementTop <= (visibleBottom + bufferSize);
|
spanIndex: number;
|
||||||
|
textNode: Text;
|
||||||
|
text: string;
|
||||||
|
startOffset: number;
|
||||||
|
endOffset: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const tokens: Token[] = [];
|
||||||
|
|
||||||
|
spanNodes.forEach((span, spanIndex) => {
|
||||||
|
const node = span.firstChild;
|
||||||
|
if (!node || node.nodeType !== Node.TEXT_NODE) return;
|
||||||
|
|
||||||
|
const textNode = node as Text;
|
||||||
|
const textContent = textNode.textContent || '';
|
||||||
|
const wordRegex = /\S+/g;
|
||||||
|
let match: RegExpExecArray | null;
|
||||||
|
|
||||||
|
while ((match = wordRegex.exec(textContent)) !== null) {
|
||||||
|
const word = match[0];
|
||||||
|
tokens.push({
|
||||||
|
spanIndex,
|
||||||
|
textNode,
|
||||||
|
text: word,
|
||||||
|
startOffset: match.index,
|
||||||
|
endOffset: match.index + word.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
let bestMatch = findBestTextMatch(visibleNodes, cleanPattern, cleanPattern.length * 2);
|
if (!tokens.length) return;
|
||||||
|
|
||||||
if (bestMatch.rating < 0.3) {
|
const patternLen = cleanPattern.length;
|
||||||
bestMatch = findBestTextMatch(allText, cleanPattern, cleanPattern.length * 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
const similarityThreshold = bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5;
|
// Core application of highlight logic once we know the best token window (if any)
|
||||||
|
const applyHighlightFromTokens = (
|
||||||
|
tokenMatch:
|
||||||
|
| {
|
||||||
|
bestStart: number;
|
||||||
|
bestEnd: number;
|
||||||
|
rating: number;
|
||||||
|
lengthDiff: number;
|
||||||
|
}
|
||||||
|
| null
|
||||||
|
) => {
|
||||||
|
const highlightRanges: Array<{
|
||||||
|
textNode: Text;
|
||||||
|
startOffset: number;
|
||||||
|
endOffset: number;
|
||||||
|
span: HTMLElement;
|
||||||
|
}> = [];
|
||||||
|
|
||||||
if (bestMatch.rating >= similarityThreshold) {
|
let bestStart = -1;
|
||||||
bestMatch.elements.forEach((element) => {
|
let bestEnd = -1;
|
||||||
element.style.backgroundColor = 'grey';
|
let bestRating = 0;
|
||||||
element.style.opacity = '0.4';
|
let bestLengthDiff = Infinity;
|
||||||
});
|
|
||||||
|
|
||||||
if (bestMatch.elements.length > 0) {
|
if (tokenMatch) {
|
||||||
const element = bestMatch.elements[0];
|
bestStart = tokenMatch.bestStart;
|
||||||
const elementRect = element.getBoundingClientRect();
|
bestEnd = tokenMatch.bestEnd;
|
||||||
const elementTop = elementRect.top - containerRect.top + container.scrollTop;
|
bestRating = tokenMatch.rating;
|
||||||
|
bestLengthDiff = tokenMatch.lengthDiff;
|
||||||
|
}
|
||||||
|
|
||||||
if (elementTop < visibleTop || elementTop > visibleBottom) {
|
const hasTokenMatch = bestStart !== -1;
|
||||||
container.scrollTo({
|
const similarityThreshold =
|
||||||
top: elementTop - containerRect.height / 3,
|
bestLengthDiff < patternLen * 0.3 ? 0.3 : 0.5;
|
||||||
behavior: 'smooth',
|
|
||||||
|
if (hasTokenMatch && bestRating >= similarityThreshold) {
|
||||||
|
const rangesBySpan = new Map<
|
||||||
|
number,
|
||||||
|
{ startOffset: number; endOffset: number }
|
||||||
|
>();
|
||||||
|
|
||||||
|
for (let i = bestStart; i <= bestEnd; i++) {
|
||||||
|
const token = tokens[i];
|
||||||
|
const existing = rangesBySpan.get(token.spanIndex);
|
||||||
|
if (!existing) {
|
||||||
|
rangesBySpan.set(token.spanIndex, {
|
||||||
|
startOffset: token.startOffset,
|
||||||
|
endOffset: token.endOffset,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
existing.startOffset = Math.min(
|
||||||
|
existing.startOffset,
|
||||||
|
token.startOffset
|
||||||
|
);
|
||||||
|
existing.endOffset = Math.max(
|
||||||
|
existing.endOffset,
|
||||||
|
token.endOffset
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rangesBySpan.forEach(({ startOffset, endOffset }, spanIndex) => {
|
||||||
|
const span = spanNodes[spanIndex];
|
||||||
|
const node = span.firstChild;
|
||||||
|
if (!node || node.nodeType !== Node.TEXT_NODE) return;
|
||||||
|
|
||||||
|
highlightRanges.push({
|
||||||
|
textNode: node as Text,
|
||||||
|
startOffset,
|
||||||
|
endOffset,
|
||||||
|
span,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: if token-level matching failed, use span-based fuzzy matching
|
||||||
|
if (!highlightRanges.length) {
|
||||||
|
const spanEntries = spanNodes
|
||||||
|
.map((node) => ({
|
||||||
|
element: node as HTMLElement,
|
||||||
|
text: (node.textContent || '').trim(),
|
||||||
|
}))
|
||||||
|
.filter((entry) => entry.text.length > 0);
|
||||||
|
|
||||||
|
const containerRect = container.getBoundingClientRect();
|
||||||
|
const visibleTop = container.scrollTop;
|
||||||
|
const visibleBottom = visibleTop + containerRect.height;
|
||||||
|
const bufferSize = containerRect.height;
|
||||||
|
|
||||||
|
const visibleNodes = spanEntries.filter(({ element }) => {
|
||||||
|
const rect = element.getBoundingClientRect();
|
||||||
|
const elementTop =
|
||||||
|
rect.top - containerRect.top + container.scrollTop;
|
||||||
|
return (
|
||||||
|
elementTop >= visibleTop - bufferSize &&
|
||||||
|
elementTop <= visibleBottom + bufferSize
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
let bestMatch = findBestTextMatch(
|
||||||
|
visibleNodes,
|
||||||
|
cleanPattern,
|
||||||
|
cleanPattern.length * 2
|
||||||
|
);
|
||||||
|
|
||||||
|
if (bestMatch.rating < 0.3) {
|
||||||
|
bestMatch = findBestTextMatch(
|
||||||
|
spanEntries,
|
||||||
|
cleanPattern,
|
||||||
|
cleanPattern.length * 2
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const spanSimilarityThreshold =
|
||||||
|
bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5;
|
||||||
|
|
||||||
|
if (bestMatch.rating >= spanSimilarityThreshold) {
|
||||||
|
bestMatch.elements.forEach((element) => {
|
||||||
|
const node = element.firstChild;
|
||||||
|
if (!node || node.nodeType !== Node.TEXT_NODE) return;
|
||||||
|
const textNode = node as Text;
|
||||||
|
const content = textNode.textContent || '';
|
||||||
|
if (!content) return;
|
||||||
|
|
||||||
|
highlightRanges.push({
|
||||||
|
textNode,
|
||||||
|
startOffset: 0,
|
||||||
|
endOffset: content.length,
|
||||||
|
span: element,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
if (!highlightRanges.length) return;
|
||||||
|
|
||||||
|
// Create overlay rectangles for each range, relative to its page text layer
|
||||||
|
const scrollIntoViewRects: DOMRect[] = [];
|
||||||
|
|
||||||
|
highlightRanges.forEach(({ textNode, startOffset, endOffset, span }) => {
|
||||||
|
try {
|
||||||
|
const range = document.createRange();
|
||||||
|
range.setStart(textNode, startOffset);
|
||||||
|
range.setEnd(textNode, endOffset);
|
||||||
|
|
||||||
|
const pageLayer = span.closest(
|
||||||
|
'.react-pdf__Page__textContent'
|
||||||
|
) as HTMLElement | null;
|
||||||
|
if (!pageLayer) return;
|
||||||
|
|
||||||
|
const pageRect = pageLayer.getBoundingClientRect();
|
||||||
|
const rects = Array.from(range.getClientRects());
|
||||||
|
|
||||||
|
rects.forEach((rect) => {
|
||||||
|
const highlight = document.createElement('div');
|
||||||
|
highlight.className = 'pdf-text-highlight-overlay';
|
||||||
|
highlight.style.position = 'absolute';
|
||||||
|
highlight.style.backgroundColor = 'grey';
|
||||||
|
highlight.style.opacity = '0.4';
|
||||||
|
highlight.style.pointerEvents = 'none';
|
||||||
|
highlight.style.left = `${rect.left - pageRect.left}px`;
|
||||||
|
highlight.style.top = `${rect.top - pageRect.top}px`;
|
||||||
|
highlight.style.width = `${rect.width}px`;
|
||||||
|
highlight.style.height = `${rect.height}px`;
|
||||||
|
pageLayer.appendChild(highlight);
|
||||||
|
|
||||||
|
scrollIntoViewRects.push(rect);
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// If range creation fails for any reason, skip this segment
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!scrollIntoViewRects.length) return;
|
||||||
|
|
||||||
|
// Scroll the first highlighted rect into view if needed
|
||||||
|
const containerRect = container.getBoundingClientRect();
|
||||||
|
const visibleTop = container.scrollTop;
|
||||||
|
const visibleBottom = visibleTop + containerRect.height;
|
||||||
|
|
||||||
|
const firstRect = scrollIntoViewRects[0];
|
||||||
|
const elementTop =
|
||||||
|
firstRect.top - containerRect.top + container.scrollTop;
|
||||||
|
|
||||||
|
if (elementTop < visibleTop || elementTop > visibleBottom) {
|
||||||
|
container.scrollTo({
|
||||||
|
top: elementTop - containerRect.height / 3,
|
||||||
|
behavior: 'smooth',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const tokenTexts = tokens.map((t) => t.text);
|
||||||
|
|
||||||
|
// Fire-and-forget async worker call; UI thread returns immediately
|
||||||
|
runHighlightTokenMatch(cleanPattern, tokenTexts)
|
||||||
|
.then((result) => {
|
||||||
|
if (!result || result.bestStart === -1) {
|
||||||
|
// No worker result or no good match; rely on span-level fallback
|
||||||
|
applyHighlightFromTokens(null);
|
||||||
|
} else {
|
||||||
|
applyHighlightFromTokens({
|
||||||
|
bestStart: result.bestStart,
|
||||||
|
bestEnd: result.bestEnd,
|
||||||
|
rating: result.rating,
|
||||||
|
lengthDiff: result.lengthDiff,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.error(
|
||||||
|
'Error in PDF highlight worker, falling back to span-based matching:',
|
||||||
|
error
|
||||||
|
);
|
||||||
|
applyHighlightFromTokens(null);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Text Click Handler
|
// Text Click Handler
|
||||||
|
|
@ -313,7 +603,8 @@ export function handleTextClick(
|
||||||
pdfText: string,
|
pdfText: string,
|
||||||
containerRef: React.RefObject<HTMLDivElement>,
|
containerRef: React.RefObject<HTMLDivElement>,
|
||||||
stopAndPlayFromIndex: (index: number) => void,
|
stopAndPlayFromIndex: (index: number) => void,
|
||||||
isProcessing: boolean
|
isProcessing: boolean,
|
||||||
|
enableHighlight = true
|
||||||
) {
|
) {
|
||||||
if (isProcessing) return;
|
if (isProcessing) return;
|
||||||
|
|
||||||
|
|
@ -337,6 +628,26 @@ export function handleTextClick(
|
||||||
if (!contextText?.trim()) return;
|
if (!contextText?.trim()) return;
|
||||||
|
|
||||||
const cleanContext = contextText.trim().replace(/\s+/g, ' ');
|
const cleanContext = contextText.trim().replace(/\s+/g, ' ');
|
||||||
|
|
||||||
|
// Fast path when highlight overlays are disabled:
|
||||||
|
// avoid expensive span-level fuzzy matching and just map
|
||||||
|
// the clicked context to a sentence using cheap string checks.
|
||||||
|
if (!enableHighlight) {
|
||||||
|
const sentences = processTextToSentences(pdfText);
|
||||||
|
const idx = sentences.findIndex((sentence) => {
|
||||||
|
const cleanSentence = sentence.trim().replace(/\s+/g, ' ');
|
||||||
|
return (
|
||||||
|
cleanSentence.includes(cleanContext) ||
|
||||||
|
cleanContext.includes(cleanSentence)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (idx !== -1) {
|
||||||
|
stopAndPlayFromIndex(idx);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const allText = Array.from(parentElement.querySelectorAll('span')).map((node) => ({
|
const allText = Array.from(parentElement.querySelectorAll('span')).map((node) => ({
|
||||||
element: node as HTMLElement,
|
element: node as HTMLElement,
|
||||||
text: (node.textContent || '').trim(),
|
text: (node.textContent || '').trim(),
|
||||||
|
|
@ -363,7 +674,9 @@ export function handleTextClick(
|
||||||
const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence);
|
const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence);
|
||||||
if (sentenceIndex !== -1) {
|
if (sentenceIndex !== -1) {
|
||||||
stopAndPlayFromIndex(sentenceIndex);
|
stopAndPlayFromIndex(sentenceIndex);
|
||||||
highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef);
|
if (enableHighlight) {
|
||||||
|
highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
135
src/utils/pdfHighlightWorker.ts
Normal file
135
src/utils/pdfHighlightWorker.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
/// <reference lib="webworker" />
|
||||||
|
|
||||||
|
import { CmpStr } from 'cmpstr';
|
||||||
|
|
||||||
|
const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
|
||||||
|
|
||||||
|
interface TokenMatchRequest {
|
||||||
|
id: string;
|
||||||
|
type: 'tokenMatch';
|
||||||
|
pattern: string;
|
||||||
|
tokenTexts: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TokenMatchResponse {
|
||||||
|
id: string;
|
||||||
|
type: 'tokenMatchResult';
|
||||||
|
bestStart: number;
|
||||||
|
bestEnd: number;
|
||||||
|
rating: number;
|
||||||
|
lengthDiff: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.onmessage = (event: MessageEvent<TokenMatchRequest>) => {
|
||||||
|
const data = event.data;
|
||||||
|
if (!data || data.type !== 'tokenMatch') return;
|
||||||
|
|
||||||
|
const { id, pattern, tokenTexts } = data;
|
||||||
|
|
||||||
|
const cleanPattern = pattern.trim().replace(/\s+/g, ' ');
|
||||||
|
const patternLen = cleanPattern.length;
|
||||||
|
|
||||||
|
const responseBase: TokenMatchResponse = {
|
||||||
|
id,
|
||||||
|
type: 'tokenMatchResult',
|
||||||
|
bestStart: -1,
|
||||||
|
bestEnd: -1,
|
||||||
|
rating: 0,
|
||||||
|
lengthDiff: Number.POSITIVE_INFINITY,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!patternLen || !tokenTexts.length) {
|
||||||
|
(self as unknown as DedicatedWorkerGlobalScope).postMessage(responseBase);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const patternTokens = cleanPattern.split(' ').filter(Boolean);
|
||||||
|
const patternTokenCount = patternTokens.length || 1;
|
||||||
|
|
||||||
|
const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6));
|
||||||
|
const maxWindowTokens = Math.max(
|
||||||
|
minWindowTokens,
|
||||||
|
Math.ceil(patternTokenCount * 1.4)
|
||||||
|
);
|
||||||
|
|
||||||
|
let bestStart = -1;
|
||||||
|
let bestEnd = -1;
|
||||||
|
let bestRating = 0;
|
||||||
|
let bestLengthDiff = Number.POSITIVE_INFINITY;
|
||||||
|
|
||||||
|
for (let start = 0; start < tokenTexts.length; start++) {
|
||||||
|
let combined = '';
|
||||||
|
|
||||||
|
for (
|
||||||
|
let offset = 0;
|
||||||
|
offset < maxWindowTokens && start + offset < tokenTexts.length;
|
||||||
|
offset++
|
||||||
|
) {
|
||||||
|
const token = tokenTexts[start + offset];
|
||||||
|
combined = combined ? `${combined} ${token}` : token;
|
||||||
|
|
||||||
|
const windowSize = offset + 1;
|
||||||
|
if (windowSize < minWindowTokens) continue;
|
||||||
|
if (combined.length > patternLen * 2) break;
|
||||||
|
|
||||||
|
const similarity = cmp.compare(combined, cleanPattern);
|
||||||
|
const lengthDiff = Math.abs(combined.length - patternLen);
|
||||||
|
const lengthPenalty = lengthDiff / patternLen;
|
||||||
|
const adjustedRating = similarity * (1 - lengthPenalty * 0.3);
|
||||||
|
|
||||||
|
// Prefix-alignment boost:
|
||||||
|
// Favour windows whose first few tokens closely match the beginning
|
||||||
|
// of the pattern, so we are less likely to cut off the first 1–2 words.
|
||||||
|
let boostedRating = adjustedRating;
|
||||||
|
if (patternTokens.length > 0) {
|
||||||
|
const windowTokens = tokenTexts.slice(start, start + windowSize);
|
||||||
|
const maxPrefixCheck = Math.min(
|
||||||
|
windowTokens.length,
|
||||||
|
patternTokens.length,
|
||||||
|
5
|
||||||
|
);
|
||||||
|
|
||||||
|
let prefixMatches = 0;
|
||||||
|
for (let i = 0; i < maxPrefixCheck; i++) {
|
||||||
|
const tokenText = windowTokens[i];
|
||||||
|
const patternToken = patternTokens[i];
|
||||||
|
// Require reasonably strong similarity for a prefix match
|
||||||
|
const tokenSim = cmp.compare(tokenText, patternToken);
|
||||||
|
if (tokenSim >= 0.8) {
|
||||||
|
prefixMatches++;
|
||||||
|
} else {
|
||||||
|
// Stop at the first non-matching leading token
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (prefixMatches > 0) {
|
||||||
|
const prefixRatio = prefixMatches / maxPrefixCheck;
|
||||||
|
const PREFIX_BOOST_FACTOR = 0.25; // up to +25% boost
|
||||||
|
boostedRating = adjustedRating * (1 + prefixRatio * PREFIX_BOOST_FACTOR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
boostedRating > bestRating ||
|
||||||
|
(Math.abs(boostedRating - bestRating) < 1e-3 &&
|
||||||
|
lengthDiff < bestLengthDiff)
|
||||||
|
) {
|
||||||
|
bestRating = boostedRating;
|
||||||
|
bestLengthDiff = lengthDiff;
|
||||||
|
bestStart = start;
|
||||||
|
bestEnd = start + offset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const response: TokenMatchResponse = {
|
||||||
|
...responseBase,
|
||||||
|
bestStart,
|
||||||
|
bestEnd,
|
||||||
|
rating: bestRating,
|
||||||
|
lengthDiff: bestLengthDiff,
|
||||||
|
};
|
||||||
|
|
||||||
|
(self as unknown as DedicatedWorkerGlobalScope).postMessage(response);
|
||||||
|
};
|
||||||
Loading…
Reference in a new issue