perf(highlight): optimize exact token sequence matching for large documents

Add a fast linear scan to resolve exact token sequence highlights before
falling back to the fuzzy window search, significantly improving
performance for large documents. Update unit tests to verify that
exact matches are found efficiently without invoking expensive
comparisons.
This commit is contained in:
Richard R 2026-06-06 11:41:47 -06:00
parent fed782eb0b
commit 4b226858b1
2 changed files with 38 additions and 0 deletions

View file

@ -36,6 +36,29 @@ export function findBestHighlightTokenMatch(
if (!patternLen || !normalizedTargets.length) return responseBase;
// Most viewer highlights are exact token sequences. Resolve those in one
// linear pass before entering the fuzzy window search, which is expensive
// enough to stall the UI when the target is a full HTML/TXT/MD document.
const firstPatternToken = normalizedPattern[0];
for (let start = 0; start < normalizedTargets.length; start += 1) {
if (normalizedTargets[start] !== firstPatternToken) continue;
let matches = true;
for (let offset = 1; offset < normalizedPattern.length; offset += 1) {
if (normalizedTargets[start + offset] !== normalizedPattern[offset]) {
matches = false;
break;
}
}
if (matches) {
return {
start,
end: start + normalizedPattern.length - 1,
rating: 1,
lengthDiff: 0,
};
}
}
const patternTokenCount = normalizedPattern.length;
const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6));
const maxWindowTokens = Math.max(

View file

@ -19,6 +19,21 @@ const alignmentWords = (
}));
describe('shared viewer highlight token alignment', () => {
test('finds an exact sentence in a large document without entering fuzzy comparison', () => {
const targets = Array.from({ length: 20_000 }, (_, index) => `token-${index}`);
targets.splice(15_000, 4, 'the', 'exact', 'sentence', 'here');
expect(findBestHighlightTokenMatch(
['the', 'exact', 'sentence', 'here'],
targets,
)).toEqual({
start: 15_000,
end: 15_003,
rating: 1,
lengthDiff: 0,
});
});
test('matches a complete Japanese sentence using locale-aware token count', () => {
const sentence = 'これは日本語です。';
const patternTokens = segmentWords(sentence, 'ja').map((token) => token.text);