perf(highlight): optimize exact token sequence matching for large documents
Add a fast linear scan to resolve exact token sequence highlights before falling back to the fuzzy window search, significantly improving performance for large documents. Update unit tests to verify that exact matches are found efficiently without invoking expensive comparisons.
This commit is contained in:
parent
fed782eb0b
commit
4b226858b1
2 changed files with 38 additions and 0 deletions
|
|
@ -36,6 +36,29 @@ export function findBestHighlightTokenMatch(
|
|||
|
||||
if (!patternLen || !normalizedTargets.length) return responseBase;
|
||||
|
||||
// Most viewer highlights are exact token sequences. Resolve those in one
|
||||
// linear pass before entering the fuzzy window search, which is expensive
|
||||
// enough to stall the UI when the target is a full HTML/TXT/MD document.
|
||||
const firstPatternToken = normalizedPattern[0];
|
||||
for (let start = 0; start < normalizedTargets.length; start += 1) {
|
||||
if (normalizedTargets[start] !== firstPatternToken) continue;
|
||||
let matches = true;
|
||||
for (let offset = 1; offset < normalizedPattern.length; offset += 1) {
|
||||
if (normalizedTargets[start + offset] !== normalizedPattern[offset]) {
|
||||
matches = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matches) {
|
||||
return {
|
||||
start,
|
||||
end: start + normalizedPattern.length - 1,
|
||||
rating: 1,
|
||||
lengthDiff: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const patternTokenCount = normalizedPattern.length;
|
||||
const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6));
|
||||
const maxWindowTokens = Math.max(
|
||||
|
|
|
|||
|
|
@ -19,6 +19,21 @@ const alignmentWords = (
|
|||
}));
|
||||
|
||||
describe('shared viewer highlight token alignment', () => {
|
||||
test('finds an exact sentence in a large document without entering fuzzy comparison', () => {
|
||||
const targets = Array.from({ length: 20_000 }, (_, index) => `token-${index}`);
|
||||
targets.splice(15_000, 4, 'the', 'exact', 'sentence', 'here');
|
||||
|
||||
expect(findBestHighlightTokenMatch(
|
||||
['the', 'exact', 'sentence', 'here'],
|
||||
targets,
|
||||
)).toEqual({
|
||||
start: 15_000,
|
||||
end: 15_003,
|
||||
rating: 1,
|
||||
lengthDiff: 0,
|
||||
});
|
||||
});
|
||||
|
||||
test('matches a complete Japanese sentence using locale-aware token count', () => {
|
||||
const sentence = 'これは日本語です。';
|
||||
const patternTokens = segmentWords(sentence, 'ja').map((token) => token.text);
|
||||
|
|
|
|||
Loading…
Reference in a new issue