257 lines
10 KiB
TypeScript
257 lines
10 KiB
TypeScript
import { describe, expect, test } from 'vitest';
|
|
|
|
import { buildSegmentKey, buildSegmentKeyPrefix, planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan';
|
|
|
|
describe('planCanonicalTtsSegments', () => {
|
|
test('emits a cross-boundary segment once and assigns it to the source where it starts', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'page:1',
|
|
locator: { page: 1, readerType: 'pdf' },
|
|
text: 'The boundary sentence begins on page one and',
|
|
},
|
|
{
|
|
sourceKey: 'page:2',
|
|
locator: { page: 2, readerType: 'pdf' },
|
|
text: 'finishes on page two with enough words to stand alone. A short follow up.',
|
|
},
|
|
], {
|
|
readerType: 'pdf',
|
|
maxBlockLength: 60,
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
|
|
expect(plan.segments.length).toBeGreaterThanOrEqual(2);
|
|
expect(plan.segments[0]).toMatchObject({
|
|
ownerSourceKey: 'page:1',
|
|
ownerLocator: { page: 1, readerType: 'pdf' },
|
|
spansSourceBoundary: true,
|
|
});
|
|
expect(plan.segments[0].text).toContain('begins on page one and finishes on page two');
|
|
expect(plan.segments[1]).toMatchObject({
|
|
ownerSourceKey: 'page:2',
|
|
spansSourceBoundary: false,
|
|
});
|
|
expect(plan.segments[1].text).toContain('A short follow up.');
|
|
});
|
|
|
|
test('returns the same segment keys for the same canonical source anchors', () => {
|
|
const options = {
|
|
readerType: 'epub' as const,
|
|
maxBlockLength: 70,
|
|
keyPrefix: 'book:v1',
|
|
};
|
|
|
|
const firstPlan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'chapter:1',
|
|
text: 'First sentence with enough words to be its own block. Second sentence with enough words to be its own block.',
|
|
},
|
|
], options);
|
|
|
|
const secondPlan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'chapter:1',
|
|
text: 'First sentence with enough words to be its own block. Second sentence with enough words to be its own block.',
|
|
},
|
|
], options);
|
|
|
|
expect(secondPlan.segments.map((segment) => segment.text)).toEqual(
|
|
firstPlan.segments.map((segment) => segment.text),
|
|
);
|
|
expect(secondPlan.segments.map((segment) => segment.key)).toEqual(
|
|
firstPlan.segments.map((segment) => segment.key),
|
|
);
|
|
});
|
|
|
|
test('repeated identical text produces the same content-addressed key', () => {
|
|
const repeated = 'Repeat this exact sentence with enough filler words here.';
|
|
const plan = planCanonicalTtsSegments([
|
|
{ sourceKey: 'chapter:1', text: `${repeated} ${repeated}` },
|
|
], {
|
|
readerType: 'epub',
|
|
maxBlockLength: 50,
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
expect(plan.segments).toHaveLength(2);
|
|
expect(plan.segments[0].text).toBe(repeated);
|
|
expect(plan.segments[1].text).toBe(repeated);
|
|
// Content-addressed keys: same text → same key (viewport-independent).
|
|
// Segments remain distinct via ordinal and anchors for client-side use.
|
|
expect(plan.segments[0].key).toBe(plan.segments[1].key);
|
|
expect(plan.segments[0].ordinal).not.toBe(plan.segments[1].ordinal);
|
|
});
|
|
|
|
test('does not let locator changes alter canonical keys', () => {
|
|
const text = 'Locator changes should not alter this stable segment identity.';
|
|
const a = planCanonicalTtsSegments([
|
|
{ sourceKey: 'page:1', locator: { page: 1, readerType: 'pdf' }, text },
|
|
], {
|
|
readerType: 'pdf',
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
const b = planCanonicalTtsSegments([
|
|
{ sourceKey: 'page:1', locator: { page: 99, readerType: 'pdf' }, text },
|
|
], {
|
|
readerType: 'pdf',
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
|
|
expect(a.segments.map((segment) => segment.key)).toEqual(
|
|
b.segments.map((segment) => segment.key),
|
|
);
|
|
expect(a.segments[0].ownerLocator).toEqual({ page: 1, readerType: 'pdf' });
|
|
expect(b.segments[0].ownerLocator).toEqual({ page: 99, readerType: 'pdf' });
|
|
});
|
|
|
|
test('generates identical segment keys for the same text from different viewports (sourceKeys)', () => {
|
|
const sentence = 'This sentence is rendered on different viewports.';
|
|
|
|
// Viewport A (e.g. narrow device)
|
|
const planA = planCanonicalTtsSegments([
|
|
{ sourceKey: 'epubcfi(/6/10!/4/2)', text: sentence },
|
|
], {
|
|
readerType: 'epub',
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
// Viewport B (e.g. wide device)
|
|
const planB = planCanonicalTtsSegments([
|
|
{ sourceKey: 'epubcfi(/6/12!/4/4)', text: sentence },
|
|
], {
|
|
readerType: 'epub',
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
// Content-addressed keys should match perfectly because the text is the same
|
|
expect(planA.segments[0].key).toBe(planB.segments[0].key);
|
|
|
|
// But the client-side anchoring accurately reflects the different viewport source
|
|
expect(planA.segments[0].ownerSourceKey).toBe('epubcfi(/6/10!/4/2)');
|
|
expect(planB.segments[0].ownerSourceKey).toBe('epubcfi(/6/12!/4/4)');
|
|
expect(planA.segments[0].startAnchor.sourceKey).not.toBe(planB.segments[0].startAnchor.sourceKey);
|
|
});
|
|
|
|
test('anchors segment offsets back to normalized source text', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{ sourceKey: 'page:1', text: 'First sentence. ' },
|
|
{ sourceKey: 'page:2', text: ' Second sentence with enough words to be separate.' },
|
|
], {
|
|
readerType: 'epub',
|
|
maxBlockLength: 50,
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
expect(plan.segments).toHaveLength(2);
|
|
expect(plan.segments[0].startAnchor).toEqual({ sourceKey: 'page:1', offset: 0 });
|
|
expect(plan.segments[0].endAnchor.sourceKey).toBe('page:1');
|
|
expect(plan.segments[1].startAnchor).toEqual({ sourceKey: 'page:2', offset: 0 });
|
|
expect(plan.text).toBe('First sentence. Second sentence with enough words to be separate.');
|
|
});
|
|
|
|
test('ignores empty source units', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{ sourceKey: 'empty:1', text: ' ' },
|
|
{ sourceKey: 'page:1', text: 'Only useful text remains.' },
|
|
{ sourceKey: 'empty:2', text: '\n\n' },
|
|
], {
|
|
readerType: 'pdf',
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
|
|
expect(plan.text).toBe('Only useful text remains.');
|
|
expect(plan.segments).toHaveLength(1);
|
|
expect(plan.segments[0].ownerSourceKey).toBe('page:1');
|
|
});
|
|
|
|
test('keeps paragraph-title boundaries when source boundaries are enforced', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'abstract',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 'a1' },
|
|
text: 'Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions.',
|
|
},
|
|
{
|
|
sourceKey: 'intro-title',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 't1' },
|
|
text: '1 INTRODUCTION',
|
|
},
|
|
{
|
|
sourceKey: 'intro-body',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 'p1' },
|
|
text: 'Powered by large language models (LLMs; OpenAI 2024b; Team et al. 2023), user-facing AI systems have become increasingly capable of performing complex tasks such as accurately responding to user queries, solving math problems, and generating code.',
|
|
},
|
|
], {
|
|
readerType: 'pdf',
|
|
maxBlockLength: 450,
|
|
keyPrefix: 'doc:v1',
|
|
enforceSourceBoundaries: true,
|
|
});
|
|
|
|
expect(plan.segments.some((segment) => segment.ownerSourceKey === 'intro-title' && segment.text === '1 INTRODUCTION')).toBeTruthy();
|
|
expect(plan.segments.some((segment) => segment.ownerSourceKey === 'intro-body' && segment.text.startsWith('Powered by large language models'))).toBeTruthy();
|
|
expect(plan.segments.some((segment) => segment.text.startsWith('1 INTRODUCTION Powered by'))).toBeFalsy();
|
|
});
|
|
|
|
test('does not drop first sentence when canonical rematch fails in enforced boundary mode', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'title',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 't1' },
|
|
text: '1 INTRODUCTION',
|
|
},
|
|
{
|
|
sourceKey: 'intro',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 'p1' },
|
|
// Missing whitespace after sentence terminal is a common PDF extraction artifact.
|
|
text: 'Powered by large language models have become increasingly capable of generating code.In particular, AI agents have recently received ever-increasing research focus.',
|
|
},
|
|
], {
|
|
readerType: 'pdf',
|
|
maxBlockLength: 450,
|
|
keyPrefix: 'doc:v1',
|
|
enforceSourceBoundaries: true,
|
|
});
|
|
|
|
const introSegments = plan.segments
|
|
.filter((segment) => segment.ownerSourceKey === 'intro')
|
|
.map((segment) => segment.text);
|
|
const combinedIntro = introSegments.join(' ');
|
|
expect(combinedIntro.includes('Powered by large language models')).toBeTruthy();
|
|
expect(combinedIntro.includes('In particular, AI agents')).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => {
|
|
// These keys are the bridge between persistence and the sidebar's merge of
|
|
// synth rows with manifest rows. Both sides must produce identical keys for
|
|
// the same `(documentId, readerType, text)` triple, or the sidebar will
|
|
// show duplicates.
|
|
|
|
test('prefix shape is `${documentId}:${readerType}:v1` (or "document" when documentId is falsy)', () => {
|
|
expect(buildSegmentKeyPrefix('abc123', 'epub')).toBe('abc123:epub:v1');
|
|
expect(buildSegmentKeyPrefix(null, 'pdf')).toBe('document:pdf:v1');
|
|
expect(buildSegmentKeyPrefix('', 'epub')).toBe('document:epub:v1');
|
|
});
|
|
|
|
test('same prefix + same text → same key (sidebar can match synth to manifest)', () => {
|
|
const prefix = buildSegmentKeyPrefix('doc-1', 'epub');
|
|
const k1 = buildSegmentKey(prefix, 'Hello world.');
|
|
const k2 = buildSegmentKey(prefix, 'Hello world.');
|
|
expect(k1).toBe(k2);
|
|
});
|
|
|
|
test('normalization makes whitespace + casing differences identity-equivalent', () => {
|
|
const prefix = buildSegmentKeyPrefix('doc-1', 'epub');
|
|
const a = buildSegmentKey(prefix, ' Hello world. ');
|
|
const b = buildSegmentKey(prefix, 'hello world.');
|
|
expect(a).toBe(b);
|
|
});
|
|
|
|
test('different readerType yields different keys for the same text', () => {
|
|
const a = buildSegmentKey(buildSegmentKeyPrefix('doc-1', 'epub'), 'Same text.');
|
|
const b = buildSegmentKey(buildSegmentKeyPrefix('doc-1', 'pdf'), 'Same text.');
|
|
expect(a).not.toBe(b);
|
|
});
|
|
});
|