6ab5c230c2
Enhance PDF cross-page block stitching to move only sentence continuations and preserve remaining text on the following page. Update TTS segment planning to support strict source boundary enforcement, ensuring segments do not cross block or paragraph-title boundaries. Add new tests for both features and refactor PDF text item normalization to filter out skewed or rotated runs.
257 lines
10 KiB
TypeScript
257 lines
10 KiB
TypeScript
import { expect, test } from '@playwright/test';
|
|
|
|
import { buildSegmentKey, buildSegmentKeyPrefix, planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan';
|
|
|
|
test.describe('planCanonicalTtsSegments', () => {
|
|
test('emits a cross-boundary segment once and assigns it to the source where it starts', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'page:1',
|
|
locator: { page: 1, readerType: 'pdf' },
|
|
text: 'The boundary sentence begins on page one and',
|
|
},
|
|
{
|
|
sourceKey: 'page:2',
|
|
locator: { page: 2, readerType: 'pdf' },
|
|
text: 'finishes on page two with enough words to stand alone. A short follow up.',
|
|
},
|
|
], {
|
|
readerType: 'pdf',
|
|
maxBlockLength: 60,
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
|
|
expect(plan.segments.length).toBeGreaterThanOrEqual(2);
|
|
expect(plan.segments[0]).toMatchObject({
|
|
ownerSourceKey: 'page:1',
|
|
ownerLocator: { page: 1, readerType: 'pdf' },
|
|
spansSourceBoundary: true,
|
|
});
|
|
expect(plan.segments[0].text).toContain('begins on page one and finishes on page two');
|
|
expect(plan.segments[1]).toMatchObject({
|
|
ownerSourceKey: 'page:2',
|
|
spansSourceBoundary: false,
|
|
});
|
|
expect(plan.segments[1].text).toContain('A short follow up.');
|
|
});
|
|
|
|
test('returns the same segment keys for the same canonical source anchors', () => {
|
|
const options = {
|
|
readerType: 'epub' as const,
|
|
maxBlockLength: 70,
|
|
keyPrefix: 'book:v1',
|
|
};
|
|
|
|
const firstPlan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'chapter:1',
|
|
text: 'First sentence with enough words to be its own block. Second sentence with enough words to be its own block.',
|
|
},
|
|
], options);
|
|
|
|
const secondPlan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'chapter:1',
|
|
text: 'First sentence with enough words to be its own block. Second sentence with enough words to be its own block.',
|
|
},
|
|
], options);
|
|
|
|
expect(secondPlan.segments.map((segment) => segment.text)).toEqual(
|
|
firstPlan.segments.map((segment) => segment.text),
|
|
);
|
|
expect(secondPlan.segments.map((segment) => segment.key)).toEqual(
|
|
firstPlan.segments.map((segment) => segment.key),
|
|
);
|
|
});
|
|
|
|
test('repeated identical text produces the same content-addressed key', () => {
|
|
const repeated = 'Repeat this exact sentence with enough filler words here.';
|
|
const plan = planCanonicalTtsSegments([
|
|
{ sourceKey: 'chapter:1', text: `${repeated} ${repeated}` },
|
|
], {
|
|
readerType: 'epub',
|
|
maxBlockLength: 50,
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
expect(plan.segments).toHaveLength(2);
|
|
expect(plan.segments[0].text).toBe(repeated);
|
|
expect(plan.segments[1].text).toBe(repeated);
|
|
// Content-addressed keys: same text → same key (viewport-independent).
|
|
// Segments remain distinct via ordinal and anchors for client-side use.
|
|
expect(plan.segments[0].key).toBe(plan.segments[1].key);
|
|
expect(plan.segments[0].ordinal).not.toBe(plan.segments[1].ordinal);
|
|
});
|
|
|
|
test('does not let locator changes alter canonical keys', () => {
|
|
const text = 'Locator changes should not alter this stable segment identity.';
|
|
const a = planCanonicalTtsSegments([
|
|
{ sourceKey: 'page:1', locator: { page: 1, readerType: 'pdf' }, text },
|
|
], {
|
|
readerType: 'pdf',
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
const b = planCanonicalTtsSegments([
|
|
{ sourceKey: 'page:1', locator: { page: 99, readerType: 'pdf' }, text },
|
|
], {
|
|
readerType: 'pdf',
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
|
|
expect(a.segments.map((segment) => segment.key)).toEqual(
|
|
b.segments.map((segment) => segment.key),
|
|
);
|
|
expect(a.segments[0].ownerLocator).toEqual({ page: 1, readerType: 'pdf' });
|
|
expect(b.segments[0].ownerLocator).toEqual({ page: 99, readerType: 'pdf' });
|
|
});
|
|
|
|
test('generates identical segment keys for the same text from different viewports (sourceKeys)', () => {
|
|
const sentence = 'This sentence is rendered on different viewports.';
|
|
|
|
// Viewport A (e.g. narrow device)
|
|
const planA = planCanonicalTtsSegments([
|
|
{ sourceKey: 'epubcfi(/6/10!/4/2)', text: sentence },
|
|
], {
|
|
readerType: 'epub',
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
// Viewport B (e.g. wide device)
|
|
const planB = planCanonicalTtsSegments([
|
|
{ sourceKey: 'epubcfi(/6/12!/4/4)', text: sentence },
|
|
], {
|
|
readerType: 'epub',
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
// Content-addressed keys should match perfectly because the text is the same
|
|
expect(planA.segments[0].key).toBe(planB.segments[0].key);
|
|
|
|
// But the client-side anchoring accurately reflects the different viewport source
|
|
expect(planA.segments[0].ownerSourceKey).toBe('epubcfi(/6/10!/4/2)');
|
|
expect(planB.segments[0].ownerSourceKey).toBe('epubcfi(/6/12!/4/4)');
|
|
expect(planA.segments[0].startAnchor.sourceKey).not.toBe(planB.segments[0].startAnchor.sourceKey);
|
|
});
|
|
|
|
test('anchors segment offsets back to normalized source text', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{ sourceKey: 'page:1', text: 'First sentence. ' },
|
|
{ sourceKey: 'page:2', text: ' Second sentence with enough words to be separate.' },
|
|
], {
|
|
readerType: 'epub',
|
|
maxBlockLength: 50,
|
|
keyPrefix: 'book:v1',
|
|
});
|
|
|
|
expect(plan.segments).toHaveLength(2);
|
|
expect(plan.segments[0].startAnchor).toEqual({ sourceKey: 'page:1', offset: 0 });
|
|
expect(plan.segments[0].endAnchor.sourceKey).toBe('page:1');
|
|
expect(plan.segments[1].startAnchor).toEqual({ sourceKey: 'page:2', offset: 0 });
|
|
expect(plan.text).toBe('First sentence. Second sentence with enough words to be separate.');
|
|
});
|
|
|
|
test('ignores empty source units', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{ sourceKey: 'empty:1', text: ' ' },
|
|
{ sourceKey: 'page:1', text: 'Only useful text remains.' },
|
|
{ sourceKey: 'empty:2', text: '\n\n' },
|
|
], {
|
|
readerType: 'pdf',
|
|
keyPrefix: 'doc:v1',
|
|
});
|
|
|
|
expect(plan.text).toBe('Only useful text remains.');
|
|
expect(plan.segments).toHaveLength(1);
|
|
expect(plan.segments[0].ownerSourceKey).toBe('page:1');
|
|
});
|
|
|
|
test('keeps paragraph-title boundaries when source boundaries are enforced', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'abstract',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 'a1' },
|
|
text: 'Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions.',
|
|
},
|
|
{
|
|
sourceKey: 'intro-title',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 't1' },
|
|
text: '1 INTRODUCTION',
|
|
},
|
|
{
|
|
sourceKey: 'intro-body',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 'p1' },
|
|
text: 'Powered by large language models (LLMs; OpenAI 2024b; Team et al. 2023), user-facing AI systems have become increasingly capable of performing complex tasks such as accurately responding to user queries, solving math problems, and generating code.',
|
|
},
|
|
], {
|
|
readerType: 'pdf',
|
|
maxBlockLength: 450,
|
|
keyPrefix: 'doc:v1',
|
|
enforceSourceBoundaries: true,
|
|
});
|
|
|
|
expect(plan.segments.some((segment) => segment.ownerSourceKey === 'intro-title' && segment.text === '1 INTRODUCTION')).toBeTruthy();
|
|
expect(plan.segments.some((segment) => segment.ownerSourceKey === 'intro-body' && segment.text.startsWith('Powered by large language models'))).toBeTruthy();
|
|
expect(plan.segments.some((segment) => segment.text.startsWith('1 INTRODUCTION Powered by'))).toBeFalsy();
|
|
});
|
|
|
|
test('does not drop first sentence when canonical rematch fails in enforced boundary mode', () => {
|
|
const plan = planCanonicalTtsSegments([
|
|
{
|
|
sourceKey: 'title',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 't1' },
|
|
text: '1 INTRODUCTION',
|
|
},
|
|
{
|
|
sourceKey: 'intro',
|
|
locator: { page: 1, readerType: 'pdf', blockId: 'p1' },
|
|
// Missing whitespace after sentence terminal is a common PDF extraction artifact.
|
|
text: 'Powered by large language models have become increasingly capable of generating code.In particular, AI agents have recently received ever-increasing research focus.',
|
|
},
|
|
], {
|
|
readerType: 'pdf',
|
|
maxBlockLength: 450,
|
|
keyPrefix: 'doc:v1',
|
|
enforceSourceBoundaries: true,
|
|
});
|
|
|
|
const introSegments = plan.segments
|
|
.filter((segment) => segment.ownerSourceKey === 'intro')
|
|
.map((segment) => segment.text);
|
|
const combinedIntro = introSegments.join(' ');
|
|
expect(combinedIntro.includes('Powered by large language models')).toBeTruthy();
|
|
expect(combinedIntro.includes('In particular, AI agents')).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
test.describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => {
|
|
// These keys are the bridge between persistence and the sidebar's merge of
|
|
// synth rows with manifest rows. Both sides must produce identical keys for
|
|
// the same `(documentId, readerType, text)` triple, or the sidebar will
|
|
// show duplicates.
|
|
|
|
test('prefix shape is `${documentId}:${readerType}:v1` (or "document" when documentId is falsy)', () => {
|
|
expect(buildSegmentKeyPrefix('abc123', 'epub')).toBe('abc123:epub:v1');
|
|
expect(buildSegmentKeyPrefix(null, 'pdf')).toBe('document:pdf:v1');
|
|
expect(buildSegmentKeyPrefix('', 'epub')).toBe('document:epub:v1');
|
|
});
|
|
|
|
test('same prefix + same text → same key (sidebar can match synth to manifest)', () => {
|
|
const prefix = buildSegmentKeyPrefix('doc-1', 'epub');
|
|
const k1 = buildSegmentKey(prefix, 'Hello world.');
|
|
const k2 = buildSegmentKey(prefix, 'Hello world.');
|
|
expect(k1).toBe(k2);
|
|
});
|
|
|
|
test('normalization makes whitespace + casing differences identity-equivalent', () => {
|
|
const prefix = buildSegmentKeyPrefix('doc-1', 'epub');
|
|
const a = buildSegmentKey(prefix, ' Hello world. ');
|
|
const b = buildSegmentKey(prefix, 'hello world.');
|
|
expect(a).toBe(b);
|
|
});
|
|
|
|
test('different readerType yields different keys for the same text', () => {
|
|
const a = buildSegmentKey(buildSegmentKeyPrefix('doc-1', 'epub'), 'Same text.');
|
|
const b = buildSegmentKey(buildSegmentKeyPrefix('doc-1', 'pdf'), 'Same text.');
|
|
expect(a).not.toBe(b);
|
|
});
|
|
});
|