3d5dfd2a88
- Remove legacy compute-worker monolith under `src/compute/`, including PDF and Whisper inference, control-plane, and runtime orchestration - Move PDF and Whisper inference logic to new `src/inference/` module - Move control-plane, orchestrator, and state machine logic to `src/operations/` - Move NATS/JetStream adapters to `src/infrastructure/` - Move job orchestration, progress, and artifact persistence to `src/jobs/` - Update imports throughout tests and main app to reference new module structure - Update Dockerfile and scripts to use new asset paths - Add new entrypoints: `src/api/app.ts` and `src/api/contracts.ts` - Remove obsolete files and update `.gitignore` for new dev artifacts This refactor modularizes the compute-worker, improving maintainability and separation of concerns. No functional changes to inference or orchestration logic. BREAKING CHANGE: compute-worker internal APIs, directory structure, and imports have changed; downstream code must update imports and integration points.
106 lines
3.3 KiB
TypeScript
106 lines
3.3 KiB
TypeScript
import { describe, expect, test } from 'vitest';
|
|
import { stitchCrossPageBlocks } from '../../../src/inference';
|
|
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/inference/types';
|
|
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
|
|
|
function makeBlock(
|
|
id: string,
|
|
kind: ParsedPdfBlockKind,
|
|
text: string,
|
|
page: number,
|
|
readingOrder: number,
|
|
): ParsedPdfBlock {
|
|
return {
|
|
id,
|
|
kind,
|
|
text,
|
|
fragments: [{
|
|
page,
|
|
bbox: [0, 0, 100, 10],
|
|
text,
|
|
readingOrder,
|
|
}],
|
|
};
|
|
}
|
|
|
|
function makeDoc(page1Blocks: ParsedPdfBlock[], page2Blocks: ParsedPdfBlock[]): ParsedPdfDocument {
|
|
return makeParsedPdfDocument([
|
|
makeParsedPdfPage(1, page1Blocks),
|
|
makeParsedPdfPage(2, page2Blocks),
|
|
]);
|
|
}
|
|
|
|
describe('stitchCrossPageBlocks', () => {
|
|
test('stitches paragraph continuation across footer/header noise', () => {
|
|
const doc = makeDoc(
|
|
[
|
|
makeBlock('b1', 'text', 'This sentence continues', 1, 0),
|
|
makeBlock('b2', 'footer', 'Footer text', 1, 1),
|
|
],
|
|
[
|
|
makeBlock('b3', 'header', 'Header text', 2, 0),
|
|
makeBlock('b4', 'text', 'into the next page.', 2, 1),
|
|
],
|
|
);
|
|
|
|
const stitched = stitchCrossPageBlocks(doc);
|
|
const page1 = stitched.pages[0];
|
|
const page2 = stitched.pages[1];
|
|
|
|
expect(page1?.blocks[0]?.text).toBe('This sentence continues into the next page.');
|
|
expect(page1?.blocks[0]?.fragments).toHaveLength(2);
|
|
expect(page2?.blocks.map((b) => b.id)).toEqual(['b3']);
|
|
});
|
|
|
|
test('moves only the continuation sentence and keeps remaining text on next page', () => {
|
|
const doc = makeDoc(
|
|
[
|
|
makeBlock('b1', 'text', 'This sentence continues', 1, 0),
|
|
],
|
|
[
|
|
makeBlock('b2', 'text', 'into the next page. This should stay on page two.', 2, 0),
|
|
],
|
|
);
|
|
|
|
const stitched = stitchCrossPageBlocks(doc);
|
|
const page1 = stitched.pages[0];
|
|
const page2 = stitched.pages[1];
|
|
|
|
expect(page1?.blocks[0]?.text).toBe('This sentence continues into the next page.');
|
|
expect(page1?.blocks[0]?.fragments).toHaveLength(2);
|
|
expect(page2?.blocks).toHaveLength(1);
|
|
expect(page2?.blocks[0]?.id).toBe('b2');
|
|
expect(page2?.blocks[0]?.text).toBe('This should stay on page two.');
|
|
});
|
|
|
|
test('does not stitch across paragraph-title boundary', () => {
|
|
const doc = makeDoc(
|
|
[
|
|
makeBlock('b1', 'text', 'This sentence continues', 1, 0),
|
|
],
|
|
[
|
|
makeBlock('b2', 'paragraph_title', '2 New Section', 2, 0),
|
|
makeBlock('b3', 'text', 'into the next page.', 2, 1),
|
|
],
|
|
);
|
|
|
|
const stitched = stitchCrossPageBlocks(doc);
|
|
expect(stitched.pages[0]?.blocks[0]?.fragments).toHaveLength(1);
|
|
expect(stitched.pages[1]?.blocks.map((b) => b.id)).toEqual(['b2', 'b3']);
|
|
});
|
|
|
|
test('does not stitch when tail has sentence terminal', () => {
|
|
const doc = makeDoc(
|
|
[
|
|
makeBlock('b1', 'text', 'This sentence is complete.', 1, 0),
|
|
],
|
|
[
|
|
makeBlock('b2', 'text', 'next sentence starts here', 2, 0),
|
|
],
|
|
);
|
|
|
|
const stitched = stitchCrossPageBlocks(doc);
|
|
expect(stitched.pages[0]?.blocks[0]?.fragments).toHaveLength(1);
|
|
expect(stitched.pages[1]?.blocks.map((b) => b.id)).toEqual(['b2']);
|
|
});
|
|
});
|