6e69b48103
- Remove legacy inference and platform files from `src/inference/` and migrate responsibilities to new modular locations under `src/infrastructure/`, `src/inference/pdf/`, and `src/inference/whisper/` - Consolidate environment/config logic into `src/infrastructure/config.ts` - Move docstore and ffmpeg platform utilities to `src/infrastructure/platform.ts` - Refactor PDF and Whisper inference code to use new document layout, layout model, and timestamp utilities - Update API contracts to define and export types and constants previously scattered in inference/types - Adjust all imports in jobs, storage, and tests to reference new module structure and type locations - Inline parser version and encoding logic into API contracts - Remove obsolete files and update test fixtures for new type locations This update improves codebase modularity, maintainability, and separation of concerns. No changes to inference or orchestration functionality. BREAKING CHANGE: inference module structure, type imports, and config utilities have changed; downstream code must update imports and integration points.
94 lines
2.8 KiB
TypeScript
94 lines
2.8 KiB
TypeScript
import { describe, expect, test } from 'vitest';
|
|
|
|
import { normalizeTextItemsForLayout } from '../../../src/inference/pdf/normalize-text';
|
|
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
|
|
|
function makeTextItem(
|
|
str: string,
|
|
transform: [number, number, number, number, number, number],
|
|
width = 100,
|
|
): TextItem {
|
|
return {
|
|
str,
|
|
transform,
|
|
width,
|
|
height: Math.abs(transform[3]),
|
|
dir: 'ltr',
|
|
fontName: 'test',
|
|
hasEOL: false,
|
|
} as unknown as TextItem;
|
|
}
|
|
|
|
describe('normalizeTextItemsForLayout', () => {
|
|
test('keeps horizontal body text and drops rotated/skewed margin text', () => {
|
|
const horizontal = makeTextItem(
|
|
'Powered by large language models',
|
|
[10, 0, 0, 10, 100, 600],
|
|
);
|
|
|
|
// Typical 90deg-ish rotated/skewed run (like side metadata labels).
|
|
const rotated = makeTextItem(
|
|
'arXiv:2407.16741v3 [cs.SE] 18 Apr 2025',
|
|
[0, 10, -10, 0, 30, 400],
|
|
);
|
|
|
|
const normalized = normalizeTextItemsForLayout([horizontal, rotated], {
|
|
height: 800,
|
|
transform: [1, 0, 0, -1, 0, 800],
|
|
}, {
|
|
test: { ascent: 0.8, descent: -0.2 },
|
|
});
|
|
expect(normalized).toHaveLength(1);
|
|
expect(normalized[0]?.text).toBe('Powered by large language models');
|
|
});
|
|
|
|
test('drops malformed/vertical-only runs so downstream layout planning sees no body text', () => {
|
|
const vertical = makeTextItem('Side label', [0, 10, -10, 0, 30, 200]);
|
|
const skewed = makeTextItem('Watermark', [10, 5, 2, 10, 200, 500]);
|
|
|
|
const normalized = normalizeTextItemsForLayout([vertical, skewed], {
|
|
height: 800,
|
|
transform: [1, 0, 0, -1, 0, 800],
|
|
}, {
|
|
test: { ascent: 0.8, descent: -0.2 },
|
|
});
|
|
expect(normalized).toEqual([]);
|
|
});
|
|
|
|
test('accounts for non-zero page origins in the viewport transform', () => {
|
|
const croppedPageLine = makeTextItem(
|
|
'Vasher turned away.',
|
|
[11.2, 0, 0, 11.2, 127.5, 644.4128],
|
|
100,
|
|
);
|
|
|
|
const normalized = normalizeTextItemsForLayout([croppedPageLine], {
|
|
height: 666.0074,
|
|
transform: [1, 0, 0, -1, -53.4352, 720.565],
|
|
}, {
|
|
test: { ascent: 0.716, descent: -0.269 },
|
|
});
|
|
|
|
expect(normalized).toHaveLength(1);
|
|
expect(normalized[0]?.x).toBeCloseTo(74.0648, 4);
|
|
expect(normalized[0]?.y).toBeCloseTo(68.1356, 2);
|
|
});
|
|
|
|
test('uses font ascent to place decorative initials closer to the visible glyph top', () => {
|
|
const dropCap = makeTextItem(
|
|
'I',
|
|
[60, 0, 0, 60, 111.1326, 434.46],
|
|
9.18,
|
|
);
|
|
|
|
const normalized = normalizeTextItemsForLayout([dropCap], {
|
|
height: 666.0074,
|
|
transform: [1, 0, 0, -1, -53.4352, 720.565],
|
|
}, {
|
|
test: { ascent: 0.638, descent: -0.134 },
|
|
});
|
|
|
|
expect(normalized).toHaveLength(1);
|
|
expect(normalized[0]?.y).toBeCloseTo(247.825, 3);
|
|
});
|
|
});
|