This change removes legacy server-side PDF parse state management and transitions
to a fully worker-owned model for PDF parsing operations. Key updates include:
- Deletes all code related to server-managed parse state, including:
- parse-state.ts, parse-state-backfill.ts, parse-state-healing.ts,
parsed-pdf-reuse.ts, pdf-parse-operation.ts, and related job logic
- Removes the user-pdf-layout-job queue and associated job logic
- Refactors API routes for parsed PDF documents and events to use the new
worker-owned PDF parse operation flow under src/lib/server/pdf-parse/
- Updates S3 parsed PDF artifact keying to include parser version for
deduplication and compatibility
- Refactors client API and hooks to handle new error and progress reporting
- Removes all tests for the legacy parse state and job system, updating remaining
tests to mock the new worker-owned flow
BREAKING CHANGE: Server no longer manages per-user PDF parse state or jobs.
All PDF parsing is now managed by the compute worker and new artifact keying.
Legacy parse state and jobs are no longer supported. Existing parsed PDFs may
need to be reprocessed for compatibility with the new model.
177 lines
5.8 KiB
TypeScript
177 lines
5.8 KiB
TypeScript
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
|
import { NextRequest } from 'next/server';
|
|
|
|
const hoisted = vi.hoisted(() => ({
|
|
db: null as {
|
|
select: ReturnType<typeof vi.fn>;
|
|
} | null,
|
|
row: {
|
|
id: 'doc-1',
|
|
type: 'pdf',
|
|
},
|
|
requireAuthContext: vi.fn(),
|
|
readCurrentParsedPdfArtifact: vi.fn(),
|
|
readParsedPdfArtifactByKey: vi.fn(),
|
|
lookupCurrentPdfParseOperation: vi.fn(),
|
|
createOrReuseCurrentPdfParseOperation: vi.fn(),
|
|
checkJobRate: vi.fn(),
|
|
getPdfLayoutRateConfig: vi.fn(),
|
|
getResolvedRuntimeConfig: vi.fn(),
|
|
buildComputeRateLimitedResponse: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('@/db', () => ({
|
|
get db() {
|
|
return hoisted.db;
|
|
},
|
|
}));
|
|
|
|
vi.mock('@/lib/server/auth/auth', () => ({
|
|
requireAuthContext: hoisted.requireAuthContext,
|
|
}));
|
|
|
|
vi.mock('@/lib/server/pdf-parse/artifact', () => ({
|
|
readCurrentParsedPdfArtifact: hoisted.readCurrentParsedPdfArtifact,
|
|
readParsedPdfArtifactByKey: hoisted.readParsedPdfArtifactByKey,
|
|
}));
|
|
|
|
vi.mock('@/lib/server/pdf-parse/operation', () => ({
|
|
lookupCurrentPdfParseOperation: hoisted.lookupCurrentPdfParseOperation,
|
|
createOrReuseCurrentPdfParseOperation: hoisted.createOrReuseCurrentPdfParseOperation,
|
|
}));
|
|
|
|
vi.mock('@/lib/server/rate-limit/job-rate-limiter', () => ({
|
|
checkJobRate: hoisted.checkJobRate,
|
|
getPdfLayoutRateConfig: hoisted.getPdfLayoutRateConfig,
|
|
}));
|
|
|
|
vi.mock('@/lib/server/rate-limit/problem-response', () => ({
|
|
buildComputeRateLimitedResponse: hoisted.buildComputeRateLimitedResponse,
|
|
}));
|
|
|
|
vi.mock('@/lib/server/runtime-config', () => ({
|
|
getResolvedRuntimeConfig: hoisted.getResolvedRuntimeConfig,
|
|
}));
|
|
|
|
vi.mock('@/lib/server/documents/blobstore', () => ({
|
|
isValidDocumentId: vi.fn(() => true),
|
|
}));
|
|
|
|
vi.mock('@/lib/server/storage/s3', () => ({
|
|
isS3Configured: vi.fn(() => true),
|
|
}));
|
|
|
|
vi.mock('@/lib/server/testing/test-namespace', () => ({
|
|
getOpenReaderTestNamespace: vi.fn(() => null),
|
|
}));
|
|
|
|
vi.mock('@/lib/server/logger', () => ({
|
|
createRequestLogger: vi.fn(() => ({
|
|
logger: {
|
|
warn: vi.fn(),
|
|
info: vi.fn(),
|
|
error: vi.fn(),
|
|
},
|
|
requestId: 'req-test',
|
|
})),
|
|
}));
|
|
|
|
describe('GET/POST /api/documents/[id]/parsed worker-owned flow', () => {
|
|
beforeEach(() => {
|
|
hoisted.db = {
|
|
select: vi.fn(() => ({
|
|
from: vi.fn(() => ({
|
|
where: vi.fn(() => ({
|
|
limit: vi.fn(async () => [{ ...hoisted.row }]),
|
|
})),
|
|
})),
|
|
})),
|
|
};
|
|
hoisted.requireAuthContext.mockReset();
|
|
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
|
hoisted.readCurrentParsedPdfArtifact.mockReset();
|
|
hoisted.readCurrentParsedPdfArtifact.mockResolvedValue(null);
|
|
hoisted.readParsedPdfArtifactByKey.mockReset();
|
|
hoisted.readParsedPdfArtifactByKey.mockResolvedValue(null);
|
|
hoisted.lookupCurrentPdfParseOperation.mockReset();
|
|
hoisted.lookupCurrentPdfParseOperation.mockResolvedValue(null);
|
|
hoisted.createOrReuseCurrentPdfParseOperation.mockReset();
|
|
hoisted.checkJobRate.mockReset();
|
|
hoisted.checkJobRate.mockResolvedValue({ allowed: true });
|
|
hoisted.getPdfLayoutRateConfig.mockReset();
|
|
hoisted.getPdfLayoutRateConfig.mockReturnValue({});
|
|
hoisted.getResolvedRuntimeConfig.mockReset();
|
|
hoisted.getResolvedRuntimeConfig.mockResolvedValue({});
|
|
hoisted.buildComputeRateLimitedResponse.mockReset();
|
|
});
|
|
|
|
test('GET returns pending when no current artifact or worker op exists', async () => {
|
|
const { GET } = await import('../../src/app/api/documents/[id]/parsed/route');
|
|
const response = await GET(new NextRequest('http://localhost/api/documents/doc-1/parsed'), {
|
|
params: Promise.resolve({ id: 'doc-1' }),
|
|
});
|
|
|
|
expect(response.status).toBe(409);
|
|
await expect(response.json()).resolves.toMatchObject({
|
|
parseStatus: 'pending',
|
|
parseProgress: null,
|
|
opId: null,
|
|
});
|
|
});
|
|
|
|
test('GET reads the artifact referenced by a succeeded worker op', async () => {
|
|
hoisted.lookupCurrentPdfParseOperation.mockResolvedValue({
|
|
opId: 'op-1',
|
|
opKey: 'pdf_layout|v1|parser|doc-1||doc-key|',
|
|
kind: 'pdf_layout',
|
|
jobId: 'job-1',
|
|
status: 'succeeded',
|
|
queuedAt: Date.now() - 1000,
|
|
updatedAt: Date.now(),
|
|
result: { parsedObjectKey: 'parsed-key.json' },
|
|
});
|
|
hoisted.readParsedPdfArtifactByKey.mockResolvedValue({
|
|
key: 'parsed-key.json',
|
|
bytes: Buffer.from(JSON.stringify({ documentId: 'doc-1', pages: [] })),
|
|
parsed: { documentId: 'doc-1', pages: [] },
|
|
});
|
|
|
|
const { GET } = await import('../../src/app/api/documents/[id]/parsed/route');
|
|
const response = await GET(new NextRequest('http://localhost/api/documents/doc-1/parsed'), {
|
|
params: Promise.resolve({ id: 'doc-1' }),
|
|
});
|
|
|
|
expect(response.status).toBe(200);
|
|
await expect(response.json()).resolves.toMatchObject({ documentId: 'doc-1' });
|
|
expect(hoisted.readParsedPdfArtifactByKey).toHaveBeenCalledWith('parsed-key.json');
|
|
});
|
|
|
|
test('POST creates a worker op when replace is requested', async () => {
|
|
hoisted.createOrReuseCurrentPdfParseOperation.mockResolvedValue({
|
|
opId: 'op-force-1',
|
|
opKey: 'pdf_layout|v1|parser|doc-1||doc-key|force',
|
|
kind: 'pdf_layout',
|
|
jobId: 'job-force-1',
|
|
status: 'queued',
|
|
queuedAt: Date.now(),
|
|
updatedAt: Date.now(),
|
|
});
|
|
|
|
const { POST } = await import('../../src/app/api/documents/[id]/parsed/route');
|
|
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed', {
|
|
method: 'POST',
|
|
body: JSON.stringify({ replace: true }),
|
|
headers: { 'Content-Type': 'application/json' },
|
|
});
|
|
const response = await POST(request, {
|
|
params: Promise.resolve({ id: 'doc-1' }),
|
|
});
|
|
|
|
expect(response.status).toBe(202);
|
|
await expect(response.json()).resolves.toMatchObject({
|
|
parseStatus: 'pending',
|
|
opId: 'op-force-1',
|
|
});
|
|
expect(hoisted.createOrReuseCurrentPdfParseOperation).toHaveBeenCalled();
|
|
});
|
|
});
|