feat(pdf): implement granular PDF parse progress tracking and migrate to parseState
Introduce detailed progress reporting for PDF layout parsing, exposing phase and page-level updates to clients. Replace legacy parseStatus with a structured parseState field in the database schema, updating all relevant backend and API logic. Add SSE endpoint for real-time parse progress updates. Update client hooks and UI to reflect granular progress and improve user feedback during document parsing. Includes migration scripts and new parse-state utility module.
This commit is contained in:
parent
a7b955e1ca
commit
4497f610c0
26 changed files with 4226 additions and 99 deletions
|
|
@ -61,6 +61,15 @@ export interface WorkerJobTiming {
|
||||||
computeMs?: number;
|
computeMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type PdfLayoutProgressPhase = 'infer' | 'merge';
|
||||||
|
|
||||||
|
export interface PdfLayoutProgress {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
phase: PdfLayoutProgressPhase;
|
||||||
|
}
|
||||||
|
|
||||||
export interface WorkerJobStatusResponse<Result> {
|
export interface WorkerJobStatusResponse<Result> {
|
||||||
status: WorkerJobState;
|
status: WorkerJobState;
|
||||||
result?: Result;
|
result?: Result;
|
||||||
|
|
@ -96,4 +105,5 @@ export interface WorkerOperationState<Result = unknown> {
|
||||||
result?: Result;
|
result?: Result;
|
||||||
error?: WorkerJobErrorShape;
|
error?: WorkerJobErrorShape;
|
||||||
timing?: WorkerJobTiming;
|
timing?: WorkerJobTiming;
|
||||||
|
progress?: PdfLayoutProgress;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,10 +25,16 @@ export async function runWhisperAlignmentFromAudioBuffer(input: {
|
||||||
export async function runPdfLayoutFromPdfBuffer(input: {
|
export async function runPdfLayoutFromPdfBuffer(input: {
|
||||||
documentId: string;
|
documentId: string;
|
||||||
pdfBytes: ArrayBuffer;
|
pdfBytes: ArrayBuffer;
|
||||||
|
onPageParsed?: (input: {
|
||||||
|
pageNumber: number;
|
||||||
|
totalPages: number;
|
||||||
|
pageMs: number;
|
||||||
|
}) => void | Promise<void>;
|
||||||
}) {
|
}) {
|
||||||
const parsed = await parsePdf({
|
const parsed = await parsePdf({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
pdfBytes: input.pdfBytes,
|
pdfBytes: input.pdfBytes,
|
||||||
|
onPageParsed: input.onPageParsed,
|
||||||
});
|
});
|
||||||
return { parsed };
|
return { parsed };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,11 @@ import { renderPage } from './renderPage';
|
||||||
interface ParsePdfInput {
|
interface ParsePdfInput {
|
||||||
documentId: string;
|
documentId: string;
|
||||||
pdfBytes: ArrayBuffer;
|
pdfBytes: ArrayBuffer;
|
||||||
|
onPageParsed?: (input: {
|
||||||
|
pageNumber: number;
|
||||||
|
totalPages: number;
|
||||||
|
pageMs: number;
|
||||||
|
}) => void | Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
const LAYOUT_RENDER_SCALE = 1.5;
|
const LAYOUT_RENDER_SCALE = 1.5;
|
||||||
|
|
@ -80,6 +85,7 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
|
||||||
let sawText = false;
|
let sawText = false;
|
||||||
|
|
||||||
for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) {
|
for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) {
|
||||||
|
const pageStartedAt = Date.now();
|
||||||
const page = await pdf.getPage(pageNumber);
|
const page = await pdf.getPage(pageNumber);
|
||||||
const viewport = page.getViewport({ scale: 1.0 });
|
const viewport = page.getViewport({ scale: 1.0 });
|
||||||
const textContent = await page.getTextContent();
|
const textContent = await page.getTextContent();
|
||||||
|
|
@ -143,6 +149,14 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
|
||||||
height: viewport.height,
|
height: viewport.height,
|
||||||
blocks,
|
blocks,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (input.onPageParsed) {
|
||||||
|
await input.onPageParsed({
|
||||||
|
pageNumber,
|
||||||
|
totalPages: pdf.numPages,
|
||||||
|
pageMs: Date.now() - pageStartedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!sawText) {
|
if (!sawText) {
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,7 @@ import {
|
||||||
type WorkerOperationKind,
|
type WorkerOperationKind,
|
||||||
type WorkerOperationRequest,
|
type WorkerOperationRequest,
|
||||||
type WorkerOperationState,
|
type WorkerOperationState,
|
||||||
|
type PdfLayoutProgress,
|
||||||
} from '@openreader/compute-core/contracts';
|
} from '@openreader/compute-core/contracts';
|
||||||
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||||
|
|
||||||
|
|
@ -75,6 +76,7 @@ interface StoredJobState<Result> {
|
||||||
result?: Result;
|
result?: Result;
|
||||||
error?: WorkerJobErrorShape;
|
error?: WorkerJobErrorShape;
|
||||||
timing?: WorkerJobTiming;
|
timing?: WorkerJobTiming;
|
||||||
|
progress?: PdfLayoutProgress;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface OpIndexEntry {
|
interface OpIndexEntry {
|
||||||
|
|
@ -772,6 +774,7 @@ async function main(): Promise<void> {
|
||||||
const runLayout = async (
|
const runLayout = async (
|
||||||
payload: PdfLayoutJobRequest,
|
payload: PdfLayoutJobRequest,
|
||||||
queueWaitMs: number,
|
queueWaitMs: number,
|
||||||
|
hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise<void> },
|
||||||
): Promise<PdfLayoutJobResult> => {
|
): Promise<PdfLayoutJobResult> => {
|
||||||
const parsed = layoutSchema.parse(payload);
|
const parsed = layoutSchema.parse(payload);
|
||||||
|
|
||||||
|
|
@ -779,17 +782,38 @@ async function main(): Promise<void> {
|
||||||
const pdfBytes = await readObjectByKey(parsed.documentObjectKey);
|
const pdfBytes = await readObjectByKey(parsed.documentObjectKey);
|
||||||
const s3FetchMs = Date.now() - s3FetchStartedAt;
|
const s3FetchMs = Date.now() - s3FetchStartedAt;
|
||||||
|
|
||||||
|
let lastTotalPages = 0;
|
||||||
|
let lastPagesParsed = 0;
|
||||||
const computeStartedAt = Date.now();
|
const computeStartedAt = Date.now();
|
||||||
const result = await withTimeout(
|
const result = await withTimeout(
|
||||||
runPdfLayoutFromPdfBuffer({
|
runPdfLayoutFromPdfBuffer({
|
||||||
documentId: parsed.documentId,
|
documentId: parsed.documentId,
|
||||||
pdfBytes,
|
pdfBytes,
|
||||||
|
onPageParsed: async ({ pageNumber, totalPages }) => {
|
||||||
|
lastTotalPages = totalPages;
|
||||||
|
lastPagesParsed = pageNumber;
|
||||||
|
if (!hooks?.onProgress) return;
|
||||||
|
await hooks.onProgress({
|
||||||
|
totalPages,
|
||||||
|
pagesParsed: pageNumber,
|
||||||
|
currentPage: pageNumber,
|
||||||
|
phase: 'infer',
|
||||||
|
});
|
||||||
|
},
|
||||||
}),
|
}),
|
||||||
pdfTimeoutMs,
|
pdfTimeoutMs,
|
||||||
'pdf layout job',
|
'pdf layout job',
|
||||||
);
|
);
|
||||||
|
|
||||||
const computeMs = Date.now() - computeStartedAt;
|
const computeMs = Date.now() - computeStartedAt;
|
||||||
|
if (hooks?.onProgress && lastTotalPages > 0) {
|
||||||
|
await hooks.onProgress({
|
||||||
|
totalPages: lastTotalPages,
|
||||||
|
pagesParsed: lastPagesParsed,
|
||||||
|
currentPage: lastPagesParsed || undefined,
|
||||||
|
phase: 'merge',
|
||||||
|
});
|
||||||
|
}
|
||||||
const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed);
|
const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed);
|
||||||
return {
|
return {
|
||||||
parsedObjectKey,
|
parsedObjectKey,
|
||||||
|
|
@ -804,11 +828,16 @@ async function main(): Promise<void> {
|
||||||
async function processMessage<TPayload, TResult>(input: {
|
async function processMessage<TPayload, TResult>(input: {
|
||||||
msg: JsMsg;
|
msg: JsMsg;
|
||||||
codec: JsonCodec<QueuedJob<TPayload>>;
|
codec: JsonCodec<QueuedJob<TPayload>>;
|
||||||
run: (payload: TPayload, queueWaitMs: number) => Promise<TResult>;
|
run: (
|
||||||
|
payload: TPayload,
|
||||||
|
queueWaitMs: number,
|
||||||
|
hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise<void> },
|
||||||
|
) => Promise<TResult>;
|
||||||
workerLabel: string;
|
workerLabel: string;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
let decoded: QueuedJob<TPayload> | null = null;
|
let decoded: QueuedJob<TPayload> | null = null;
|
||||||
let heartbeat: NodeJS.Timeout | null = null;
|
let heartbeat: NodeJS.Timeout | null = null;
|
||||||
|
let latestProgress: PdfLayoutProgress | undefined;
|
||||||
try {
|
try {
|
||||||
decoded = input.codec.decode(input.msg.data);
|
decoded = input.codec.decode(input.msg.data);
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
|
|
@ -824,6 +853,7 @@ async function main(): Promise<void> {
|
||||||
startedAt,
|
startedAt,
|
||||||
updatedAt: startedAt,
|
updatedAt: startedAt,
|
||||||
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
await putOpState(runningState);
|
await putOpState(runningState);
|
||||||
|
|
@ -837,11 +867,11 @@ async function main(): Promise<void> {
|
||||||
startedAt,
|
startedAt,
|
||||||
updatedAt: startedAt,
|
updatedAt: startedAt,
|
||||||
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
heartbeat = setInterval(() => {
|
const persistRunningState = async (updatedAt: number): Promise<void> => {
|
||||||
const now = Date.now();
|
const runningOpState: WorkerOperationState<WhisperAlignJobResult | PdfLayoutJobResult> = {
|
||||||
const heartbeatState: WorkerOperationState<WhisperAlignJobResult | PdfLayoutJobResult> = {
|
|
||||||
opId: decoded!.opId,
|
opId: decoded!.opId,
|
||||||
opKey: decoded!.opKey,
|
opKey: decoded!.opKey,
|
||||||
kind: decoded!.kind,
|
kind: decoded!.kind,
|
||||||
|
|
@ -849,18 +879,13 @@ async function main(): Promise<void> {
|
||||||
status: 'running',
|
status: 'running',
|
||||||
queuedAt: decoded!.queuedAt,
|
queuedAt: decoded!.queuedAt,
|
||||||
startedAt,
|
startedAt,
|
||||||
updatedAt: now,
|
updatedAt,
|
||||||
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
};
|
};
|
||||||
void putOpState(heartbeatState).catch((stateError) => {
|
|
||||||
app.log.error({
|
await putOpState(runningOpState);
|
||||||
worker: input.workerLabel,
|
await putJobState({
|
||||||
opId: decoded?.opId,
|
|
||||||
jobId: decoded?.jobId,
|
|
||||||
error: toErrorMessage(stateError),
|
|
||||||
}, 'failed to persist running heartbeat op state');
|
|
||||||
});
|
|
||||||
void putJobState({
|
|
||||||
jobId: decoded!.jobId,
|
jobId: decoded!.jobId,
|
||||||
opId: decoded!.opId,
|
opId: decoded!.opId,
|
||||||
opKey: decoded!.opKey,
|
opKey: decoded!.opKey,
|
||||||
|
|
@ -868,19 +893,30 @@ async function main(): Promise<void> {
|
||||||
status: 'running',
|
status: 'running',
|
||||||
timestamp: decoded!.queuedAt,
|
timestamp: decoded!.queuedAt,
|
||||||
startedAt,
|
startedAt,
|
||||||
updatedAt: now,
|
updatedAt,
|
||||||
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
||||||
}).catch((stateError) => {
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
heartbeat = setInterval(() => {
|
||||||
|
const now = Date.now();
|
||||||
|
void persistRunningState(now).catch((stateError) => {
|
||||||
app.log.error({
|
app.log.error({
|
||||||
worker: input.workerLabel,
|
worker: input.workerLabel,
|
||||||
opId: decoded?.opId,
|
opId: decoded?.opId,
|
||||||
jobId: decoded?.jobId,
|
jobId: decoded?.jobId,
|
||||||
error: toErrorMessage(stateError),
|
error: toErrorMessage(stateError),
|
||||||
}, 'failed to persist running heartbeat job state');
|
}, 'failed to persist running heartbeat state');
|
||||||
});
|
});
|
||||||
}, RUNNING_HEARTBEAT_MS);
|
}, RUNNING_HEARTBEAT_MS);
|
||||||
|
|
||||||
const result = await input.run(decoded.payload, queueWaitMs ?? 0);
|
const result = await input.run(decoded.payload, queueWaitMs ?? 0, {
|
||||||
|
onProgress: async (progress) => {
|
||||||
|
latestProgress = progress;
|
||||||
|
await persistRunningState(Date.now());
|
||||||
|
},
|
||||||
|
});
|
||||||
const resultTiming = result && typeof result === 'object' && 'timing' in result
|
const resultTiming = result && typeof result === 'object' && 'timing' in result
|
||||||
? (result as { timing?: WorkerJobTiming }).timing
|
? (result as { timing?: WorkerJobTiming }).timing
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
@ -897,6 +933,7 @@ async function main(): Promise<void> {
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
result: result as WhisperAlignJobResult | PdfLayoutJobResult,
|
result: result as WhisperAlignJobResult | PdfLayoutJobResult,
|
||||||
...(resultTiming ? { timing: resultTiming } : {}),
|
...(resultTiming ? { timing: resultTiming } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
await putOpState(succeededState);
|
await putOpState(succeededState);
|
||||||
|
|
@ -911,6 +948,7 @@ async function main(): Promise<void> {
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
result: result as WhisperAlignJobResult | PdfLayoutJobResult,
|
result: result as WhisperAlignJobResult | PdfLayoutJobResult,
|
||||||
...(resultTiming ? { timing: resultTiming } : {}),
|
...(resultTiming ? { timing: resultTiming } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
input.msg.ack();
|
input.msg.ack();
|
||||||
|
|
@ -941,6 +979,7 @@ async function main(): Promise<void> {
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
...(status === 'failed' ? { error: { message } } : {}),
|
...(status === 'failed' ? { error: { message } } : {}),
|
||||||
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
await putOpState(opState).catch((stateError) => {
|
await putOpState(opState).catch((stateError) => {
|
||||||
|
|
@ -962,6 +1001,7 @@ async function main(): Promise<void> {
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
...(status === 'failed' ? { error: { message } } : {}),
|
...(status === 'failed' ? { error: { message } } : {}),
|
||||||
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}),
|
||||||
|
...(latestProgress ? { progress: latestProgress } : {}),
|
||||||
}).catch((stateError) => {
|
}).catch((stateError) => {
|
||||||
app.log.error({
|
app.log.error({
|
||||||
worker: input.workerLabel,
|
worker: input.workerLabel,
|
||||||
|
|
@ -1001,7 +1041,11 @@ async function main(): Promise<void> {
|
||||||
async function createWorkerLoop<TPayload, TResult>(input: {
|
async function createWorkerLoop<TPayload, TResult>(input: {
|
||||||
consumer: Consumer;
|
consumer: Consumer;
|
||||||
codec: JsonCodec<QueuedJob<TPayload>>;
|
codec: JsonCodec<QueuedJob<TPayload>>;
|
||||||
run: (payload: TPayload, queueWaitMs: number) => Promise<TResult>;
|
run: (
|
||||||
|
payload: TPayload,
|
||||||
|
queueWaitMs: number,
|
||||||
|
hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise<void> },
|
||||||
|
) => Promise<TResult>;
|
||||||
workerLabel: string;
|
workerLabel: string;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
while (!stopping) {
|
while (!stopping) {
|
||||||
|
|
|
||||||
2
drizzle/postgres/0007_pdf_parse_progress.sql
Normal file
2
drizzle/postgres/0007_pdf_parse_progress.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
ALTER TABLE "documents" ADD COLUMN "parse_state" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "documents" DROP COLUMN "parse_status";
|
||||||
1831
drizzle/postgres/meta/0007_snapshot.json
Normal file
1831
drizzle/postgres/meta/0007_snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -50,6 +50,13 @@
|
||||||
"when": 1779102950715,
|
"when": 1779102950715,
|
||||||
"tag": "0006_preview-defaults-cleanup",
|
"tag": "0006_preview-defaults-cleanup",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 7,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1779378127846,
|
||||||
|
"tag": "0007_pdf_parse_progress",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
2
drizzle/sqlite/0007_pdf_parse_progress.sql
Normal file
2
drizzle/sqlite/0007_pdf_parse_progress.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
ALTER TABLE `documents` ADD `parse_state` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `documents` DROP COLUMN `parse_status`;
|
||||||
1693
drizzle/sqlite/meta/0007_snapshot.json
Normal file
1693
drizzle/sqlite/meta/0007_snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -50,6 +50,13 @@
|
||||||
"when": 1779102950385,
|
"when": 1779102950385,
|
||||||
"tag": "0006_preview-defaults-cleanup",
|
"tag": "0006_preview-defaults-cleanup",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 7,
|
||||||
|
"version": "6",
|
||||||
|
"when": 1779378125905,
|
||||||
|
"tag": "0007_pdf_parse_progress",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
@ -42,6 +42,7 @@ export default function PDFViewerPage() {
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocPages,
|
currDocPages,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
|
parseProgress,
|
||||||
documentSettings,
|
documentSettings,
|
||||||
updateDocumentSettings,
|
updateDocumentSettings,
|
||||||
parsedOverlayEnabled,
|
parsedOverlayEnabled,
|
||||||
|
|
@ -240,30 +241,113 @@ export default function PDFViewerPage() {
|
||||||
}
|
}
|
||||||
|
|
||||||
const renderPdfStatusLoader = () => {
|
const renderPdfStatusLoader = () => {
|
||||||
|
const totalPages = parseProgress?.totalPages ?? 0;
|
||||||
|
const pagesParsed = parseProgress?.pagesParsed ?? 0;
|
||||||
|
const progressPercent = totalPages > 0
|
||||||
|
? Math.max(0, Math.min(100, (pagesParsed / totalPages) * 100))
|
||||||
|
: 0;
|
||||||
|
const hasMeasuredProgress = totalPages > 0;
|
||||||
|
const isMerging = parseProgress?.phase === 'merge';
|
||||||
|
|
||||||
let statusText = 'Loading PDF...';
|
let statusText = 'Loading PDF...';
|
||||||
|
let statusSubText = 'Initializing document renderer';
|
||||||
if (!isLoading) {
|
if (!isLoading) {
|
||||||
if (parseState === 'pending') {
|
if (parseState === 'pending') {
|
||||||
statusText = 'Preparing PDF layout...';
|
statusText = parseProgress
|
||||||
|
? `Page ${Math.max(0, parseProgress.pagesParsed)} / ${parseProgress.totalPages} parsed`
|
||||||
|
: 'Preparing PDF layout...';
|
||||||
|
statusSubText = parseProgress?.phase === 'merge'
|
||||||
|
? 'Finalizing stitched block structure'
|
||||||
|
: 'Queueing parser and preparing page extraction';
|
||||||
} else if (parseState === 'running') {
|
} else if (parseState === 'running') {
|
||||||
statusText = 'Parsing PDF layout blocks...';
|
statusText = parseProgress
|
||||||
|
? `Page ${Math.max(0, parseProgress.pagesParsed)} / ${parseProgress.totalPages} parsed`
|
||||||
|
: 'Parsing PDF layout blocks...';
|
||||||
|
statusSubText = parseProgress?.phase === 'merge'
|
||||||
|
? 'Merging cross-page sections'
|
||||||
|
: 'Inferring reading order and text regions';
|
||||||
} else if (parseState === 'failed') {
|
} else if (parseState === 'failed') {
|
||||||
statusText = 'PDF parsing failed. Retry to continue.';
|
statusText = 'PDF parsing failed. Retry to continue.';
|
||||||
|
statusSubText = 'The parser could not build a usable layout map';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const stageOneComplete = !isLoading;
|
||||||
|
const stageTwoActive = parseState === 'running' && !isMerging;
|
||||||
|
const stageTwoComplete = (parseState === 'running' && isMerging) || parseState === 'ready';
|
||||||
|
const stageThreeActive = parseState === 'running' && isMerging;
|
||||||
|
const stageThreeComplete = parseState === 'ready';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="h-full w-full flex flex-col items-center justify-center gap-4 bg-base">
|
<div className="h-full w-full bg-base">
|
||||||
<LoadingSpinner className="w-8 h-8 text-accent" />
|
<div className="mx-auto flex h-full max-w-2xl items-center px-4 py-8">
|
||||||
<p className="text-sm text-muted animate-pulse">{statusText}</p>
|
<div className="w-full space-y-3">
|
||||||
{!isLoading && parseState === 'failed' ? (
|
<div className="rounded-lg border border-offbase bg-offbase p-4 sm:p-5">
|
||||||
<button
|
<div className="flex items-start justify-between gap-3">
|
||||||
type="button"
|
<div className="min-w-0">
|
||||||
onClick={() => forceReparseParsedPdf()}
|
<p className="text-accent font-semibold text-[11px] uppercase tracking-wide">PDF Layout Parse</p>
|
||||||
className="inline-flex items-center rounded-md border border-offbase bg-offbase px-2.5 py-1 text-xs text-foreground hover:text-accent transition-colors"
|
<p className="mt-1 text-sm font-medium text-foreground">{statusText}</p>
|
||||||
>
|
<p className="mt-0.5 text-xs text-muted">{statusSubText}</p>
|
||||||
Retry Parse
|
</div>
|
||||||
</button>
|
<div className="shrink-0 inline-flex items-center gap-1.5 rounded-md border border-offbase bg-base px-2 py-1">
|
||||||
) : null}
|
<LoadingSpinner className="h-3.5 w-3.5 text-accent" />
|
||||||
|
<span className="text-[10px] font-semibold uppercase tracking-wide text-muted">
|
||||||
|
{parseState === 'failed' ? 'blocked' : (isMerging ? 'merge' : 'infer')}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="mt-3">
|
||||||
|
<div className="mb-1.5 flex items-center justify-between text-xs">
|
||||||
|
<span className="text-muted">Progress</span>
|
||||||
|
<span className="font-medium text-foreground">
|
||||||
|
{hasMeasuredProgress ? `${Math.round(progressPercent)}%` : 'Starting'}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="w-full bg-background rounded-full overflow-hidden h-1.5">
|
||||||
|
<div
|
||||||
|
className="h-full bg-accent transition-all duration-300 ease-out"
|
||||||
|
style={{ width: `${hasMeasuredProgress ? progressPercent : 7}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="mt-2 flex flex-wrap items-center gap-2 text-[11px] text-muted">
|
||||||
|
<span className="font-medium">{hasMeasuredProgress ? `Page ${pagesParsed}/${totalPages}` : 'Awaiting first page'}</span>
|
||||||
|
<span>•</span>
|
||||||
|
<span>{isMerging ? 'Cross-page merge' : 'Page inference'}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="rounded-lg border border-offbase bg-offbase px-4 py-3">
|
||||||
|
<div className="flex flex-wrap items-center gap-2 text-[11px]">
|
||||||
|
<span className={`inline-flex items-center gap-1 rounded-full px-2 py-1 ${stageOneComplete ? 'bg-accent/15 text-foreground' : 'bg-background text-muted'}`}>
|
||||||
|
<span className={`h-1.5 w-1.5 rounded-full ${stageOneComplete ? 'bg-accent' : 'bg-muted'}`} />
|
||||||
|
Prepare
|
||||||
|
</span>
|
||||||
|
<span className={`inline-flex items-center gap-1 rounded-full px-2 py-1 ${stageTwoActive || stageTwoComplete ? 'bg-accent/15 text-foreground' : 'bg-background text-muted'}`}>
|
||||||
|
<span className={`h-1.5 w-1.5 rounded-full ${(stageTwoActive || stageTwoComplete) ? 'bg-accent' : 'bg-muted'} ${stageTwoActive ? 'animate-pulse' : ''}`} />
|
||||||
|
Infer
|
||||||
|
</span>
|
||||||
|
<span className={`inline-flex items-center gap-1 rounded-full px-2 py-1 ${stageThreeActive || stageThreeComplete ? 'bg-accent/15 text-foreground' : 'bg-background text-muted'}`}>
|
||||||
|
<span className={`h-1.5 w-1.5 rounded-full ${(stageThreeActive || stageThreeComplete) ? 'bg-accent' : 'bg-muted'} ${stageThreeActive ? 'animate-pulse' : ''}`} />
|
||||||
|
Merge
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{!isLoading && parseState === 'failed' ? (
|
||||||
|
<div className="flex justify-start">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => forceReparseParsedPdf()}
|
||||||
|
className="inline-flex items-center rounded-md border border-offbase bg-offbase px-2.5 py-1 text-xs text-foreground hover:text-accent transition-colors"
|
||||||
|
>
|
||||||
|
Retry Parse
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ import {
|
||||||
getDocumentSettings,
|
getDocumentSettings,
|
||||||
getParsedPdfDocument,
|
getParsedPdfDocument,
|
||||||
putDocumentSettings,
|
putDocumentSettings,
|
||||||
|
subscribeParsedPdfDocumentEvents,
|
||||||
} from '@/lib/client/api/documents';
|
} from '@/lib/client/api/documents';
|
||||||
import { createPdfAudiobookSourceAdapter } from '@/lib/client/audiobooks/adapters/pdf';
|
import { createPdfAudiobookSourceAdapter } from '@/lib/client/audiobooks/adapters/pdf';
|
||||||
import { regenerateAudiobookChapter, runAudiobookGeneration } from '@/lib/client/audiobooks/pipeline';
|
import { regenerateAudiobookChapter, runAudiobookGeneration } from '@/lib/client/audiobooks/pipeline';
|
||||||
|
|
@ -44,7 +45,7 @@ import {
|
||||||
type DocumentSettings,
|
type DocumentSettings,
|
||||||
} from '@/types/document-settings';
|
} from '@/types/document-settings';
|
||||||
import { mergeDocumentSettings } from '@/lib/shared/document-settings';
|
import { mergeDocumentSettings } from '@/lib/shared/document-settings';
|
||||||
import type { ParsedPdfDocument, ParsedPdfPage, PdfParseStatus } from '@/types/parsed-pdf';
|
import type { ParsedPdfDocument, ParsedPdfPage, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
|
|
||||||
import type {
|
import type {
|
||||||
TTSSentenceAlignment,
|
TTSSentenceAlignment,
|
||||||
|
|
@ -68,6 +69,7 @@ export interface PdfDocumentState {
|
||||||
pdfDocument: PDFDocumentProxy | undefined;
|
pdfDocument: PDFDocumentProxy | undefined;
|
||||||
parsedDocument: ParsedPdfDocument | null;
|
parsedDocument: ParsedPdfDocument | null;
|
||||||
parseStatus: PdfParseStatus | null;
|
parseStatus: PdfParseStatus | null;
|
||||||
|
parseProgress: PdfParseProgress | null;
|
||||||
documentSettings: DocumentSettings;
|
documentSettings: DocumentSettings;
|
||||||
updateDocumentSettings: (settings: DocumentSettings) => Promise<void>;
|
updateDocumentSettings: (settings: DocumentSettings) => Promise<void>;
|
||||||
parsedOverlayEnabled: boolean;
|
parsedOverlayEnabled: boolean;
|
||||||
|
|
@ -143,6 +145,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
||||||
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
||||||
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
||||||
|
const [parseProgress, setParseProgress] = useState<PdfParseProgress | null>(null);
|
||||||
const [documentSettings, setDocumentSettings] = useState<DocumentSettings>(DEFAULT_DOCUMENT_SETTINGS);
|
const [documentSettings, setDocumentSettings] = useState<DocumentSettings>(DEFAULT_DOCUMENT_SETTINGS);
|
||||||
const [parsedOverlayEnabled, setParsedOverlayEnabled] = useState(false);
|
const [parsedOverlayEnabled, setParsedOverlayEnabled] = useState(false);
|
||||||
const [isAudioCombining] = useState(false);
|
const [isAudioCombining] = useState(false);
|
||||||
|
|
@ -164,6 +167,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
const docLoadSeqRef = useRef(0);
|
const docLoadSeqRef = useRef(0);
|
||||||
const docLoadAbortRef = useRef<AbortController | null>(null);
|
const docLoadAbortRef = useRef<AbortController | null>(null);
|
||||||
const parsePollAbortRef = useRef<AbortController | null>(null);
|
const parsePollAbortRef = useRef<AbortController | null>(null);
|
||||||
|
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
||||||
|
|
||||||
const fetchParsedDocument = useCallback(async (
|
const fetchParsedDocument = useCallback(async (
|
||||||
documentId: string,
|
documentId: string,
|
||||||
|
|
@ -174,11 +178,11 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
// document backfills parse output via the parsed endpoint polling path.
|
// document backfills parse output via the parsed endpoint polling path.
|
||||||
const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending';
|
const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending';
|
||||||
setParseStatus(effectiveInitialStatus);
|
setParseStatus(effectiveInitialStatus);
|
||||||
|
setParseProgress(null);
|
||||||
const maxAttempts = 25;
|
|
||||||
const delayMs = 1200;
|
const delayMs = 1200;
|
||||||
const retryFailed = effectiveInitialStatus === 'failed';
|
const retryFailed = effectiveInitialStatus === 'failed';
|
||||||
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
let attempt = 0;
|
||||||
|
while (!signal.aborted) {
|
||||||
if (signal.aborted) return;
|
if (signal.aborted) return;
|
||||||
const result = await getParsedPdfDocument(documentId, {
|
const result = await getParsedPdfDocument(documentId, {
|
||||||
signal,
|
signal,
|
||||||
|
|
@ -187,13 +191,16 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
if (result.status === 'ready') {
|
if (result.status === 'ready') {
|
||||||
setParsedDocument(result.parsed);
|
setParsedDocument(result.parsed);
|
||||||
setParseStatus('ready');
|
setParseStatus('ready');
|
||||||
|
setParseProgress(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
setParseStatus(result.status);
|
setParseStatus(result.status);
|
||||||
|
setParseProgress(result.parseProgress ?? null);
|
||||||
if (result.status === 'failed') {
|
if (result.status === 'failed') {
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
attempt += 1;
|
||||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
@ -211,13 +218,54 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
|
|
||||||
const startParsedPolling = useCallback((documentId: string, initialStatus: PdfParseStatus | null) => {
|
const startParsedPolling = useCallback((documentId: string, initialStatus: PdfParseStatus | null) => {
|
||||||
parsePollAbortRef.current?.abort();
|
parsePollAbortRef.current?.abort();
|
||||||
|
parseSseCloseRef.current?.();
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
|
setParseProgress(null);
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
parsePollAbortRef.current = controller;
|
parsePollAbortRef.current = controller;
|
||||||
void fetchParsedDocument(documentId, initialStatus, controller.signal).finally(() => {
|
|
||||||
|
const closeSse = subscribeParsedPdfDocumentEvents(documentId, {
|
||||||
|
onSnapshot: (snapshot) => {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
setParseStatus(snapshot.parseStatus);
|
||||||
|
setParseProgress(snapshot.parseProgress);
|
||||||
|
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
|
||||||
|
if (snapshot.parseStatus === 'failed') {
|
||||||
|
setParsedDocument(null);
|
||||||
|
} else {
|
||||||
|
void fetchParsedDocument(documentId, 'ready', controller.signal);
|
||||||
|
}
|
||||||
|
closeSse();
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
|
if (parsePollAbortRef.current === controller) {
|
||||||
|
parsePollAbortRef.current = null;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onError: () => {
|
||||||
|
// Fall back to parsed polling if browser SSE disconnects.
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
closeSse();
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
|
void fetchParsedDocument(documentId, initialStatus, controller.signal).finally(() => {
|
||||||
|
if (parsePollAbortRef.current === controller) {
|
||||||
|
parsePollAbortRef.current = null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
parseSseCloseRef.current = closeSse;
|
||||||
|
|
||||||
|
controller.signal.addEventListener('abort', () => {
|
||||||
|
closeSse();
|
||||||
|
if (parseSseCloseRef.current === closeSse) {
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
|
}
|
||||||
if (parsePollAbortRef.current === controller) {
|
if (parsePollAbortRef.current === controller) {
|
||||||
parsePollAbortRef.current = null;
|
parsePollAbortRef.current = null;
|
||||||
}
|
}
|
||||||
});
|
}, { once: true });
|
||||||
}, [fetchParsedDocument]);
|
}, [fetchParsedDocument]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|
@ -392,6 +440,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
loadSeqRef.current += 1;
|
loadSeqRef.current += 1;
|
||||||
parsePollAbortRef.current?.abort();
|
parsePollAbortRef.current?.abort();
|
||||||
parsePollAbortRef.current = null;
|
parsePollAbortRef.current = null;
|
||||||
|
parseSseCloseRef.current?.();
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
setPdfDocument(undefined);
|
setPdfDocument(undefined);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
|
|
@ -401,6 +451,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
setParseStatus(null);
|
setParseStatus(null);
|
||||||
|
setParseProgress(null);
|
||||||
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
||||||
|
|
||||||
const meta = await getDocumentMetadata(id, { signal: controller.signal });
|
const meta = await getDocumentMetadata(id, { signal: controller.signal });
|
||||||
|
|
@ -465,6 +516,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
await forceReparsePdfDocument(currDocId);
|
await forceReparsePdfDocument(currDocId);
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
setParseStatus('pending');
|
setParseStatus('pending');
|
||||||
|
setParseProgress(null);
|
||||||
startParsedPolling(currDocId, 'pending');
|
startParsedPolling(currDocId, 'pending');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to force PDF reparse:', error);
|
console.error('Failed to force PDF reparse:', error);
|
||||||
|
|
@ -485,6 +537,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
docLoadAbortRef.current = null;
|
docLoadAbortRef.current = null;
|
||||||
parsePollAbortRef.current?.abort();
|
parsePollAbortRef.current?.abort();
|
||||||
parsePollAbortRef.current = null;
|
parsePollAbortRef.current = null;
|
||||||
|
parseSseCloseRef.current?.();
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
setCurrDocId(undefined);
|
setCurrDocId(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
|
|
@ -493,6 +547,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setPdfDocument(undefined);
|
setPdfDocument(undefined);
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
setParseStatus(null);
|
setParseStatus(null);
|
||||||
|
setParseProgress(null);
|
||||||
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
stop();
|
stop();
|
||||||
|
|
@ -595,6 +650,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
currDocText,
|
currDocText,
|
||||||
parsedDocument,
|
parsedDocument,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
|
parseProgress,
|
||||||
documentSettings,
|
documentSettings,
|
||||||
updateDocumentSettings,
|
updateDocumentSettings,
|
||||||
parsedOverlayEnabled,
|
parsedOverlayEnabled,
|
||||||
|
|
@ -621,6 +677,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
currDocText,
|
currDocText,
|
||||||
parsedDocument,
|
parsedDocument,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
|
parseProgress,
|
||||||
documentSettings,
|
documentSettings,
|
||||||
updateDocumentSettings,
|
updateDocumentSettings,
|
||||||
parsedOverlayEnabled,
|
parsedOverlayEnabled,
|
||||||
|
|
|
||||||
191
src/app/api/documents/[id]/parsed/events/route.ts
Normal file
191
src/app/api/documents/[id]/parsed/events/route.ts
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { and, eq, inArray } from 'drizzle-orm';
|
||||||
|
import { db } from '@/db';
|
||||||
|
import { documents } from '@/db/schema';
|
||||||
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
||||||
|
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
|
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
|
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
|
import { normalizeParseStatus, parseDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
const SSE_POLL_INTERVAL_MS = 1200;
|
||||||
|
|
||||||
|
type ParseRow = {
|
||||||
|
id: string;
|
||||||
|
userId: string;
|
||||||
|
parseState: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ParsedSnapshot = {
|
||||||
|
parseStatus: PdfParseStatus;
|
||||||
|
parseProgress: PdfParseProgress | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
function s3NotConfiguredResponse(): NextResponse {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
function toSnapshot(row: ParseRow): ParsedSnapshot {
|
||||||
|
const state = parseDocumentParseState(row.parseState);
|
||||||
|
const parseStatus = normalizeParseStatus(state.status);
|
||||||
|
return {
|
||||||
|
parseStatus,
|
||||||
|
parseProgress: state.progress ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadPreferredRow(input: {
|
||||||
|
documentId: string;
|
||||||
|
storageUserId: string;
|
||||||
|
allowedUserIds: string[];
|
||||||
|
}): Promise<ParseRow | null> {
|
||||||
|
const rows = (await db
|
||||||
|
.select({
|
||||||
|
id: documents.id,
|
||||||
|
userId: documents.userId,
|
||||||
|
parseState: documents.parseState,
|
||||||
|
})
|
||||||
|
.from(documents)
|
||||||
|
.where(and(eq(documents.id, input.documentId), inArray(documents.userId, input.allowedUserIds)))) as ParseRow[];
|
||||||
|
|
||||||
|
return rows.find((candidate) => candidate.userId === input.storageUserId) ?? rows[0] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||||
|
try {
|
||||||
|
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||||
|
|
||||||
|
const authCtxOrRes = await requireAuthContext(req);
|
||||||
|
if (authCtxOrRes instanceof Response) return authCtxOrRes;
|
||||||
|
|
||||||
|
const params = await ctx.params;
|
||||||
|
const id = (params.id || '').trim().toLowerCase();
|
||||||
|
if (!isValidDocumentId(id)) {
|
||||||
|
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||||
|
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||||
|
const storageUserId = authCtxOrRes.userId ?? unclaimedUserId;
|
||||||
|
const allowedUserIds = authCtxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
|
||||||
|
|
||||||
|
const row = await loadPreferredRow({
|
||||||
|
documentId: id,
|
||||||
|
storageUserId,
|
||||||
|
allowedUserIds,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!row) {
|
||||||
|
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const initial = toSnapshot(row);
|
||||||
|
if (initial.parseStatus === 'pending') {
|
||||||
|
enqueueParsePdfJob({
|
||||||
|
documentId: id,
|
||||||
|
userId: row.userId,
|
||||||
|
namespace: testNamespace,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const encoder = new TextEncoder();
|
||||||
|
|
||||||
|
const stream = new ReadableStream<Uint8Array>({
|
||||||
|
start(controller) {
|
||||||
|
let closed = false;
|
||||||
|
|
||||||
|
const writeSnapshot = (snapshot: ParsedSnapshot): void => {
|
||||||
|
controller.enqueue(encoder.encode(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`));
|
||||||
|
};
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
let current = initial;
|
||||||
|
writeSnapshot(current);
|
||||||
|
let signature = JSON.stringify(current);
|
||||||
|
|
||||||
|
while (!closed) {
|
||||||
|
if (current.parseStatus === 'ready' || current.parseStatus === 'failed') break;
|
||||||
|
await sleep(SSE_POLL_INTERVAL_MS);
|
||||||
|
if (closed) break;
|
||||||
|
|
||||||
|
const nextRow = await loadPreferredRow({
|
||||||
|
documentId: id,
|
||||||
|
storageUserId,
|
||||||
|
allowedUserIds,
|
||||||
|
});
|
||||||
|
if (!nextRow) break;
|
||||||
|
|
||||||
|
const next = toSnapshot(nextRow);
|
||||||
|
if (next.parseStatus === 'pending') {
|
||||||
|
enqueueParsePdfJob({
|
||||||
|
documentId: id,
|
||||||
|
userId: nextRow.userId,
|
||||||
|
namespace: testNamespace,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextSignature = JSON.stringify(next);
|
||||||
|
if (nextSignature !== signature) {
|
||||||
|
current = next;
|
||||||
|
signature = nextSignature;
|
||||||
|
writeSnapshot(current);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
void run()
|
||||||
|
.catch((error) => {
|
||||||
|
if (!closed) {
|
||||||
|
controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify({ error: String(error) })}\n\n`));
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!closed) {
|
||||||
|
closed = true;
|
||||||
|
try {
|
||||||
|
controller.close();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
req.signal.addEventListener('abort', () => {
|
||||||
|
if (closed) return;
|
||||||
|
closed = true;
|
||||||
|
try {
|
||||||
|
controller.close();
|
||||||
|
} catch {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
}, { once: true });
|
||||||
|
},
|
||||||
|
cancel() {
|
||||||
|
return;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return new Response(stream, {
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'text/event-stream; charset=utf-8',
|
||||||
|
'Cache-Control': 'no-cache, no-transform',
|
||||||
|
Connection: 'keep-alive',
|
||||||
|
'X-Accel-Buffering': 'no',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error streaming parsed PDF progress:', error);
|
||||||
|
return NextResponse.json({ error: 'Failed to stream parsed PDF progress' }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -10,6 +10,11 @@ import {
|
||||||
isValidDocumentId,
|
isValidDocumentId,
|
||||||
} from '@/lib/server/documents/blobstore';
|
} from '@/lib/server/documents/blobstore';
|
||||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
||||||
|
import {
|
||||||
|
normalizeParseStatus,
|
||||||
|
parseDocumentParseState,
|
||||||
|
stringifyDocumentParseState,
|
||||||
|
} from '@/lib/server/documents/parse-state';
|
||||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
||||||
|
|
@ -28,15 +33,6 @@ function hasAnyParsedBlocks(doc: ParsedPdfDocument | null): boolean {
|
||||||
return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0);
|
return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeParseStatus(
|
|
||||||
status: string | null,
|
|
||||||
): 'pending' | 'running' | 'ready' | 'failed' {
|
|
||||||
if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') {
|
|
||||||
return status;
|
|
||||||
}
|
|
||||||
return 'pending';
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||||
try {
|
try {
|
||||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||||
|
|
@ -60,14 +56,14 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
.select({
|
.select({
|
||||||
id: documents.id,
|
id: documents.id,
|
||||||
userId: documents.userId,
|
userId: documents.userId,
|
||||||
parseStatus: documents.parseStatus,
|
parseState: documents.parseState,
|
||||||
parsedJsonKey: documents.parsedJsonKey,
|
parsedJsonKey: documents.parsedJsonKey,
|
||||||
})
|
})
|
||||||
.from(documents)
|
.from(documents)
|
||||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
|
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
|
||||||
id: string;
|
id: string;
|
||||||
userId: string;
|
userId: string;
|
||||||
parseStatus: string | null;
|
parseState: string | null;
|
||||||
parsedJsonKey: string | null;
|
parsedJsonKey: string | null;
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
|
@ -76,31 +72,26 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const effectiveStatus = normalizeParseStatus(row.parseStatus);
|
const state = parseDocumentParseState(row.parseState);
|
||||||
if (row.parseStatus !== effectiveStatus) {
|
const effectiveStatus = normalizeParseStatus(state.status);
|
||||||
await db
|
|
||||||
.update(documents)
|
|
||||||
.set({ parseStatus: 'pending' })
|
|
||||||
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
|
|
||||||
enqueueParsePdfJob({
|
|
||||||
documentId: id,
|
|
||||||
userId: row.userId,
|
|
||||||
namespace: testNamespace,
|
|
||||||
});
|
|
||||||
return NextResponse.json({ parseStatus: 'pending' }, { status: 202 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (effectiveStatus === 'failed' && retryFailed) {
|
if (effectiveStatus === 'failed' && retryFailed) {
|
||||||
await db
|
await db
|
||||||
.update(documents)
|
.update(documents)
|
||||||
.set({ parseStatus: 'pending' })
|
.set({
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
}),
|
||||||
|
})
|
||||||
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
|
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
|
||||||
enqueueParsePdfJob({
|
enqueueParsePdfJob({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
namespace: testNamespace,
|
namespace: testNamespace,
|
||||||
});
|
});
|
||||||
return NextResponse.json({ parseStatus: 'pending' }, { status: 202 });
|
return NextResponse.json({ parseStatus: 'pending', parseProgress: null }, { status: 202 });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (effectiveStatus !== 'ready') {
|
if (effectiveStatus !== 'ready') {
|
||||||
|
|
@ -111,7 +102,10 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
namespace: testNamespace,
|
namespace: testNamespace,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return NextResponse.json({ parseStatus: effectiveStatus }, { status: 202 });
|
return NextResponse.json({
|
||||||
|
parseStatus: effectiveStatus,
|
||||||
|
parseProgress: state.progress ?? null,
|
||||||
|
}, { status: 202 });
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -174,14 +168,14 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
.select({
|
.select({
|
||||||
id: documents.id,
|
id: documents.id,
|
||||||
userId: documents.userId,
|
userId: documents.userId,
|
||||||
parseStatus: documents.parseStatus,
|
parseState: documents.parseState,
|
||||||
parsedJsonKey: documents.parsedJsonKey,
|
parsedJsonKey: documents.parsedJsonKey,
|
||||||
})
|
})
|
||||||
.from(documents)
|
.from(documents)
|
||||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
|
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
|
||||||
id: string;
|
id: string;
|
||||||
userId: string;
|
userId: string;
|
||||||
parseStatus: string | null;
|
parseState: string | null;
|
||||||
parsedJsonKey: string | null;
|
parsedJsonKey: string | null;
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
|
@ -190,12 +184,19 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const effectiveStatus = normalizeParseStatus(row.parseStatus);
|
const state = parseDocumentParseState(row.parseState);
|
||||||
|
const effectiveStatus = normalizeParseStatus(state.status);
|
||||||
|
|
||||||
if (effectiveStatus !== 'running') {
|
if (effectiveStatus !== 'running') {
|
||||||
await db
|
await db
|
||||||
.update(documents)
|
.update(documents)
|
||||||
.set({ parseStatus: 'pending' })
|
.set({
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
}),
|
||||||
|
})
|
||||||
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
|
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -206,7 +207,10 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
});
|
});
|
||||||
|
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ parseStatus: effectiveStatus === 'running' ? 'running' : 'pending' },
|
{
|
||||||
|
parseStatus: effectiveStatus === 'running' ? 'running' : 'pending',
|
||||||
|
parseProgress: effectiveStatus === 'running' ? (state.progress ?? null) : null,
|
||||||
|
},
|
||||||
{ status: 202 },
|
{ status: 202 },
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,11 @@ import {
|
||||||
} from '@/lib/server/documents/previews';
|
} from '@/lib/server/documents/previews';
|
||||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
||||||
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
|
import {
|
||||||
|
normalizeParseStatus,
|
||||||
|
parseDocumentParseState,
|
||||||
|
stringifyDocumentParseState,
|
||||||
|
} from '@/lib/server/documents/parse-state';
|
||||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
|
|
@ -25,16 +30,6 @@ type RegisterDocument = {
|
||||||
lastModified: number;
|
lastModified: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
function normalizeParseStatus(
|
|
||||||
status: string | null,
|
|
||||||
): 'pending' | 'running' | 'ready' | 'failed' | null {
|
|
||||||
if (status === null) return null;
|
|
||||||
if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') {
|
|
||||||
return status;
|
|
||||||
}
|
|
||||||
return 'pending';
|
|
||||||
}
|
|
||||||
|
|
||||||
function s3NotConfiguredResponse(): NextResponse {
|
function s3NotConfiguredResponse(): NextResponse {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||||
|
|
@ -140,7 +135,9 @@ export async function POST(req: NextRequest) {
|
||||||
size: headSize,
|
size: headSize,
|
||||||
lastModified: doc.lastModified,
|
lastModified: doc.lastModified,
|
||||||
filePath: doc.id,
|
filePath: doc.id,
|
||||||
parseStatus: doc.type === 'pdf' ? 'pending' : null,
|
parseState: doc.type === 'pdf'
|
||||||
|
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
||||||
|
: null,
|
||||||
parsedJsonKey: null,
|
parsedJsonKey: null,
|
||||||
})
|
})
|
||||||
.onConflictDoUpdate({
|
.onConflictDoUpdate({
|
||||||
|
|
@ -151,7 +148,9 @@ export async function POST(req: NextRequest) {
|
||||||
size: headSize,
|
size: headSize,
|
||||||
lastModified: doc.lastModified,
|
lastModified: doc.lastModified,
|
||||||
filePath: doc.id,
|
filePath: doc.id,
|
||||||
parseStatus: doc.type === 'pdf' ? 'pending' : null,
|
parseState: doc.type === 'pdf'
|
||||||
|
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
||||||
|
: null,
|
||||||
parsedJsonKey: null,
|
parsedJsonKey: null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
@ -229,7 +228,7 @@ export async function GET(req: NextRequest) {
|
||||||
size: number;
|
size: number;
|
||||||
lastModified: number;
|
lastModified: number;
|
||||||
filePath: string;
|
filePath: string;
|
||||||
parseStatus: string | null;
|
parseState: string | null;
|
||||||
parsedJsonKey: string | null;
|
parsedJsonKey: string | null;
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
|
@ -255,7 +254,7 @@ export async function GET(req: NextRequest) {
|
||||||
size: Number(doc.size),
|
size: Number(doc.size),
|
||||||
lastModified: Number(doc.lastModified),
|
lastModified: Number(doc.lastModified),
|
||||||
type,
|
type,
|
||||||
parseStatus: normalizeParseStatus(doc.parseStatus),
|
parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null,
|
||||||
parsedJsonKey: doc.parsedJsonKey,
|
parsedJsonKey: doc.parsedJsonKey,
|
||||||
scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user',
|
scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user',
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ export const documents = pgTable('documents', {
|
||||||
size: bigint('size', { mode: 'number' }).notNull(),
|
size: bigint('size', { mode: 'number' }).notNull(),
|
||||||
lastModified: bigint('last_modified', { mode: 'number' }).notNull(),
|
lastModified: bigint('last_modified', { mode: 'number' }).notNull(),
|
||||||
filePath: text('file_path').notNull(),
|
filePath: text('file_path').notNull(),
|
||||||
parseStatus: text('parse_status'),
|
parseState: text('parse_state'),
|
||||||
parsedJsonKey: text('parsed_json_key'),
|
parsedJsonKey: text('parsed_json_key'),
|
||||||
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
|
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
|
||||||
}, (table) => [
|
}, (table) => [
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ export const documents = sqliteTable('documents', {
|
||||||
size: integer('size').notNull(),
|
size: integer('size').notNull(),
|
||||||
lastModified: integer('last_modified').notNull(),
|
lastModified: integer('last_modified').notNull(),
|
||||||
filePath: text('file_path').notNull(),
|
filePath: text('file_path').notNull(),
|
||||||
parseStatus: text('parse_status'),
|
parseState: text('parse_state'),
|
||||||
parsedJsonKey: text('parsed_json_key'),
|
parsedJsonKey: text('parsed_json_key'),
|
||||||
createdAt: integer('created_at').default(SQLITE_NOW_MS),
|
createdAt: integer('created_at').default(SQLITE_NOW_MS),
|
||||||
}, (table) => [
|
}, (table) => [
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { sha256HexFromArrayBuffer } from '@/lib/client/sha256';
|
import { sha256HexFromArrayBuffer } from '@/lib/client/sha256';
|
||||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
import type { DocumentSettings } from '@/types/document-settings';
|
import type { DocumentSettings } from '@/types/document-settings';
|
||||||
|
|
||||||
export type UploadSource = {
|
export type UploadSource = {
|
||||||
|
|
@ -81,7 +81,10 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
|
||||||
export async function getParsedPdfDocument(
|
export async function getParsedPdfDocument(
|
||||||
id: string,
|
id: string,
|
||||||
options?: { signal?: AbortSignal; retryFailed?: boolean },
|
options?: { signal?: AbortSignal; retryFailed?: boolean },
|
||||||
): Promise<{ status: 'ready'; parsed: ParsedPdfDocument } | { status: 'pending' | 'running' | 'failed' }> {
|
): Promise<
|
||||||
|
| { status: 'ready'; parsed: ParsedPdfDocument }
|
||||||
|
| { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null }
|
||||||
|
> {
|
||||||
const query = options?.retryFailed ? '?retry=1' : '';
|
const query = options?.retryFailed ? '?retry=1' : '';
|
||||||
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, {
|
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, {
|
||||||
signal: options?.signal,
|
signal: options?.signal,
|
||||||
|
|
@ -89,12 +92,12 @@ export async function getParsedPdfDocument(
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 202) {
|
if (res.status === 202) {
|
||||||
const data = (await res.json().catch(() => null)) as { parseStatus?: string } | null;
|
const data = (await res.json().catch(() => null)) as { parseStatus?: string; parseProgress?: PdfParseProgress | null } | null;
|
||||||
const parseStatus = data?.parseStatus;
|
const parseStatus = data?.parseStatus;
|
||||||
if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') {
|
if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') {
|
||||||
return { status: parseStatus };
|
return { status: parseStatus, parseProgress: data?.parseProgress ?? null };
|
||||||
}
|
}
|
||||||
return { status: 'pending' };
|
return { status: 'pending', parseProgress: data?.parseProgress ?? null };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
|
|
@ -106,6 +109,31 @@ export async function getParsedPdfDocument(
|
||||||
return { status: 'ready', parsed };
|
return { status: 'ready', parsed };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function subscribeParsedPdfDocumentEvents(
|
||||||
|
id: string,
|
||||||
|
handlers: {
|
||||||
|
onSnapshot: (snapshot: { parseStatus: PdfParseStatus; parseProgress: PdfParseProgress | null }) => void;
|
||||||
|
onError?: (error: Event) => void;
|
||||||
|
},
|
||||||
|
): () => void {
|
||||||
|
const source = new EventSource(`/api/documents/${encodeURIComponent(id)}/parsed/events`);
|
||||||
|
source.addEventListener('snapshot', (event) => {
|
||||||
|
if (!(event instanceof MessageEvent)) return;
|
||||||
|
try {
|
||||||
|
const payload = JSON.parse(event.data) as { parseStatus: PdfParseStatus; parseProgress: PdfParseProgress | null };
|
||||||
|
handlers.onSnapshot(payload);
|
||||||
|
} catch {
|
||||||
|
// Ignore malformed payloads to avoid breaking active streams.
|
||||||
|
}
|
||||||
|
});
|
||||||
|
source.addEventListener('error', (event) => {
|
||||||
|
handlers.onError?.(event);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
source.close();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function forceReparsePdfDocument(
|
export async function forceReparsePdfDocument(
|
||||||
id: string,
|
id: string,
|
||||||
options?: { signal?: AbortSignal },
|
options?: { signal?: AbortSignal },
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,17 @@ export class LocalComputeBackend implements ComputeBackend {
|
||||||
if (!pdfBytes) {
|
if (!pdfBytes) {
|
||||||
throw new Error('Local compute PDF layout requires pdfBytes or (documentId + namespace)');
|
throw new Error('Local compute PDF layout requires pdfBytes or (documentId + namespace)');
|
||||||
}
|
}
|
||||||
return { parsed: (await runPdfLayoutFromPdfBuffer({ documentId: input.documentId, pdfBytes })).parsed };
|
return {
|
||||||
|
parsed: (await runPdfLayoutFromPdfBuffer({
|
||||||
|
documentId: input.documentId,
|
||||||
|
pdfBytes,
|
||||||
|
onPageParsed: (page) => input.onProgress?.({
|
||||||
|
totalPages: page.totalPages,
|
||||||
|
pagesParsed: page.pageNumber,
|
||||||
|
currentPage: page.pageNumber,
|
||||||
|
phase: 'infer',
|
||||||
|
}),
|
||||||
|
})).parsed,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,9 @@
|
||||||
import type { TTSAudioBuffer, TTSSentenceAlignment, ParsedPdfDocument } from '@openreader/compute-core/contracts';
|
import type {
|
||||||
|
TTSAudioBuffer,
|
||||||
|
TTSSentenceAlignment,
|
||||||
|
ParsedPdfDocument,
|
||||||
|
PdfLayoutProgress,
|
||||||
|
} from '@openreader/compute-core/contracts';
|
||||||
|
|
||||||
export type ComputeMode = 'local' | 'worker';
|
export type ComputeMode = 'local' | 'worker';
|
||||||
|
|
||||||
|
|
@ -19,6 +24,7 @@ export interface PdfLayoutInput {
|
||||||
namespace?: string | null;
|
namespace?: string | null;
|
||||||
documentObjectKey?: string;
|
documentObjectKey?: string;
|
||||||
pdfBytes?: ArrayBuffer;
|
pdfBytes?: ArrayBuffer;
|
||||||
|
onProgress?: (progress: PdfLayoutProgress) => void | Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type PdfLayoutResult =
|
export type PdfLayoutResult =
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
export {
|
export {
|
||||||
type PdfLayoutJobRequest,
|
type PdfLayoutJobRequest,
|
||||||
type PdfLayoutJobResult,
|
type PdfLayoutJobResult,
|
||||||
|
type PdfLayoutProgress,
|
||||||
|
type PdfLayoutProgressPhase,
|
||||||
type PdfLayoutOperationRequest,
|
type PdfLayoutOperationRequest,
|
||||||
type WhisperAlignJobRequest,
|
type WhisperAlignJobRequest,
|
||||||
type WhisperAlignJobResult,
|
type WhisperAlignJobResult,
|
||||||
|
|
|
||||||
|
|
@ -375,6 +375,11 @@ export class WorkerComputeBackend implements ComputeBackend {
|
||||||
opKeyHash,
|
opKeyHash,
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
attempt,
|
attempt,
|
||||||
|
onSnapshot: (snapshot) => {
|
||||||
|
if (snapshot.progress) {
|
||||||
|
void input.onProgress?.(snapshot.progress);
|
||||||
|
}
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (final.status !== 'succeeded' || !final.result) {
|
if (final.status !== 'succeeded' || !final.result) {
|
||||||
|
|
@ -482,7 +487,9 @@ export class WorkerComputeBackend implements ComputeBackend {
|
||||||
|
|
||||||
private async waitForOperation<Result>(
|
private async waitForOperation<Result>(
|
||||||
opId: string,
|
opId: string,
|
||||||
context: Record<string, unknown> = {},
|
context: Record<string, unknown> & {
|
||||||
|
onSnapshot?: (snapshot: WorkerOperationState<Result>) => void
|
||||||
|
} = {},
|
||||||
): Promise<WorkerOperationState<Result>> {
|
): Promise<WorkerOperationState<Result>> {
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeout = setTimeout(() => controller.abort(), this.waitTimeoutMs);
|
const timeout = setTimeout(() => controller.abort(), this.waitTimeoutMs);
|
||||||
|
|
@ -577,6 +584,7 @@ export class WorkerComputeBackend implements ComputeBackend {
|
||||||
|
|
||||||
eventCount += 1;
|
eventCount += 1;
|
||||||
latest = snapshot;
|
latest = snapshot;
|
||||||
|
context.onSnapshot?.(snapshot);
|
||||||
if (snapshot.status !== lastStatus) {
|
if (snapshot.status !== lastStatus) {
|
||||||
lastStatus = snapshot.status;
|
lastStatus = snapshot.status;
|
||||||
logWorker('info', 'sse.wait.status', {
|
logWorker('info', 'sse.wait.status', {
|
||||||
|
|
|
||||||
72
src/lib/server/documents/parse-state.ts
Normal file
72
src/lib/server/documents/parse-state.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
|
|
||||||
|
export interface DocumentParseState {
|
||||||
|
status: PdfParseStatus;
|
||||||
|
progress?: PdfParseProgress | null;
|
||||||
|
updatedAt?: number;
|
||||||
|
error?: string | null;
|
||||||
|
opId?: string;
|
||||||
|
jobId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeParseStatus(status: string | null | undefined): PdfParseStatus {
|
||||||
|
if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
return 'pending';
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
||||||
|
if (!progress || typeof progress !== 'object') return null;
|
||||||
|
const rec = progress as Record<string, unknown>;
|
||||||
|
const totalPages = Number(rec.totalPages ?? 0);
|
||||||
|
if (!Number.isFinite(totalPages) || totalPages <= 0) return null;
|
||||||
|
const pagesParsedRaw = Number(rec.pagesParsed ?? 0);
|
||||||
|
const pagesParsed = Math.max(0, Math.min(totalPages, Number.isFinite(pagesParsedRaw) ? pagesParsedRaw : 0));
|
||||||
|
const phase = rec.phase === 'merge' ? 'merge' : 'infer';
|
||||||
|
const currentPageRaw = Number(rec.currentPage ?? pagesParsed);
|
||||||
|
const currentPage = Number.isFinite(currentPageRaw) && currentPageRaw > 0
|
||||||
|
? Math.max(1, Math.min(totalPages, currentPageRaw))
|
||||||
|
: undefined;
|
||||||
|
return {
|
||||||
|
totalPages,
|
||||||
|
pagesParsed,
|
||||||
|
currentPage,
|
||||||
|
phase,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseDocumentParseState(value: string | null): DocumentParseState {
|
||||||
|
if (!value) return { status: 'pending', progress: null };
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value) as Record<string, unknown>;
|
||||||
|
const status = normalizeParseStatus(typeof parsed.status === 'string' ? parsed.status : null);
|
||||||
|
const progress = normalizeProgress(parsed.progress);
|
||||||
|
const updatedAtRaw = Number(parsed.updatedAt ?? 0);
|
||||||
|
const updatedAt = Number.isFinite(updatedAtRaw) && updatedAtRaw > 0 ? updatedAtRaw : undefined;
|
||||||
|
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
||||||
|
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
||||||
|
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
progress,
|
||||||
|
...(typeof updatedAt === 'number' ? { updatedAt } : {}),
|
||||||
|
...(error ? { error } : {}),
|
||||||
|
...(opId ? { opId } : {}),
|
||||||
|
...(jobId ? { jobId } : {}),
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return { status: 'pending', progress: null };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stringifyDocumentParseState(state: DocumentParseState): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
status: normalizeParseStatus(state.status),
|
||||||
|
progress: state.progress ?? null,
|
||||||
|
updatedAt: typeof state.updatedAt === 'number' ? state.updatedAt : Date.now(),
|
||||||
|
...(state.error ? { error: state.error } : {}),
|
||||||
|
...(state.opId ? { opId: state.opId } : {}),
|
||||||
|
...(state.jobId ? { jobId: state.jobId } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
@ -2,8 +2,10 @@ import { and, eq } from 'drizzle-orm';
|
||||||
import { db } from '@/db';
|
import { db } from '@/db';
|
||||||
import { documents } from '@/db/schema';
|
import { documents } from '@/db/schema';
|
||||||
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||||
|
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
import { getCompute } from '@/lib/server/compute';
|
import { getCompute } from '@/lib/server/compute';
|
||||||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||||
|
import type { PdfLayoutProgress } from '@openreader/compute-core/contracts';
|
||||||
|
|
||||||
interface ParsePdfJobInput {
|
interface ParsePdfJobInput {
|
||||||
documentId: string;
|
documentId: string;
|
||||||
|
|
@ -23,16 +25,41 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise<void> {
|
||||||
running.add(key);
|
running.add(key);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const now = Date.now();
|
||||||
await db
|
await db
|
||||||
.update(documents)
|
.update(documents)
|
||||||
.set({ parseStatus: 'running' })
|
.set({
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'running',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: now,
|
||||||
|
}),
|
||||||
|
})
|
||||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||||
|
|
||||||
const compute = await getCompute();
|
const compute = await getCompute();
|
||||||
|
const writeProgress = async (progress: PdfLayoutProgress): Promise<void> => {
|
||||||
|
await db
|
||||||
|
.update(documents)
|
||||||
|
.set({
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'running',
|
||||||
|
progress: {
|
||||||
|
totalPages: progress.totalPages,
|
||||||
|
pagesParsed: progress.pagesParsed,
|
||||||
|
currentPage: progress.currentPage,
|
||||||
|
phase: progress.phase,
|
||||||
|
},
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||||
|
};
|
||||||
const layout = await compute.parsePdfLayout({
|
const layout = await compute.parsePdfLayout({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
namespace: input.namespace,
|
namespace: input.namespace,
|
||||||
documentObjectKey: documentKey(input.documentId, input.namespace),
|
documentObjectKey: documentKey(input.documentId, input.namespace),
|
||||||
|
onProgress: writeProgress,
|
||||||
});
|
});
|
||||||
|
|
||||||
let parsedJsonKey = layout.parsedObjectKey ?? null;
|
let parsedJsonKey = layout.parsedObjectKey ?? null;
|
||||||
|
|
@ -56,7 +83,14 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise<void> {
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.update(documents)
|
.update(documents)
|
||||||
.set({ parseStatus: 'ready', parsedJsonKey })
|
.set({
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
}),
|
||||||
|
parsedJsonKey,
|
||||||
|
})
|
||||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
|
@ -66,7 +100,14 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise<void> {
|
||||||
try {
|
try {
|
||||||
await db
|
await db
|
||||||
.update(documents)
|
.update(documents)
|
||||||
.set({ parseStatus })
|
.set({
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'failed',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
error: message,
|
||||||
|
}),
|
||||||
|
})
|
||||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||||
} catch (statusError) {
|
} catch (statusError) {
|
||||||
console.error('[parsePdfJob] failed to write parse status', {
|
console.error('[parsePdfJob] failed to write parse status', {
|
||||||
|
|
|
||||||
|
|
@ -77,3 +77,11 @@ export interface ParsedPdfDocument {
|
||||||
}
|
}
|
||||||
|
|
||||||
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed';
|
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed';
|
||||||
|
export type PdfParsePhase = 'infer' | 'merge';
|
||||||
|
|
||||||
|
export interface PdfParseProgress {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
phase: PdfParsePhase;
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ test.describe('transferUserDocuments', () => {
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
last_modified INTEGER NOT NULL,
|
last_modified INTEGER NOT NULL,
|
||||||
file_path TEXT NOT NULL,
|
file_path TEXT NOT NULL,
|
||||||
parse_status TEXT,
|
parse_state TEXT,
|
||||||
parsed_json_key TEXT,
|
parsed_json_key TEXT,
|
||||||
created_at INTEGER,
|
created_at INTEGER,
|
||||||
PRIMARY KEY (id, user_id)
|
PRIMARY KEY (id, user_id)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue