diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index abd8e0f..9660453 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -3,10 +3,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { readComputeMode } from '@/lib/server/compute/mode'; import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; -import { getParseProgressBus } from '@/lib/server/documents/parse-progress-bus'; -import { hashUserId } from '@/lib/server/documents/parse-progress-events'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { normalizeParseStatus, parseDocumentParseState } from '@/lib/server/documents/parse-state'; import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing'; @@ -161,22 +158,16 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } const initialState = await toSnapshotState(row); - const computeMode = readComputeMode(); - const bus = computeMode === 'local' ? await getParseProgressBus() : null; - - const workerCfg = computeMode === 'worker' ? getWorkerClientConfigFromEnv() : null; + const workerCfg = getWorkerClientConfigFromEnv(); const encoder = new TextEncoder(); const stream = new ReadableStream({ start(controller) { let closed = false; - let unsubscribe: (() => void) | null = null; let keepaliveTimer: ReturnType | null = null; let resyncTimer: ReturnType | null = null; let workerAbort: AbortController | null = null; - const allowedUserHashes = new Set(allowedUserIds.map((candidate) => hashUserId(candidate))); - const writeSnapshot = (snapshot: ParsedSnapshot): void => { controller.enqueue(encoder.encode(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`)); }; @@ -192,10 +183,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string clearInterval(resyncTimer); resyncTimer = null; } - if (unsubscribe) { - unsubscribe(); - unsubscribe = null; - } if (workerAbort) { workerAbort.abort(); workerAbort = null; @@ -207,85 +194,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } }; - const runLocalRealtime = async () => { - let current = initialState.snapshot; - let signature = JSON.stringify(current); - writeSnapshot(current); - - if (current.parseStatus === 'ready' || current.parseStatus === 'failed') { - closeStream(); - return; - } - - keepaliveTimer = setInterval(() => { - if (closed) return; - controller.enqueue(encoder.encode(': keepalive\n\n')); - }, SSE_KEEPALIVE_MS); - - resyncTimer = setInterval(() => { - if (closed) return; - void syncFromDb({ - documentId: id, - storageUserId, - allowedUserIds, - signature, - writeSnapshot, - }).then((next) => { - if (closed) return; - if (!next) { - closeStream(); - return; - } - current = next.snapshot; - signature = next.signature; - if (current.parseStatus === 'ready' || current.parseStatus === 'failed') { - closeStream(); - } - }).catch((error) => { - if (closed) return; - controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify({ error: String(error) })}\n\n`)); - closeStream(); - }); - }, SSE_RESYNC_INTERVAL_MS); - - if (!bus) throw new Error('Local parse progress bus unavailable'); - const nextUnsubscribe = await bus.subscribe({ - documentId: id, - userIdHashes: allowedUserHashes, - onEvent: async () => { - const next = await syncFromDb({ - documentId: id, - storageUserId, - allowedUserIds, - signature, - writeSnapshot, - }); - if (!next) { - closeStream(); - return; - } - current = next.snapshot; - signature = next.signature; - if (current.parseStatus === 'ready' || current.parseStatus === 'failed') { - closeStream(); - } - }, - onError: (error) => { - if (closed) return; - controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify({ error: String(error) })}\n\n`)); - closeStream(); - }, - }); - if (closed) { - nextUnsubscribe(); - return; - } - unsubscribe = nextUnsubscribe; - }; - const runWorkerProxy = async () => { - if (!workerCfg) throw new Error('Worker mode selected but worker config is unavailable'); - let current = initialState.snapshot; let signature = JSON.stringify(current); let currentOpId = initialState.opId; @@ -480,9 +389,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } }; - const run = computeMode === 'local' ? runLocalRealtime : runWorkerProxy; - - void run() + void runWorkerProxy() .catch((error) => { if (!closed) { controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify({ error: String(error) })}\n\n`)); diff --git a/src/lib/server/documents/parse-progress-bus.memory.ts b/src/lib/server/documents/parse-progress-bus.memory.ts deleted file mode 100644 index 9a707f1..0000000 --- a/src/lib/server/documents/parse-progress-bus.memory.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { EventEmitter } from 'node:events'; -import type { ParseProgressBus, ParseProgressBusSubscribeInput } from '@/lib/server/documents/parse-progress-bus.types'; -import type { ParseProgressEvent } from '@/lib/server/documents/parse-progress-events'; - -type Listener = (event: ParseProgressEvent) => void; - -function topicForDocument(documentId: string): string { - return `parse-progress.${documentId}`; -} - -export class InMemoryParseProgressBus implements ParseProgressBus { - readonly kind = 'memory' as const; - private readonly emitter = new EventEmitter(); - - constructor() { - this.emitter.setMaxListeners(0); - } - - async publish(event: ParseProgressEvent): Promise { - this.emitter.emit(topicForDocument(event.documentId), event); - } - - async subscribe(input: ParseProgressBusSubscribeInput): Promise<() => void> { - const listener: Listener = (event) => { - if (!input.userIdHashes.has(event.userIdHash)) return; - Promise.resolve(input.onEvent(event)).catch((error) => { - input.onError?.(error); - }); - }; - const topic = topicForDocument(input.documentId); - this.emitter.on(topic, listener); - return () => { - this.emitter.off(topic, listener); - }; - } -} diff --git a/src/lib/server/documents/parse-progress-bus.ts b/src/lib/server/documents/parse-progress-bus.ts deleted file mode 100644 index df6e27b..0000000 --- a/src/lib/server/documents/parse-progress-bus.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { InMemoryParseProgressBus } from '@/lib/server/documents/parse-progress-bus.memory'; -import type { ParseProgressBus } from '@/lib/server/documents/parse-progress-bus.types'; - -let busPromise: Promise | null = null; - -export async function getParseProgressBus(): Promise { - if (busPromise) return busPromise; - busPromise = Promise.resolve(new InMemoryParseProgressBus()); - return busPromise; -} - -export function __resetParseProgressBusForTests(): void { - busPromise = null; -} - -export type { ParseProgressBus, ParseProgressBusSubscribeInput } from '@/lib/server/documents/parse-progress-bus.types'; diff --git a/src/lib/server/documents/parse-progress-bus.types.ts b/src/lib/server/documents/parse-progress-bus.types.ts deleted file mode 100644 index 423d257..0000000 --- a/src/lib/server/documents/parse-progress-bus.types.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { ParseProgressEvent } from '@/lib/server/documents/parse-progress-events'; - -export interface ParseProgressBusSubscribeInput { - documentId: string; - userIdHashes: Set; - onEvent: (event: ParseProgressEvent) => void | Promise; - onError?: (error: unknown) => void; -} - -export interface ParseProgressBus { - readonly kind: 'memory'; - publish(event: ParseProgressEvent): Promise; - subscribe(input: ParseProgressBusSubscribeInput): Promise<() => void>; -} diff --git a/src/lib/server/documents/parse-progress-events.ts b/src/lib/server/documents/parse-progress-events.ts deleted file mode 100644 index f43fc91..0000000 --- a/src/lib/server/documents/parse-progress-events.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { createHash } from 'node:crypto'; -import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; - -export interface ParseProgressEvent { - version: 1; - documentId: string; - userIdHash: string; - parseStatus: PdfParseStatus; - parseProgress: PdfParseProgress | null; - updatedAt: number; - opId?: string; - jobId?: string; - error?: string | null; -} - -export function hashUserId(userId: string): string { - return createHash('sha256').update(userId).digest('hex').slice(0, 24); -} diff --git a/src/lib/server/documents/parse-state-healing.ts b/src/lib/server/documents/parse-state-healing.ts index 909bbe3..5824e37 100644 --- a/src/lib/server/documents/parse-state-healing.ts +++ b/src/lib/server/documents/parse-state-healing.ts @@ -8,8 +8,6 @@ import { stringifyDocumentParseState, type DocumentParseState, } from '@/lib/server/documents/parse-state'; -import { getParseProgressBus } from '@/lib/server/documents/parse-progress-bus'; -import { hashUserId } from '@/lib/server/documents/parse-progress-events'; export async function healStaleDocumentParseState(input: { documentId: string; @@ -31,26 +29,5 @@ export async function healStaleDocumentParseState(input: { .set({ parseState: stringifyDocumentParseState(nextState) }) .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); - try { - const bus = await getParseProgressBus(); - await bus.publish({ - version: 1, - documentId: input.documentId, - userIdHash: hashUserId(input.userId), - parseStatus: nextState.status, - parseProgress: nextState.progress ?? null, - updatedAt: nextState.updatedAt ?? Date.now(), - ...(nextState.opId ? { opId: nextState.opId } : {}), - ...(nextState.jobId ? { jobId: nextState.jobId } : {}), - ...(nextState.error ? { error: nextState.error } : {}), - }); - } catch (error) { - console.warn('[parse-state-healing] failed to publish stale state event', { - documentId: input.documentId, - userId: input.userId, - error: error instanceof Error ? error.message : String(error), - }); - } - return nextState; } diff --git a/src/lib/server/jobs/user-pdf-layout-job.ts b/src/lib/server/jobs/user-pdf-layout-job.ts index 1b75556..0dd7eba 100644 --- a/src/lib/server/jobs/user-pdf-layout-job.ts +++ b/src/lib/server/jobs/user-pdf-layout-job.ts @@ -8,8 +8,6 @@ import { stringifyDocumentParseState, type DocumentParseState, } from '@/lib/server/documents/parse-state'; -import { getParseProgressBus } from '@/lib/server/documents/parse-progress-bus'; -import { hashUserId } from '@/lib/server/documents/parse-progress-events'; import { getCompute } from '@/lib/server/compute'; import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache'; import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy'; @@ -29,7 +27,7 @@ const running = new Set(); const FOLLOWER_WAIT_TIMEOUT_MS = 180_000; const FOLLOWER_POLL_MS = 1_200; -const WORKER_PROGRESS_DB_THROTTLE_MS = 10_000; +const PROGRESS_DB_THROTTLE_MS = 10_000; type ParseRow = { userId: string; @@ -122,41 +120,6 @@ async function updateParseStateForUsers(input: { ); } -function normalizeEventState(state: DocumentParseState): DocumentParseState { - return parseDocumentParseState(stringifyDocumentParseState(state)); -} - -async function emitParseStateEvents(input: { - documentId: string; - userIds: string[]; - state: DocumentParseState; -}): Promise { - if (input.userIds.length === 0) return; - const normalized = normalizeEventState(input.state); - const bus = await getParseProgressBus(); - - const publishResults = await Promise.allSettled(input.userIds.map(async (userId) => bus.publish({ - version: 1, - documentId: input.documentId, - userIdHash: hashUserId(userId), - parseStatus: normalized.status, - parseProgress: normalized.progress ?? null, - updatedAt: normalized.updatedAt ?? Date.now(), - ...(normalized.opId ? { opId: normalized.opId } : {}), - ...(normalized.jobId ? { jobId: normalized.jobId } : {}), - ...(normalized.error ? { error: normalized.error } : {}), - }))); - - publishResults.forEach((result, index) => { - if (result.status === 'fulfilled') return; - console.warn('[parsePdfJob] failed to publish parse progress event', { - documentId: input.documentId, - userId: input.userIds[index], - error: result.reason instanceof Error ? result.reason.message : String(result.reason), - }); - }); -} - async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise { const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS; @@ -175,11 +138,6 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise parseState: stringifyDocumentParseState(readyState), parsedJsonKey: ready.parsedJsonKey, }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: [input.userId], - state: readyState, - }); return; } @@ -198,11 +156,6 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise userIds: [input.userId], parseState: stringifyDocumentParseState(nextFailedState), }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: [input.userId], - state: nextFailedState, - }); return; } @@ -240,11 +193,6 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise parseState: stringifyDocumentParseState(readyState), parsedJsonKey: ready.parsedJsonKey, }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: [input.userId], - state: readyState, - }); return; } } @@ -289,21 +237,15 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise userIds: cohortUserIds, parseState: runningState, }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: cohortUserIds, - state: runningStateData, - }); const compute = await getCompute(); - const isWorkerCompute = compute.mode === 'worker'; let activeOpId: string = claimOpId; let activeJobId: string | undefined; - let lastWorkerProgressWriteAt = 0; - let lastWorkerSnapshotWriteAt = 0; - let lastWorkerSnapshotStatus: 'pending' | 'running' | null = null; - let lastWorkerSnapshotOpId: string = claimOpId; - let lastWorkerSnapshotJobId: string | undefined; + let lastProgressWriteAt = 0; + let lastSnapshotWriteAt = 0; + let lastSnapshotStatus: 'pending' | 'running' | null = null; + let lastSnapshotOpId: string = claimOpId; + let lastSnapshotJobId: string | undefined; const persistRunningState = async (state: DocumentParseState): Promise => { await updateParseStateForUsers({ @@ -311,15 +253,9 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise userIds: cohortUserIds, parseState: stringifyDocumentParseState(state), }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: cohortUserIds, - state, - }); }; const onWorkerSnapshot = async (snapshot: WorkerOperationState): Promise => { - if (!isWorkerCompute) return; if (snapshot.opId) activeOpId = snapshot.opId; if (snapshot.jobId) activeJobId = snapshot.jobId; @@ -332,11 +268,11 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise const now = Date.now(); const forceWrite = ( - mappedStatus !== lastWorkerSnapshotStatus - || activeOpId !== lastWorkerSnapshotOpId - || activeJobId !== lastWorkerSnapshotJobId + mappedStatus !== lastSnapshotStatus + || activeOpId !== lastSnapshotOpId + || activeJobId !== lastSnapshotJobId ); - if (!forceWrite && (now - lastWorkerSnapshotWriteAt) < WORKER_PROGRESS_DB_THROTTLE_MS) { + if (!forceWrite && (now - lastSnapshotWriteAt) < PROGRESS_DB_THROTTLE_MS) { return; } @@ -348,20 +284,19 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise ...(activeJobId ? { jobId: activeJobId } : {}), }; await persistRunningState(nextState); - lastWorkerSnapshotWriteAt = now; - lastWorkerSnapshotStatus = mappedStatus; - lastWorkerSnapshotOpId = activeOpId; - lastWorkerSnapshotJobId = activeJobId; + lastSnapshotWriteAt = now; + lastSnapshotStatus = mappedStatus; + lastSnapshotOpId = activeOpId; + lastSnapshotJobId = activeJobId; }; const writeProgress = async (progress: PdfLayoutProgress): Promise => { - if (isWorkerCompute) { - const now = Date.now(); - if ((now - lastWorkerProgressWriteAt) < WORKER_PROGRESS_DB_THROTTLE_MS && progress.pagesParsed < progress.totalPages) { - return; - } - lastWorkerProgressWriteAt = now; + const now = Date.now(); + if ((now - lastProgressWriteAt) < PROGRESS_DB_THROTTLE_MS && progress.pagesParsed < progress.totalPages) { + return; } + lastProgressWriteAt = now; + const runningProgressState: DocumentParseState = { status: 'running', progress: { @@ -376,6 +311,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise }; await persistRunningState(runningProgressState); }; + const layout = await compute.parsePdfLayout({ documentId: input.documentId, namespace: input.namespace, @@ -406,11 +342,6 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise parseState: stringifyDocumentParseState(readyState), parsedJsonKey, }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: readyUserIds, - state: readyState, - }); // Best-effort cache invalidation should not block parse readiness. for (const userId of readyUserIds) { @@ -454,11 +385,6 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise userIds: failedUserIds, parseState: stringifyDocumentParseState(failedState), }); - await emitParseStateEvents({ - documentId: input.documentId, - userIds: failedUserIds, - state: failedState, - }); } catch (statusError) { console.error('[parsePdfJob] failed to write parse status', { documentId: input.documentId, diff --git a/tests/unit/parse-progress-bus.spec.ts b/tests/unit/parse-progress-bus.spec.ts deleted file mode 100644 index 754c71d..0000000 --- a/tests/unit/parse-progress-bus.spec.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { __resetParseProgressBusForTests, getParseProgressBus } from '../../src/lib/server/documents/parse-progress-bus'; -import { InMemoryParseProgressBus } from '../../src/lib/server/documents/parse-progress-bus.memory'; -import { hashUserId, type ParseProgressEvent } from '../../src/lib/server/documents/parse-progress-events'; - -test.describe('parse progress bus', () => { - test.afterEach(() => { - __resetParseProgressBusForTests(); - }); - - test('uses in-memory singleton bus', async () => { - const busA = await getParseProgressBus(); - const busB = await getParseProgressBus(); - expect(busA.kind).toBe('memory'); - expect(busA).toBe(busB); - }); - - test('in-memory bus publishes only to subscribed user hashes', async () => { - const bus = new InMemoryParseProgressBus(); - const documentId = 'd'.repeat(64); - const wantedHash = hashUserId('user-a'); - const otherHash = hashUserId('user-b'); - const received: ParseProgressEvent[] = []; - - const close = await bus.subscribe({ - documentId, - userIdHashes: new Set([wantedHash]), - onEvent: (event) => { - received.push(event); - }, - }); - - await bus.publish({ - version: 1, - documentId, - userIdHash: otherHash, - parseStatus: 'running', - parseProgress: { totalPages: 4, pagesParsed: 1, currentPage: 2, phase: 'infer' }, - updatedAt: Date.now(), - }); - - await bus.publish({ - version: 1, - documentId, - userIdHash: wantedHash, - parseStatus: 'running', - parseProgress: { totalPages: 4, pagesParsed: 2, currentPage: 3, phase: 'infer' }, - updatedAt: Date.now(), - opId: 'op-1', - }); - - close(); - expect(received).toHaveLength(1); - expect(received[0].userIdHash).toBe(wantedHash); - expect(received[0].opId).toBe('op-1'); - }); -});