From 2e4f36f5c5f90d29e9d8114a421449d02cd3bc36 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 3 Jun 2026 18:51:29 -0600 Subject: [PATCH 1/9] feat(pdf): introduce parser versioning and playback readiness state Add PDF_PARSER_VERSION constant and propagate parser versioning throughout the PDF parsing, job, and API layers. Implement normalization of parse state to ensure compatibility with the current parser version, and enable reuse of parsed PDF results when possible. Add isPlaybackReady state to document hooks and TTS player, improving playback UX by disabling controls until content is ready. Update tests to reflect new playback readiness logic. BREAKING CHANGE: PDF parse state and job logic now require explicit parserVersion; older parse states may be treated as pending until reprocessed. --- compute/core/src/index.ts | 1 + compute/core/src/pdf/parse.ts | 3 +- compute/core/src/pdf/parser-version.ts | 1 + compute/core/src/pdf/pdfjs-runtime.ts | 5 +- src/app/(app)/epub/[id]/page.tsx | 3 +- src/app/(app)/epub/[id]/useEpubDocument.ts | 8 ++ src/app/(app)/html/[id]/page.tsx | 3 +- src/app/(app)/html/[id]/useHtmlDocument.ts | 22 ++++- src/app/(app)/pdf/[id]/page.tsx | 3 +- src/app/(app)/pdf/[id]/usePdfDocument.ts | 21 ++++- .../api/documents/[id]/parsed/events/route.ts | 5 +- src/app/api/documents/[id]/parsed/route.ts | 23 +++-- .../api/documents/docx-to-pdf/upload/route.ts | 15 +++- src/app/api/documents/route.ts | 29 +++++-- src/components/player/TTSPlayer.tsx | 13 +-- src/contexts/TTSContext.tsx | 33 +++++++- src/lib/server/compute/worker-parse-state.ts | 2 + src/lib/server/compute/worker.ts | 3 +- .../server/documents/parse-state-healing.ts | 1 + src/lib/server/documents/parse-state.ts | 32 ++++++- src/lib/server/documents/parsed-pdf-reuse.ts | 33 ++++++++ src/lib/server/jobs/user-pdf-layout-job.ts | 71 +++++++++++++++- tests/helpers.ts | 7 +- tests/navigation.spec.ts | 6 +- tests/unit/pdf-op-key.vitest.spec.ts | 13 +++ tests/unit/pdf-parse-state.vitest.spec.ts | 84 +++++++++++++++++++ tests/unit/worker-parse-state.vitest.spec.ts | 6 +- 27 files changed, 398 insertions(+), 48 deletions(-) create mode 100644 compute/core/src/pdf/parser-version.ts create mode 100644 src/lib/server/documents/parsed-pdf-reuse.ts create mode 100644 tests/unit/pdf-op-key.vitest.spec.ts create mode 100644 tests/unit/pdf-parse-state.vitest.spec.ts diff --git a/compute/core/src/index.ts b/compute/core/src/index.ts index 4ba1e89..4360a58 100644 --- a/compute/core/src/index.ts +++ b/compute/core/src/index.ts @@ -16,6 +16,7 @@ export { } from './config/timeout'; export { renderPage } from './pdf/render'; export { mergeTextWithRegions } from './pdf/merge'; +export { PDF_PARSER_VERSION } from './pdf/parser-version'; export { stitchCrossPageBlocks } from './pdf/stitch'; export { normalizeTextItemsForLayout } from './pdf/normalize-text'; export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map'; diff --git a/compute/core/src/pdf/parse.ts b/compute/core/src/pdf/parse.ts index 4cf0cd3..f0bde04 100644 --- a/compute/core/src/pdf/parse.ts +++ b/compute/core/src/pdf/parse.ts @@ -4,6 +4,7 @@ import { ensureModel } from './model'; import { runLayoutModel } from './runLayoutModel'; import { mergeTextWithRegions } from './merge'; import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime'; +import { PDF_PARSER_VERSION } from './parser-version'; import { stitchCrossPageBlocks } from './stitch'; import { renderPage } from './render'; import { normalizeTextItemsForLayout } from './normalize-text'; @@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise const doc: ParsedPdfDocument = { schemaVersion: 1, documentId: input.documentId, - parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69', + parserVersion: PDF_PARSER_VERSION, parsedAt: Date.now(), pages, }; diff --git a/compute/core/src/pdf/parser-version.ts b/compute/core/src/pdf/parser-version.ts new file mode 100644 index 0000000..b5fa7d7 --- /dev/null +++ b/compute/core/src/pdf/parser-version.ts @@ -0,0 +1 @@ +export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69'; diff --git a/compute/core/src/pdf/pdfjs-runtime.ts b/compute/core/src/pdf/pdfjs-runtime.ts index 206e650..233d9c3 100644 --- a/compute/core/src/pdf/pdfjs-runtime.ts +++ b/compute/core/src/pdf/pdfjs-runtime.ts @@ -1,12 +1,15 @@ import fs from 'node:fs'; +import { createRequire } from 'node:module'; import path from 'node:path'; let cachedStandardFontDataUrl: string | null = null; +const require = createRequire(import.meta.url); export function resolvePdfjsStandardFontDataUrl(): string { if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl; - const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts'); + const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json')); + const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts'); if (!fs.existsSync(standardFontDir)) { throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`); diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index 4a25c2e..b08d4d4 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -31,6 +31,7 @@ export default function EPUBPage() { const { setCurrentDocument, currDocName, + isPlaybackReady, clearCurrDoc, createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter, @@ -242,7 +243,7 @@ export default function EPUBPage() { ) : ( - + )} Promise; clearCurrDoc: () => void; extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise; @@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [isPlaybackReady, setIsPlaybackReady] = useState(false); // Mirror state into a ref so resolveEpubLocator (registered once with // TTSContext via a stable callback) can always read the latest page text // without forcing re-registration on every page turn. @@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { setCurrDocData(undefined); setCurrDocName(undefined); setCurrDocText(undefined); + setIsPlaybackReady(false); setCurrDocPages(undefined); isEPUBSetOnce.current = false; shouldPauseRef.current = true; @@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { */ const setCurrentDocument = useCallback(async (id: string): Promise => { try { + setIsPlaybackReady(false); const meta = await getDocumentMetadata(id); if (!meta) { clearCurrDoc(); @@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { */ const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise => { try { + setIsPlaybackReady(false); const location = rendition?.location; if (!location) return ''; const { start, end } = location; @@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { nextText: continuationPreview }); setCurrDocText(textContent); + setIsPlaybackReady(true); return textContent; } catch (error) { @@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, clearCurrDoc, extractPageText, walkUpcomingRenderedLocations, @@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, clearCurrDoc, extractPageText, walkUpcomingRenderedLocations, diff --git a/src/app/(app)/html/[id]/page.tsx b/src/app/(app)/html/[id]/page.tsx index f0193a3..09a0b45 100644 --- a/src/app/(app)/html/[id]/page.tsx +++ b/src/app/(app)/html/[id]/page.tsx @@ -31,6 +31,7 @@ export default function HTMLPage() { setCurrentDocument, currDocData, currDocName, + isPlaybackReady, blocks, isTxt, clearCurrDoc, @@ -233,7 +234,7 @@ export default function HTMLPage() { ) : ( - + )} Promise; @@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState { const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); + const [isPlaybackReady, setIsPlaybackReady] = useState(false); const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]); const blocks = useMemo( @@ -91,22 +93,36 @@ export function useHtmlDocument(): HtmlDocumentState { // sentence splitting + sequential advancement from there. const lastFedDocRef = useRef(null); useEffect(() => { - if (!currDocText) return; + if (currDocData === undefined) { + setIsPlaybackReady(false); + return; + } + if (!currDocText) { + setIsPlaybackReady(true); + return; + } const key = `${currDocData ?? ''}::${currDocText.length}`; - if (lastFedDocRef.current === key) return; + if (lastFedDocRef.current === key) { + setIsPlaybackReady(true); + return; + } + setIsPlaybackReady(false); lastFedDocRef.current = key; setTTSText(currDocText); + setIsPlaybackReady(true); }, [currDocText, currDocData, setTTSText]); const clearCurrDoc = useCallback(() => { setCurrDocData(undefined); setCurrDocName(undefined); + setIsPlaybackReady(false); lastFedDocRef.current = null; stop(); }, [stop]); const setCurrentDocument = useCallback(async (id: string): Promise => { try { + setIsPlaybackReady(false); const meta = await getDocumentMetadata(id); if (!meta) { console.error('Document not found on server'); @@ -207,6 +223,7 @@ export function useHtmlDocument(): HtmlDocumentState { currDocData, currDocName, currDocText, + isPlaybackReady, blocks, isTxt, setCurrentDocument, @@ -218,6 +235,7 @@ export function useHtmlDocument(): HtmlDocumentState { currDocData, currDocName, currDocText, + isPlaybackReady, blocks, isTxt, setCurrentDocument, diff --git a/src/app/(app)/pdf/[id]/page.tsx b/src/app/(app)/pdf/[id]/page.tsx index 79d4060..6cfe8cf 100644 --- a/src/app/(app)/pdf/[id]/page.tsx +++ b/src/app/(app)/pdf/[id]/page.tsx @@ -52,6 +52,7 @@ export default function PDFViewerPage() { clearCurrDoc, currDocPage, currDocPages, + isPlaybackReady, parseStatus, parseProgress, documentSettings, @@ -487,7 +488,7 @@ export default function PDFViewerPage() { ) : isParseReady ? ( - + ) : null} (); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [isPlaybackReady, setIsPlaybackReady] = useState(false); const [pdfDocument, setPdfDocument] = useState(); const [parsedDocument, setParsedDocument] = useState(null); const [parseStatus, setParseStatus] = useState(null); @@ -169,6 +171,7 @@ export function usePdfDocument(): PdfDocumentState { const docLoadAbortRef = useRef(null); const parsePollAbortRef = useRef(null); const parseSseCloseRef = useRef<(() => void) | null>(null); + const lastPreparedPlaybackPageRef = useRef(null); const fetchParsedDocument = useCallback(async ( documentId: string, @@ -248,6 +251,8 @@ export function usePdfDocument(): PdfDocumentState { if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') { if (snapshot.parseStatus === 'failed') { setParsedDocument(null); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; setActiveParseOpId(null); } else { void fetchParsedDocument( @@ -297,6 +302,7 @@ export function usePdfDocument(): PdfDocumentState { useEffect(() => { setCurrDocPage(currDocPageNumber); + setIsPlaybackReady(false); }, [currDocPageNumber]); /** @@ -324,11 +330,13 @@ export function usePdfDocument(): PdfDocumentState { if (!currentPdf) return; const seq = ++loadSeqRef.current; const pageNumber = currDocPageNumber; + setIsPlaybackReady(false); const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined => parsedDocument?.pages.find((page) => page.pageNumber === pageNum); if (parseStatus !== 'ready' || !parsedDocument) { + lastPreparedPlaybackPageRef.current = null; setCurrDocText(undefined); setTTSText('', { location: currDocPageNumber }); return; @@ -401,7 +409,8 @@ export function usePdfDocument(): PdfDocumentState { return; } - if (text !== currDocText || text === '') { + const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber; + if (shouldPreparePlayback) { setCurrDocText(text); const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber); setTTSText(text, { @@ -414,6 +423,8 @@ export function usePdfDocument(): PdfDocumentState { ...(sourceUnits.length > 0 ? { sourceUnits } : {}), }); } + lastPreparedPlaybackPageRef.current = currDocPageNumber; + setIsPlaybackReady(true); } catch (error) { if (error instanceof DOMException && error.name === 'AbortError') { return; @@ -469,6 +480,8 @@ export function usePdfDocument(): PdfDocumentState { setPdfDocument(undefined); setCurrDocPages(undefined); setCurrDocText(undefined); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; setCurrDocId(id); setCurrDocName(undefined); setCurrDocData(undefined); @@ -546,6 +559,8 @@ export function usePdfDocument(): PdfDocumentState { pageTextCacheRef.current.clear(); setParsedDocument(null); setCurrDocText(undefined); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; setParseStatus(forced.status); setParseProgress(null); setActiveParseOpId(forced.opId ?? null); @@ -575,6 +590,7 @@ export function usePdfDocument(): PdfDocumentState { setCurrDocName(undefined); setCurrDocData(undefined); setCurrDocText(undefined); + setIsPlaybackReady(false); setCurrDocPages(undefined); setPdfDocument(undefined); setParsedDocument(null); @@ -582,6 +598,7 @@ export function usePdfDocument(): PdfDocumentState { setParseProgress(null); setActiveParseOpId(null); setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS); + lastPreparedPlaybackPageRef.current = null; pageTextCacheRef.current.clear(); stop(); }, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]); @@ -681,6 +698,7 @@ export function usePdfDocument(): PdfDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, parsedDocument, parseStatus, parseProgress, @@ -708,6 +726,7 @@ export function usePdfDocument(): PdfDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, parsedDocument, parseStatus, parseProgress, diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index e1adc9d..79d6801 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -9,6 +9,7 @@ import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/serv import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { + normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, stringifyDocumentParseState, @@ -67,7 +68,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr const state = await healStaleDocumentParseState({ documentId: row.id, userId: row.userId, - state: parseDocumentParseState(row.parseState), + state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)), }); const parseStatus = normalizeParseStatus(state.status); const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null; @@ -199,7 +200,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string let initialState = await toSnapshotState(row, requestedOpId); if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') { - const state = parseDocumentParseState(row.parseState); + const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); const created = await backfillPendingPdfParseOperation({ documentId: id, userId: row.userId, diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 650760c..11158ea 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -20,8 +20,10 @@ import { putParsedDocumentBlob, } from '@/lib/server/documents/blobstore'; import { + normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, + resolveParsedPdfParserVersion, stringifyDocumentParseState, } from '@/lib/server/documents/parse-state'; import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill'; @@ -156,13 +158,6 @@ async function finalizeFromWorkerState(input: { }); } - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState(workerParseState), - parsedJsonKey, - }); - const json = await getParsedDocumentBlobByKey(parsedJsonKey); let parsedDoc: ParsedPdfDocument | null = null; try { @@ -171,6 +166,16 @@ async function finalizeFromWorkerState(input: { parsedDoc = null; } + await writeParseRowState({ + documentId: input.row.id, + userId: input.row.userId, + parseState: stringifyDocumentParseState({ + ...workerParseState, + parserVersion: resolveParsedPdfParserVersion(parsedDoc), + }), + parsedJsonKey, + }); + if (!hasAnyParsedBlocks(parsedDoc)) { input.logger.warn({ event: 'documents.parsed.no_blocks_in_output', @@ -246,7 +251,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }); } - let state = parseDocumentParseState(row.parseState); + let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); state = await healStaleDocumentParseState({ documentId: id, userId: row.userId, @@ -409,7 +414,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - let state = parseDocumentParseState(row.parseState); + let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); state = await healStaleDocumentParseState({ documentId: id, userId: row.userId, diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 7ba5623..10d1f23 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -19,6 +19,7 @@ import { isS3Configured } from '@/lib/server/storage/s3'; import { putDocumentBlob } from '@/lib/server/documents/blobstore'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); @@ -142,7 +143,12 @@ export async function POST(req: NextRequest) { size: pdfContent.length, lastModified, filePath: id, - parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }), + parseState: stringifyDocumentParseState({ + status: 'pending', + progress: null, + updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + }), parsedJsonKey: null, }) .onConflictDoUpdate({ @@ -153,7 +159,12 @@ export async function POST(req: NextRequest) { size: pdfContent.length, lastModified, filePath: id, - parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }), + parseState: stringifyDocumentParseState({ + status: 'pending', + progress: null, + updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + }), parsedJsonKey: null, }, }); diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 903ab96..7ea9f40 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -17,12 +17,15 @@ import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/ import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; import { + normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, stringifyDocumentParseState, } from '@/lib/server/documents/parse-state'; +import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { BaseDocument, DocumentType } from '@/types/documents'; export const dynamic = 'force-dynamic'; @@ -100,6 +103,9 @@ export async function POST(req: NextRequest) { for (const doc of documentsData) { let headSize = doc.size; + const reusableParsedPdf = doc.type === 'pdf' + ? await findReusableParsedPdfResult(doc.id) + : null; // Retry HEAD check to handle S3 read-after-write propagation delays. // The client uploads bytes directly to S3 via presigned URL, then @@ -146,9 +152,13 @@ export async function POST(req: NextRequest) { lastModified: doc.lastModified, filePath: doc.id, parseState: doc.type === 'pdf' - ? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }) + ? stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, + ) : null, - parsedJsonKey: null, + parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, }) .onConflictDoUpdate({ target: [documents.id, documents.userId], @@ -159,9 +169,13 @@ export async function POST(req: NextRequest) { lastModified: doc.lastModified, filePath: doc.id, parseState: doc.type === 'pdf' - ? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }) + ? stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, + ) : null, - parsedJsonKey: null, + parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, }, }); @@ -191,7 +205,7 @@ export async function POST(req: NextRequest) { }, 'Failed to enqueue document preview'); }); - if (doc.type === 'pdf') { + if (doc.type === 'pdf' && !reusableParsedPdf) { // Account for upload-driven parse load in the same ledger the explicit // re-parse limiter reads. We record (not reject) here so a legitimate // bulk upload always parses; the recorded load still throttles @@ -260,13 +274,16 @@ export async function GET(req: NextRequest) { const results: BaseDocument[] = rows.map((doc) => { const type = normalizeDocumentType(doc.type, doc.name); + const parseState = type === 'pdf' + ? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState)) + : null; return { id: doc.id, name: doc.name, size: Number(doc.size), lastModified: Number(doc.lastModified), type, - parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null, + parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null, parsedJsonKey: doc.parsedJsonKey, scope: 'user', }; diff --git a/src/components/player/TTSPlayer.tsx b/src/components/player/TTSPlayer.tsx index 1badd9b..a93ddbd 100644 --- a/src/components/player/TTSPlayer.tsx +++ b/src/components/player/TTSPlayer.tsx @@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl'; import { Navigator } from '@/components/player/Navigator'; import { IconButton } from '@/components/ui'; -export default function TTSPlayer({ currentPage, numPages }: { +export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: { currentPage?: number; numPages?: number | undefined; + isPlaybackReady?: boolean; }) { const { isPlaying, @@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: { {isProcessing ? : } @@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: { - {isPlaying ? : } + {!isPlaying && !isPlaybackReady + ? + : (isPlaying ? : )} {isProcessing ? : } diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index f1e0e9e..b346baf 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const [currentWordIndex, setCurrentWordIndex] = useState(null); const isPlayingRef = useRef(false); const pauseEpochRef = useRef(0); + const pendingPlaybackStartRef = useRef(false); const sentencesRef = useRef([]); const currentIndexRef = useRef(0); const plannedSegmentsByLocationRef = useRef>(new Map()); @@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const recordManualPause = useCallback(() => { // Cancel any queued auto-resume intent and mark an explicit user pause. resumeAfterLocationChangeRef.current = false; + pendingPlaybackStartRef.current = false; pauseEpochRef.current += 1; }, []); @@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * Used for external control of playback state */ const pause = useCallback(() => { + pendingPlaybackStartRef.current = false; recordManualPause(); clearPendingEpubJump(); abortPendingTtsRequests(true); @@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const shouldPause = normalizedOptions.shouldPause ?? false; const pauseEpochAtStart = pauseEpochRef.current; const pendingAutoResume = resumeAfterLocationChangeRef.current; - const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume); - if (shouldPause || pendingAutoResume) { - resumeAfterLocationChangeRef.current = false; - } + const pendingPlaybackStart = pendingPlaybackStartRef.current; + const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart); // Keep track of previous state and pause playback invalidatePlaybackRun(); @@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement try { if (newSentences.length === 0) { console.warn('No sentences found in text'); + if (shouldPause || pendingAutoResume) { + resumeAfterLocationChangeRef.current = false; + } + if (pendingPlaybackStart) { + pendingPlaybackStartRef.current = false; + } setIsProcessing(false); return; } + if (shouldPause || pendingAutoResume) { + resumeAfterLocationChangeRef.current = false; + } + if (pendingPlaybackStart) { + pendingPlaybackStartRef.current = false; + } + if (!isEPUB && typeof resolvedLocation === 'number') { const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || ''); if (firstFingerprint) { @@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Ensure audio is unlocked while we're still in the click/tap handler. unlockPlaybackOnUserGesture(); + const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]); + if (!activeHowl && !hasPreparedSentence) { + pendingPlaybackStartRef.current = true; + setIsProcessing(true); + return; + } + // Resume current sentence if we already have a paused Howl. if (activeHowl) { applyPlaybackRateToHowl(activeHowl); @@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setIsPlaying(true); }, [ activeHowl, + currentIndex, applyPlaybackRateToHowl, isPlaying, pauseActiveHowl, + playbackSegments, recordManualPause, clearPendingEpubJump, abortPendingTtsRequests, + sentences, unlockPlaybackOnUserGesture, ]); @@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement abortAudio(); clearWarmAudioCache(); playbackInFlightRef.current = false; + pendingPlaybackStartRef.current = false; pendingJumpTargetRef.current = null; clearPendingEpubJump(); bumpEpubPreloadGeneration(); diff --git a/src/lib/server/compute/worker-parse-state.ts b/src/lib/server/compute/worker-parse-state.ts index cce75a4..6798a22 100644 --- a/src/lib/server/compute/worker-parse-state.ts +++ b/src/lib/server/compute/worker-parse-state.ts @@ -1,3 +1,4 @@ +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; import type { DocumentParseState } from '@/lib/server/documents/parse-state'; @@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState( ? parseProgress : null, updatedAt: nowMs, + parserVersion: PDF_PARSER_VERSION, ...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}), ...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}), ...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}), diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts index 6f94501..cdc79cc 100644 --- a/src/lib/server/compute/worker.ts +++ b/src/lib/server/compute/worker.ts @@ -1,7 +1,7 @@ import { createHash, randomUUID } from 'node:crypto'; import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; import { parseSseEventId, parseSsePayload } from '@openreader/compute-core'; -import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core'; +import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { logDegraded, logServerError } from '@/lib/server/errors/logging'; import type { @@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string { return [ 'pdf_layout', 'v1', + PDF_PARSER_VERSION, input.documentId, input.namespace ?? '', input.documentObjectKey ?? '', diff --git a/src/lib/server/documents/parse-state-healing.ts b/src/lib/server/documents/parse-state-healing.ts index 82ca5dc..d8fcbd4 100644 --- a/src/lib/server/documents/parse-state-healing.ts +++ b/src/lib/server/documents/parse-state-healing.ts @@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: { error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`, ...(input.state.opId ? { opId: input.state.opId } : {}), ...(input.state.jobId ? { jobId: input.state.jobId } : {}), + ...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}), })); await db diff --git a/src/lib/server/documents/parse-state.ts b/src/lib/server/documents/parse-state.ts index 431e20c..22a94ad 100644 --- a/src/lib/server/documents/parse-state.ts +++ b/src/lib/server/documents/parse-state.ts @@ -1,4 +1,5 @@ -import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; export interface DocumentParseState { status: PdfParseStatus; @@ -7,6 +8,7 @@ export interface DocumentParseState { error?: string | null; opId?: string; jobId?: string; + parserVersion?: string; } export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' { @@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars return 'pending'; } +export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean { + return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION; +} + +export function normalizeDocumentParseStateForCurrentParserVersion( + state: DocumentParseState, + nowMs = Date.now(), +): DocumentParseState { + const status = normalizeParseStatus(state.status); + if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) { + return state; + } + + return { + status: 'pending', + progress: null, + updatedAt: nowMs, + }; +} + +export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string { + const parserVersion = parsed?.parserVersion?.trim(); + return parserVersion || PDF_PARSER_VERSION; +} + function normalizeProgress(progress: unknown): PdfParseProgress | null { if (!progress || typeof progress !== 'object') return null; const rec = progress as Record; @@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat const error = typeof parsed.error === 'string' ? parsed.error : null; const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined; const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined; + const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined; return { status, progress, @@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat ...(error ? { error } : {}), ...(opId ? { opId } : {}), ...(jobId ? { jobId } : {}), + ...(parserVersion ? { parserVersion } : {}), }; } catch { return { status: 'pending', progress: null }; @@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string { ...(state.error ? { error: state.error } : {}), ...(state.opId ? { opId: state.opId } : {}), ...(state.jobId ? { jobId: state.jobId } : {}), + ...(state.parserVersion ? { parserVersion: state.parserVersion } : {}), }); } diff --git a/src/lib/server/documents/parsed-pdf-reuse.ts b/src/lib/server/documents/parsed-pdf-reuse.ts new file mode 100644 index 0000000..1bbede4 --- /dev/null +++ b/src/lib/server/documents/parsed-pdf-reuse.ts @@ -0,0 +1,33 @@ +import { eq } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { + normalizeDocumentParseStateForCurrentParserVersion, + parseDocumentParseState, +} from '@/lib/server/documents/parse-state'; + +type ReadyParsedPdfResult = { + parsedJsonKey: string; +}; + +export async function findReusableParsedPdfResult( + documentId: string, +): Promise { + const rows = await db + .select({ + parseState: documents.parseState, + parsedJsonKey: documents.parsedJsonKey, + }) + .from(documents) + .where(eq(documents.id, documentId)); + + for (const row of rows) { + const parsedJsonKey = row.parsedJsonKey?.trim(); + if (!parsedJsonKey) continue; + const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); + if (parseState.status !== 'ready') continue; + return { parsedJsonKey }; + } + + return null; +} diff --git a/src/lib/server/jobs/user-pdf-layout-job.ts b/src/lib/server/jobs/user-pdf-layout-job.ts index 341fd55..531dfd0 100644 --- a/src/lib/server/jobs/user-pdf-layout-job.ts +++ b/src/lib/server/jobs/user-pdf-layout-job.ts @@ -1,11 +1,14 @@ import { and, eq, inArray, isNull } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; -import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; +import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; +import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded, logServerError } from '@/lib/server/errors/logging'; import { + normalizeDocumentParseStateForCurrentParserVersion, parseDocumentParseState, + resolveParsedPdfParserVersion, stringifyDocumentParseState, type DocumentParseState, } from '@/lib/server/documents/parse-state'; @@ -13,11 +16,13 @@ import { getCompute } from '@/lib/server/compute'; import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache'; import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy'; import type { + ParsedPdfDocument, PdfLayoutJobBase, PdfLayoutJobResult, PdfLayoutProgress, WorkerOperationState, } from '@openreader/compute-core/api-contracts'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; type UserPdfLayoutJobRequest = PdfLayoutJobBase & { userId: string; @@ -41,7 +46,7 @@ function keyFor(input: UserPdfLayoutJobRequest): string { if (forceToken) { return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`; } - return `shared:${input.documentId}:${input.namespace || ''}`; + return `shared:${input.documentId}`; } function sleep(ms: number): Promise { @@ -84,7 +89,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise { + if (parsed) return parsed; + + try { + const json = await getParsedDocumentBlobByKey(parsedJsonKey); + return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument; + } catch { + return null; + } +} + async function updateParseStateForUsers(input: { documentId: string; userIds: string[]; @@ -125,6 +144,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS; while (Date.now() < deadline) { + const reusableParsed = await findReusableParsedPdfResult(input.documentId); + if (reusableParsed) { + const readyState: DocumentParseState = { + status: 'ready', + progress: null, + updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + }; + await updateParseStateForUsers({ + documentId: input.documentId, + userIds: [input.userId], + parseState: stringifyDocumentParseState(readyState), + parsedJsonKey: reusableParsed.parsedJsonKey, + }); + return; + } + const rows = await loadScopedRows(input); const ready = rows.find(isReadyRow); if (ready) { @@ -132,6 +168,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise status: 'ready', progress: null, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, }; await updateParseStateForUsers({ documentId: input.documentId, @@ -142,7 +179,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise return; } - const statuses = rows.map((row) => parseDocumentParseState(row.parseState)); + const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState))); const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running'); const failedState = statuses.find((state) => state.status === 'failed'); if (failedState && !hasInFlight) { @@ -151,6 +188,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise progress: null, updatedAt: Date.now(), ...(failedState.error ? { error: failedState.error } : {}), + ...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}), }; await updateParseStateForUsers({ documentId: input.documentId, @@ -181,12 +219,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise // Non-force jobs can short-circuit by reusing existing ready output from // any row in the same document scope. if (!input.forceToken?.trim()) { + const reusableParsed = await findReusableParsedPdfResult(input.documentId); + if (reusableParsed) { + const readyState: DocumentParseState = { + status: 'ready', + progress: null, + updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + }; + await updateParseStateForUsers({ + documentId: input.documentId, + userIds: [input.userId], + parseState: stringifyDocumentParseState(readyState), + parsedJsonKey: reusableParsed.parsedJsonKey, + }); + return; + } + const ready = scopedRows.find(isReadyRow); if (ready) { const readyState: DocumentParseState = { status: 'ready', progress: null, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, }; await updateParseStateForUsers({ documentId: input.documentId, @@ -205,6 +261,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise status: 'running', progress: null, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, }; const runningState = stringifyDocumentParseState(runningStateData); @@ -279,6 +336,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise status: mappedStatus, progress: snapshot.progress ?? null, updatedAt: now, + parserVersion: PDF_PARSER_VERSION, ...(activeOpId ? { opId: activeOpId } : {}), ...(activeJobId ? { jobId: activeJobId } : {}), }; @@ -305,6 +363,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise phase: progress.phase, }, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, ...(activeOpId ? { opId: activeOpId } : {}), ...(activeJobId ? { jobId: activeJobId } : {}), }; @@ -327,12 +386,15 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace); } + const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey); + const finalScopedRows = await loadScopedRows(input); const finalUserIds = userIdsFromRows(finalScopedRows); const readyState: DocumentParseState = { status: 'ready', progress: null, updatedAt: Date.now(), + parserVersion: resolveParsedPdfParserVersion(parsedDoc), }; const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds; await updateParseStateForUsers({ @@ -385,6 +447,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise progress: null, updatedAt: Date.now(), error: message, + parserVersion: PDF_PARSER_VERSION, }; const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId]; await updateParseStateForUsers({ diff --git a/tests/helpers.ts b/tests/helpers.ts index cfc9472..3a7835d 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -188,10 +188,11 @@ async function dismissOnboardingModals(page: Page): Promise { * Wait for the play button to be clickable and click it */ export async function waitAndClickPlay(page: Page) { - // Wait for play button selector without disabled attribute - await expect(page.getByRole('button', { name: 'Play' })).toBeVisible(); + const playButton = page.getByRole('button', { name: 'Play' }); + await expect(playButton).toBeVisible(); + await expect(playButton).toBeEnabled({ timeout: 15000 }); // Play the TTS by clicking the button - await page.getByRole('button', { name: 'Play' }).click(); + await playButton.click(); // Use resilient processing transition helper (tolerates fast completion) await expectProcessingTransition(page); } diff --git a/tests/navigation.spec.ts b/tests/navigation.spec.ts index b642b20..ea193f8 100644 --- a/tests/navigation.spec.ts +++ b/tests/navigation.spec.ts @@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => { await page.waitForTimeout(750); // After resize, playback should have paused (Play button visible) - await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 }); + const playButton = page.getByRole('button', { name: 'Play' }); + await expect(playButton).toBeVisible({ timeout: 15000 }); + await expect(playButton).toBeEnabled({ timeout: 15000 }); // Resume playback and ensure processing -> playing - await page.getByRole('button', { name: 'Play' }).click(); + await playButton.click(); await expectProcessingTransition(page); }); }); diff --git a/tests/unit/pdf-op-key.vitest.spec.ts b/tests/unit/pdf-op-key.vitest.spec.ts new file mode 100644 index 0000000..fbe746e --- /dev/null +++ b/tests/unit/pdf-op-key.vitest.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from 'vitest'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; + +describe('pdf worker op key', () => { + test('includes parser version in the reuse boundary', () => { + expect(buildPdfOpKey({ + documentId: 'doc-1', + namespace: null, + documentObjectKey: 'documents/doc-1.pdf', + })).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`); + }); +}); diff --git a/tests/unit/pdf-parse-state.vitest.spec.ts b/tests/unit/pdf-parse-state.vitest.spec.ts new file mode 100644 index 0000000..aa3384b --- /dev/null +++ b/tests/unit/pdf-parse-state.vitest.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from 'vitest'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import { + normalizeDocumentParseStateForCurrentParserVersion, + parseDocumentParseState, + stringifyDocumentParseState, +} from '../../src/lib/server/documents/parse-state'; + +describe('document parse state parser-version handling', () => { + test('preserves parserVersion through stringify/parse', () => { + const state = parseDocumentParseState(stringifyDocumentParseState({ + status: 'ready', + progress: null, + updatedAt: 1234, + parserVersion: PDF_PARSER_VERSION, + })); + + expect(state).toEqual({ + status: 'ready', + progress: null, + updatedAt: 1234, + parserVersion: PDF_PARSER_VERSION, + }); + }); + + test('invalidates ready and inflight states from older parser versions', () => { + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'ready', + progress: null, + updatedAt: 1111, + parserVersion: 'old-parser', + opId: 'op-old', + }, 2222)).toEqual({ + status: 'pending', + progress: null, + updatedAt: 2222, + }); + + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'running', + progress: { + totalPages: 10, + pagesParsed: 3, + currentPage: 4, + phase: 'infer', + }, + updatedAt: 1111, + parserVersion: 'old-parser', + opId: 'op-old', + }, 3333)).toEqual({ + status: 'pending', + progress: null, + updatedAt: 3333, + }); + }); + + test('keeps failed states and current-version ready states intact', () => { + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'failed', + progress: null, + updatedAt: 1111, + error: 'no text layer', + parserVersion: 'old-parser', + })).toEqual({ + status: 'failed', + progress: null, + updatedAt: 1111, + error: 'no text layer', + parserVersion: 'old-parser', + }); + + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'ready', + progress: null, + updatedAt: 1111, + parserVersion: PDF_PARSER_VERSION, + })).toEqual({ + status: 'ready', + progress: null, + updatedAt: 1111, + parserVersion: PDF_PARSER_VERSION, + }); + }); +}); diff --git a/tests/unit/worker-parse-state.vitest.spec.ts b/tests/unit/worker-parse-state.vitest.spec.ts index dfdd37e..91563b3 100644 --- a/tests/unit/worker-parse-state.vitest.spec.ts +++ b/tests/unit/worker-parse-state.vitest.spec.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from 'vitest'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; import { documentParseStateFromWorkerState, @@ -11,7 +12,7 @@ function makeWorkerState( ): WorkerOperationState { return { opId: 'op-123', - opKey: 'pdf_layout|v1|doc-1', + opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`, kind: 'pdf_layout', jobId: 'job-123', status: 'queued', @@ -41,6 +42,7 @@ describe('worker parse state mapping', () => { status: 'pending', progress: null, updatedAt: 1234, + parserVersion: PDF_PARSER_VERSION, opId: 'op-123', jobId: 'job-123', }); @@ -66,6 +68,7 @@ describe('worker parse state mapping', () => { phase: 'infer', }, updatedAt: 5678, + parserVersion: PDF_PARSER_VERSION, opId: 'op-123', jobId: 'job-123', }); @@ -84,6 +87,7 @@ describe('worker parse state mapping', () => { status: 'failed', progress: null, updatedAt: 9999, + parserVersion: PDF_PARSER_VERSION, opId: 'op-123', jobId: 'job-123', error: 'layout model crashed', From 58ad6212c924351f3a2737879cb49028fc18baa7 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 3 Jun 2026 19:32:46 -0600 Subject: [PATCH 2/9] fix(html): reset TTS text and playback state on document changes Ensure TTS text and playback readiness state are properly reset when the current document data or name changes, or when clearing or setting the current document. Update effect dependencies and logic to prevent stale playback state and improve user experience when switching or removing documents. --- src/app/(app)/html/[id]/useHtmlDocument.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/app/(app)/html/[id]/useHtmlDocument.ts b/src/app/(app)/html/[id]/useHtmlDocument.ts index 23bcc94..3b91553 100644 --- a/src/app/(app)/html/[id]/useHtmlDocument.ts +++ b/src/app/(app)/html/[id]/useHtmlDocument.ts @@ -94,14 +94,18 @@ export function useHtmlDocument(): HtmlDocumentState { const lastFedDocRef = useRef(null); useEffect(() => { if (currDocData === undefined) { + lastFedDocRef.current = null; + setTTSText(''); setIsPlaybackReady(false); return; } if (!currDocText) { + lastFedDocRef.current = null; + setTTSText(''); setIsPlaybackReady(true); return; } - const key = `${currDocData ?? ''}::${currDocText.length}`; + const key = `${currDocName ?? ''}::${currDocData ?? ''}::${currDocText.length}`; if (lastFedDocRef.current === key) { setIsPlaybackReady(true); return; @@ -110,19 +114,22 @@ export function useHtmlDocument(): HtmlDocumentState { lastFedDocRef.current = key; setTTSText(currDocText); setIsPlaybackReady(true); - }, [currDocText, currDocData, setTTSText]); + }, [currDocName, currDocText, currDocData, setTTSText]); const clearCurrDoc = useCallback(() => { setCurrDocData(undefined); setCurrDocName(undefined); setIsPlaybackReady(false); lastFedDocRef.current = null; + setTTSText(''); stop(); - }, [stop]); + }, [setTTSText, stop]); const setCurrentDocument = useCallback(async (id: string): Promise => { try { setIsPlaybackReady(false); + lastFedDocRef.current = null; + setTTSText(''); const meta = await getDocumentMetadata(id); if (!meta) { console.error('Document not found on server'); @@ -141,7 +148,7 @@ export function useHtmlDocument(): HtmlDocumentState { console.error('Failed to get HTML document:', error); clearCurrDoc(); } - }, [clearCurrDoc]); + }, [clearCurrDoc, setTTSText]); const audiobookAdapter = useMemo( () => From 6c86720926f991a730e7d95db5084bb9ec76cb48 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 3 Jun 2026 20:14:18 -0600 Subject: [PATCH 3/9] feat(upload): implement presigned token-based upload with finalize step Transition document upload flow to use presigned token-based temporary uploads. Add finalize endpoint to register uploaded documents after blob transfer, decoupling document ID assignment from upload initiation. Update client and server logic to operate on tokens, improve security, and enable canonical ID reuse for identical uploads. Update tests to verify deduplication behavior. --- .../documents/blob/upload/fallback/route.ts | 15 +- .../documents/blob/upload/finalize/route.ts | 224 ++++++++++++++++ .../documents/blob/upload/presign/route.ts | 21 +- src/app/api/documents/route.ts | 200 +------------- src/lib/client/api/documents.ts | 49 ++-- src/lib/client/dexie.ts | 32 +-- src/lib/server/documents/blobstore.ts | 253 ++++++++++++++++++ src/lib/server/documents/register-upload.ts | 104 +++++++ tests/upload.spec.ts | 23 ++ 9 files changed, 665 insertions(+), 256 deletions(-) create mode 100644 src/app/api/documents/blob/upload/finalize/route.ts create mode 100644 src/lib/server/documents/register-upload.ts diff --git a/src/app/api/documents/blob/upload/fallback/route.ts b/src/app/api/documents/blob/upload/fallback/route.ts index 7fc8208..f6d7f15 100644 --- a/src/app/api/documents/blob/upload/fallback/route.ts +++ b/src/app/api/documents/blob/upload/fallback/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore'; +import { isValidTempUploadToken, putTempDocumentBlob } from '@/lib/server/documents/blobstore'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; @@ -26,11 +26,12 @@ export async function PUT(req: NextRequest) { const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); const url = new URL(req.url); - const id = (url.searchParams.get('id') || '').trim().toLowerCase(); - if (!isValidDocumentId(id)) { - return NextResponse.json({ error: 'Invalid document id' }, { status: 400 }); + const token = (url.searchParams.get('token') || '').trim().toLowerCase(); + if (!isValidTempUploadToken(token)) { + return NextResponse.json({ error: 'Invalid upload token' }, { status: 400 }); } const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream'; @@ -86,7 +87,7 @@ export async function PUT(req: NextRequest) { const namespace = getOpenReaderTestNamespace(req.headers); try { - await putDocumentBlob(id, body, contentType, namespace); + await putTempDocumentBlob(token, ctxOrRes.userId, body, contentType, namespace); } catch (error) { if (!isPreconditionFailed(error)) { throw error; @@ -97,12 +98,12 @@ export async function PUT(req: NextRequest) { event: 'documents.blob.upload.fallback.proxy_used', degraded: true, fallbackPath: 'upload_proxy', - documentId: id, + uploadToken: token, contentType, bytes: body.byteLength, }, 'Document upload fallback proxy used'); - return NextResponse.json({ success: true, id }); + return NextResponse.json({ success: true, token }); } catch (error) { serverLogger.error({ event: 'documents.blob.upload.fallback.failed', diff --git a/src/app/api/documents/blob/upload/finalize/route.ts b/src/app/api/documents/blob/upload/finalize/route.ts new file mode 100644 index 0000000..ea82ce4 --- /dev/null +++ b/src/app/api/documents/blob/upload/finalize/route.ts @@ -0,0 +1,224 @@ +import { createHash } from 'node:crypto'; +import { NextRequest, NextResponse } from 'next/server'; +import { requireAuthContext } from '@/lib/server/auth/auth'; +import { + TEMP_DOCUMENT_UPLOAD_TTL_MS, + copyTempDocumentBlobToDocument, + deleteTempDocumentUpload, + getTempDocumentBlob, + getTempDocumentFinalizeReceipt, + headDocumentBlob, + headTempDocumentBlob, + isMissingBlobError, + isValidTempUploadToken, + putTempDocumentFinalizeReceipt, +} from '@/lib/server/documents/blobstore'; +import { registerUploadedDocument } from '@/lib/server/documents/register-upload'; +import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils'; +import { errorResponse } from '@/lib/server/errors/next-response'; +import { errorToLog, serverLogger } from '@/lib/server/logger'; +import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import type { BaseDocument, DocumentType } from '@/types/documents'; + +export const dynamic = 'force-dynamic'; + +type FinalizeUpload = { + token: string; + name: string; + type: DocumentType; + lastModified: number; +}; + +type FinalizeReceipt = { + stored: BaseDocument; +}; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType { + if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') { + return rawType; + } + return toDocumentTypeFromName(safeName); +} + +function normalizeLastModified(value: unknown): number { + return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now(); +} + +function parseFinalizePayload(body: unknown): FinalizeUpload[] { + if (!body || typeof body !== 'object') return []; + const rawUploads = (body as { uploads?: unknown }).uploads; + if (!Array.isArray(rawUploads)) return []; + + const uploads: FinalizeUpload[] = []; + for (const rawUpload of rawUploads) { + if (!rawUpload || typeof rawUpload !== 'object') continue; + const rec = rawUpload as Record; + const token = typeof rec.token === 'string' ? rec.token.trim().toLowerCase() : ''; + if (!isValidTempUploadToken(token)) continue; + const fallbackName = `upload-${token}.txt`; + const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName); + uploads.push({ + token, + name, + type: normalizeDocumentType(rec.type, name), + lastModified: normalizeLastModified(rec.lastModified), + }); + } + return uploads; +} + +async function loadTempUpload(input: { + token: string; + userId: string; + namespace: string | null; +}): Promise<{ contentType: string; size: number; lastModified: number; body: Buffer }> { + const RETRIES = 3; + const RETRY_DELAY_MS = 500; + + let lastError: unknown = null; + for (let attempt = 0; attempt < RETRIES; attempt += 1) { + try { + const head = await headTempDocumentBlob(input.token, input.userId, input.namespace); + const body = await getTempDocumentBlob(input.token, input.userId, input.namespace); + return { + contentType: head.contentType ?? 'application/octet-stream', + size: head.contentLength > 0 ? head.contentLength : body.byteLength, + lastModified: head.lastModified ?? Date.now(), + body, + }; + } catch (error) { + lastError = error; + if (isMissingBlobError(error) && attempt < RETRIES - 1) { + await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS)); + continue; + } + throw error; + } + } + + throw lastError instanceof Error ? lastError : new Error('Temporary upload is unavailable'); +} + +async function finalizeOne(input: { + upload: FinalizeUpload; + userId: string; + namespace: string | null; +}): Promise { + const existingReceipt = await getTempDocumentFinalizeReceipt( + input.upload.token, + input.userId, + input.namespace, + ); + if (existingReceipt?.stored) { + return existingReceipt.stored; + } + + const temp = await loadTempUpload({ + token: input.upload.token, + userId: input.userId, + namespace: input.namespace, + }); + + if (Date.now() - temp.lastModified > TEMP_DOCUMENT_UPLOAD_TTL_MS) { + await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch(() => undefined); + throw new Error('Temporary upload expired before finalize'); + } + + const documentId = createHash('sha256').update(temp.body).digest('hex'); + + try { + await headDocumentBlob(documentId, input.namespace); + } catch (error) { + if (!isMissingBlobError(error)) throw error; + await copyTempDocumentBlobToDocument( + input.upload.token, + input.userId, + documentId, + input.namespace, + temp.contentType, + ); + } + + const canonicalHead = await headDocumentBlob(documentId, input.namespace); + const stored = await registerUploadedDocument({ + documentId, + userId: input.userId, + namespace: input.namespace, + name: input.upload.name, + type: input.upload.type, + size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : temp.size, + lastModified: input.upload.lastModified, + }); + + await putTempDocumentFinalizeReceipt( + input.upload.token, + input.userId, + input.namespace, + Buffer.from(JSON.stringify({ stored }), 'utf8'), + ); + + await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch((error) => { + serverLogger.warn({ + event: 'documents.blob.upload.finalize.temp_delete_failed', + degraded: true, + fallbackPath: 'leave_temp_upload', + documentId, + token: input.upload.token, + error: errorToLog(error), + }, 'Failed to delete temp upload after finalize'); + }); + + return stored; +} + +export async function POST(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + const userId = ctxOrRes.userId; + + const namespace = getOpenReaderTestNamespace(req.headers); + const uploads = parseFinalizePayload(await req.json().catch(() => null)); + if (uploads.length === 0) { + return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 }); + } + + const stored: BaseDocument[] = []; + for (const upload of uploads) { + stored.push(await finalizeOne({ + upload, + userId, + namespace, + })); + } + + return NextResponse.json({ stored }); + } catch (error) { + if (error instanceof Error && error.message === 'Temporary upload expired before finalize') { + return NextResponse.json({ error: error.message }, { status: 410 }); + } + if (isMissingBlobError(error)) { + return NextResponse.json({ error: 'Temporary upload missing. Upload bytes again and retry finalize.' }, { status: 409 }); + } + + serverLogger.error({ + event: 'documents.blob.upload.finalize.failed', + error: errorToLog(error), + }, 'Failed to finalize uploaded documents'); + return errorResponse(error, { + apiErrorMessage: 'Failed to finalize uploaded documents', + normalize: { code: 'DOCUMENTS_BLOB_UPLOAD_FINALIZE_FAILED', errorClass: 'storage' }, + }); + } +} diff --git a/src/app/api/documents/blob/upload/presign/route.ts b/src/app/api/documents/blob/upload/presign/route.ts index 6d91ee4..183074c 100644 --- a/src/app/api/documents/blob/upload/presign/route.ts +++ b/src/app/api/documents/blob/upload/presign/route.ts @@ -1,6 +1,11 @@ +import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore'; +import { + TEMP_DOCUMENT_UPLOAD_TTL_MS, + deleteExpiredTempDocumentUploads, + presignTempPut, +} from '@/lib/server/documents/blobstore'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; @@ -10,7 +15,6 @@ import { errorResponse } from '@/lib/server/errors/next-response'; export const dynamic = 'force-dynamic'; type PresignUpload = { - id: string; contentType: string; size: number; }; @@ -24,14 +28,12 @@ function parseUploads(body: unknown): PresignUpload[] { for (const raw of rawUploads) { if (!raw || typeof raw !== 'object') continue; const rec = raw as Record; - const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : ''; - if (!isValidDocumentId(id)) continue; const contentType = typeof rec.contentType === 'string' && rec.contentType.trim() ? rec.contentType.trim() : 'application/octet-stream'; const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0; - uploads.push({ id, contentType, size }); + uploads.push({ contentType, size }); } return uploads; } @@ -47,6 +49,8 @@ export async function POST(req: NextRequest) { const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + const userId = ctxOrRes.userId; const body = await req.json().catch(() => null); const uploads = parseUploads(body); @@ -68,13 +72,16 @@ export async function POST(req: NextRequest) { } const namespace = getOpenReaderTestNamespace(req.headers); + await deleteExpiredTempDocumentUploads(userId, namespace, Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS) + .catch(() => undefined); const signed = await Promise.all( uploads.map(async (upload) => { - const res = await presignPut(upload.id, upload.contentType, namespace, { + const token = randomUUID(); + const res = await presignTempPut(token, userId, upload.contentType, namespace, { contentLength: upload.size, }); return { - id: upload.id, + token, url: res.url, headers: res.headers, }; diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 7ea9f40..8002f7a 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,43 +1,27 @@ -import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { and, count, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils'; +import { toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; import { cleanupDocumentPreviewArtifacts, deleteDocumentPreviewRows, - enqueueDocumentPreview, } from '@/lib/server/documents/previews'; -import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; -import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; -import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; +import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; import { normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, - stringifyDocumentParseState, } from '@/lib/server/documents/parse-state'; -import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { BaseDocument, DocumentType } from '@/types/documents'; export const dynamic = 'force-dynamic'; -type RegisterDocument = { - id: string; - name: string; - type: DocumentType; - size: number; - lastModified: number; -}; - function s3NotConfiguredResponse(): NextResponse { return NextResponse.json( { error: 'Documents storage is not configured. Set S3_* environment variables.' }, @@ -52,186 +36,6 @@ function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType return toDocumentTypeFromName(safeName); } -function normalizeLastModified(value: unknown): number { - return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now(); -} - -function parseDocumentPayload(body: unknown): RegisterDocument[] { - if (!body || typeof body !== 'object') return []; - const rawDocs = (body as { documents?: unknown }).documents; - if (!Array.isArray(rawDocs)) return []; - - const docs: RegisterDocument[] = []; - for (const rawDoc of rawDocs) { - if (!rawDoc || typeof rawDoc !== 'object') continue; - const rec = rawDoc as Record; - const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : ''; - if (!isValidDocumentId(id)) continue; - const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`; - const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName); - const type = normalizeDocumentType(rec.type, name); - const lastModified = normalizeLastModified(rec.lastModified); - const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0; - docs.push({ id, name, type, size, lastModified }); - } - return docs; -} - -export async function POST(req: NextRequest) { - try { - if (!isS3Configured()) return s3NotConfiguredResponse(); - - const ctxOrRes = await requireAuthContext(req); - if (ctxOrRes instanceof Response) return ctxOrRes; - if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - - const testNamespace = getOpenReaderTestNamespace(req.headers); - const storageUserId = ctxOrRes.userId; - - const body = await req.json().catch(() => null); - const documentsData = parseDocumentPayload(body); - if (documentsData.length === 0) { - return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 }); - } - - const stored: BaseDocument[] = []; - - // Resolve the parse rate-limit config once (only when a PDF is present). - const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf') - ? getPdfLayoutRateConfig(await getResolvedRuntimeConfig()) - : null; - - for (const doc of documentsData) { - let headSize = doc.size; - const reusableParsedPdf = doc.type === 'pdf' - ? await findReusableParsedPdfResult(doc.id) - : null; - - // Retry HEAD check to handle S3 read-after-write propagation delays. - // The client uploads bytes directly to S3 via presigned URL, then - // immediately calls this endpoint. On serverless platforms the HEAD - // request may reach S3 before the PUT is visible. - const HEAD_RETRIES = 3; - const HEAD_RETRY_DELAY_MS = 500; - let headError: unknown = null; - for (let attempt = 0; attempt < HEAD_RETRIES; attempt++) { - headError = null; - try { - const head = await headDocumentBlob(doc.id, testNamespace); - if (head.contentLength > 0) headSize = head.contentLength; - break; - } catch (error) { - if (isMissingBlobError(error)) { - headError = error; - if (attempt < HEAD_RETRIES - 1) { - await new Promise((r) => setTimeout(r, HEAD_RETRY_DELAY_MS)); - continue; - } - } else { - throw error; - } - } - } - if (headError && isMissingBlobError(headError)) { - return NextResponse.json( - { - error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`, - }, - { status: 409 }, - ); - } - - await db - .insert(documents) - .values({ - id: doc.id, - userId: storageUserId, - name: doc.name, - type: doc.type, - size: headSize, - lastModified: doc.lastModified, - filePath: doc.id, - parseState: doc.type === 'pdf' - ? stringifyDocumentParseState( - reusableParsedPdf - ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } - : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, - ) - : null, - parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, - }) - .onConflictDoUpdate({ - target: [documents.id, documents.userId], - set: { - name: doc.name, - type: doc.type, - size: headSize, - lastModified: doc.lastModified, - filePath: doc.id, - parseState: doc.type === 'pdf' - ? stringifyDocumentParseState( - reusableParsedPdf - ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } - : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, - ) - : null, - parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, - }, - }); - - stored.push({ - id: doc.id, - name: doc.name, - type: doc.type, - size: headSize, - lastModified: doc.lastModified, - scope: 'user', - }); - - await enqueueDocumentPreview( - { - id: doc.id, - type: doc.type, - lastModified: doc.lastModified, - }, - testNamespace, - ).catch((error) => { - serverLogger.warn({ - event: 'documents.preview.enqueue.failed', - degraded: true, - fallbackPath: 'skip_preview_enqueue', - documentId: doc.id, - error: errorToLog(error), - }, 'Failed to enqueue document preview'); - }); - - if (doc.type === 'pdf' && !reusableParsedPdf) { - // Account for upload-driven parse load in the same ledger the explicit - // re-parse limiter reads. We record (not reject) here so a legitimate - // bulk upload always parses; the recorded load still throttles - // subsequent loopable re-parse spam via /parsed. - await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] }); - enqueueParsePdfJob({ - documentId: doc.id, - userId: storageUserId, - namespace: testNamespace, - }); - } - } - - return NextResponse.json({ success: true, stored }); - } catch (error) { - serverLogger.error({ - event: 'documents.register.failed', - error: errorToLog(error), - }, 'Failed to register documents'); - return errorResponse(error, { - apiErrorMessage: 'Failed to register documents', - normalize: { code: 'DOCUMENTS_REGISTER_FAILED', errorClass: 'db' }, - }); - } -} - export async function GET(req: NextRequest) { try { if (!isS3Configured()) return s3NotConfiguredResponse(); diff --git a/src/lib/client/api/documents.ts b/src/lib/client/api/documents.ts index b943354..360914f 100644 --- a/src/lib/client/api/documents.ts +++ b/src/lib/client/api/documents.ts @@ -1,10 +1,8 @@ -import { sha256HexFromArrayBuffer } from '@/lib/client/sha256'; import type { BaseDocument, DocumentType } from '@/types/documents'; import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; import type { DocumentSettings } from '@/types/document-settings'; export type UploadSource = { - id: string; name: string; type: DocumentType; size: number; @@ -23,8 +21,12 @@ function toUploadBody(body: UploadSource['body']): BodyInit { return body as unknown as BodyInit; } -async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise { - const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, { +async function uploadDocumentSourceViaProxy( + source: UploadSource, + token: string, + options?: UploadOptions, +): Promise { + const res = await fetch(`/api/documents/blob/upload/fallback?token=${encodeURIComponent(token)}`, { method: 'PUT', headers: { 'Content-Type': source.contentType || 'application/octet-stream' }, body: toUploadBody(source.body), @@ -241,7 +243,6 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ uploads: sources.map((source) => ({ - id: source.id, contentType: source.contentType, size: source.size, })), @@ -255,14 +256,18 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U } const presigned = (await presignRes.json()) as { - uploads?: Array<{ id: string; url: string; headers?: Record }>; + uploads?: Array<{ token: string; url: string; headers?: Record }>; }; - const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload])); + const uploads = presigned.uploads || []; + if (uploads.length !== sources.length) { + throw new Error('Upload preparation returned an unexpected number of temp uploads'); + } - for (const source of sources) { - const upload = byId.get(source.id); - if (!upload?.url) { - throw new Error(`Missing presigned upload for document ${source.id}`); + for (let index = 0; index < sources.length; index += 1) { + const source = sources[index]; + const upload = uploads[index]; + if (!upload?.url || !upload.token) { + throw new Error(`Missing presigned upload for document ${source.name}`); } let putError: unknown = null; @@ -285,7 +290,7 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U } try { - await uploadDocumentSourceViaProxy(source, options); + await uploadDocumentSourceViaProxy(source, upload.token, options); } catch (proxyError) { const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error'; const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error'; @@ -293,27 +298,26 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U } } - const registerRes = await fetch('/api/documents', { + const finalizeRes = await fetch('/api/documents/blob/upload/finalize', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - documents: sources.map((source) => ({ - id: source.id, + uploads: sources.map((source, index) => ({ + token: uploads[index]?.token, name: source.name, type: source.type, - size: source.size, lastModified: source.lastModified, })), }), signal: options?.signal, }); - if (!registerRes.ok) { - const data = (await registerRes.json().catch(() => null)) as { error?: string } | null; - throw new Error(data?.error || 'Failed to register uploaded documents'); + if (!finalizeRes.ok) { + const data = (await finalizeRes.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || 'Failed to finalize uploaded documents'); } - const data = (await registerRes.json()) as { stored: BaseDocument[] }; + const data = (await finalizeRes.json()) as { stored: BaseDocument[] }; return data.stored || []; } @@ -322,13 +326,10 @@ export async function uploadDocuments(files: File[], options?: UploadOptions): P const sources: UploadSource[] = []; for (const file of files) { - const bytes = await file.arrayBuffer(); - const id = await sha256HexFromArrayBuffer(bytes); const type = documentTypeForName(file.name); - const name = file.name || `${id}.${type}`; + const name = file.name || `upload.${type}`; const contentType = file.type || mimeTypeForDoc({ name, type }); sources.push({ - id, name, type, size: file.size, diff --git a/src/lib/client/dexie.ts b/src/lib/client/dexie.ts index 40e99f6..19bc09b 100644 --- a/src/lib/client/dexie.ts +++ b/src/lib/client/dexie.ts @@ -971,11 +971,9 @@ export async function syncDocumentsToServer( for (const doc of pdfDocs) { const bytes = new Uint8Array(doc.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: doc.id, source: { - id, name: doc.name, type: 'pdf', size: bytes.byteLength, @@ -992,11 +990,9 @@ export async function syncDocumentsToServer( for (const doc of epubDocs) { const bytes = new Uint8Array(doc.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: doc.id, source: { - id, name: doc.name, type: 'epub', size: bytes.byteLength, @@ -1013,11 +1009,9 @@ export async function syncDocumentsToServer( for (const doc of htmlDocs) { const encoded = textEncoder.encode(doc.data); - const id = await sha256HexFromBytes(encoded); uploads.push({ oldId: doc.id, source: { - id, name: doc.name, type: 'html', size: encoded.byteLength, @@ -1036,11 +1030,13 @@ export async function syncDocumentsToServer( onProgress(50, 'Uploading to server...'); } - await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); + const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); - for (const entry of uploads) { - if (entry.oldId === entry.source.id) continue; - await applyDocumentIdMapping(entry.oldId, entry.source.id); + for (let index = 0; index < uploads.length; index += 1) { + const entry = uploads[index]; + const nextId = stored[index]?.id; + if (!nextId || entry.oldId === nextId) continue; + await applyDocumentIdMapping(entry.oldId, nextId); } if (onProgress) { @@ -1064,11 +1060,9 @@ export async function syncSelectedDocumentsToServer( const data = await getPdfDocument(doc.id); if (data) { const bytes = new Uint8Array(data.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: data.id, source: { - id, name: data.name, type: 'pdf', size: bytes.byteLength, @@ -1082,11 +1076,9 @@ export async function syncSelectedDocumentsToServer( const data = await getEpubDocument(doc.id); if (data) { const bytes = new Uint8Array(data.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: data.id, source: { - id, name: data.name, type: 'epub', size: bytes.byteLength, @@ -1100,11 +1092,9 @@ export async function syncSelectedDocumentsToServer( const data = await getHtmlDocument(doc.id); if (data) { const bytes = textEncoder.encode(data.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: data.id, source: { - id, name: data.name, type: 'html', size: bytes.byteLength, @@ -1121,11 +1111,13 @@ export async function syncSelectedDocumentsToServer( } if (onProgress) onProgress(50, 'Uploading to server...'); - await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); + const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); - for (const entry of uploads) { - if (entry.oldId === entry.source.id) continue; - await applyDocumentIdMapping(entry.oldId, entry.source.id); + for (let index = 0; index < uploads.length; index += 1) { + const entry = uploads[index]; + const nextId = stored[index]?.id; + if (!nextId || entry.oldId === nextId) continue; + await applyDocumentIdMapping(entry.oldId, nextId); } if (onProgress) { diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index fb84f00..b8fa988 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -1,4 +1,5 @@ import { + CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, @@ -11,6 +12,9 @@ import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const TEMP_UPLOAD_TOKEN_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const TEMP_UPLOAD_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/; +export const TEMP_DOCUMENT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000; function sanitizeNamespace(namespace: string | null): string | null { if (!namespace) return null; @@ -69,6 +73,17 @@ export function isValidDocumentId(id: string): boolean { return DOCUMENT_ID_REGEX.test(id); } +export function isValidTempUploadToken(token: string): boolean { + return TEMP_UPLOAD_TOKEN_REGEX.test(token); +} + +function sanitizeTempUploadUserId(userId: string): string { + if (!TEMP_UPLOAD_USER_ID_REGEX.test(userId)) { + throw new Error(`Invalid temp upload user id: ${userId}`); + } + return encodeURIComponent(userId); +} + export function documentKey(id: string, namespace: string | null): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); @@ -90,6 +105,27 @@ export function documentParsedKey(id: string, namespace: string | null): string return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; } +export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string { + const cfg = getS3Config(); + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + return `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}users/${sanitizeTempUploadUserId(userId)}/`; +} + +export function tempDocumentUploadKey(token: string, userId: string, namespace: string | null): string { + if (!isValidTempUploadToken(token)) { + throw new Error(`Invalid temp upload token: ${token}`); + } + return `${tempDocumentUploadPrefix(userId, namespace)}${token}.bin`; +} + +export function tempDocumentUploadReceiptKey(token: string, userId: string, namespace: string | null): string { + if (!isValidTempUploadToken(token)) { + throw new Error(`Invalid temp upload token: ${token}`); + } + return `${tempDocumentUploadPrefix(userId, namespace)}${token}.receipt.json`; +} + function legacyDocumentParsedKey(id: string, namespace: string | null): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); @@ -138,6 +174,40 @@ export async function presignPut( }; } +export async function presignTempPut( + token: string, + userId: string, + contentType: string, + namespace: string | null, + options?: { contentLength?: number }, +): Promise<{ url: string; headers: Record }> { + const cfg = getS3Config(); + const client = getS3Client(); + const key = tempDocumentUploadKey(token, userId, namespace); + const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; + const contentLength = + typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0 + ? Math.floor(options.contentLength) + : undefined; + + const command = new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + ContentType: normalizedType, + ServerSideEncryption: 'AES256', + ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), + }); + const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); + + return { + url, + headers: { + 'Content-Type': normalizedType, + 'x-amz-server-side-encryption': 'AES256', + }, + }; +} + export async function headDocumentBlob( id: string, namespace: string | null, @@ -153,6 +223,23 @@ export async function headDocumentBlob( }; } +export async function headTempDocumentBlob( + token: string, + userId: string, + namespace: string | null, +): Promise<{ contentLength: number; contentType: string | null; eTag: string | null; lastModified: number | null }> { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadKey(token, userId, namespace); + const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key })); + return { + contentLength: Number(res.ContentLength ?? 0), + contentType: res.ContentType ?? null, + eTag: res.ETag ?? null, + lastModified: res.LastModified?.getTime() ?? null, + }; +} + export async function getDocumentRange( id: string, start: number, @@ -185,6 +272,23 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro return bodyToBuffer(res.Body); } +export async function getTempDocumentBlob( + token: string, + userId: string, + namespace: string | null, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadKey(token, userId, namespace); + const res = await client.send( + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + }), + ); + return bodyToBuffer(res.Body); +} + export async function getDocumentBlobStream(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -198,6 +302,29 @@ export async function getDocumentBlobStream(id: string, namespace: string | null return res.Body as DocumentBlobBody; } +export async function getTempDocumentFinalizeReceipt( + token: string, + userId: string, + namespace: string | null, +): Promise { + try { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadReceiptKey(token, userId, namespace); + const res = await client.send( + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + }), + ); + const body = await bodyToBuffer(res.Body); + return JSON.parse(body.toString('utf8')) as T; + } catch (error) { + if (isMissingBlobError(error)) return null; + throw error; + } +} + export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -241,6 +368,26 @@ export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: return key; } +export async function putTempDocumentFinalizeReceipt( + token: string, + userId: string, + namespace: string | null, + body: Buffer, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadReceiptKey(token, userId, namespace); + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + }), + ); +} + export async function presignGet( id: string, namespace: string | null, @@ -264,6 +411,7 @@ export async function putDocumentBlob( body: Buffer, contentType: string, namespace: string | null, + options?: { ifNoneMatch?: boolean }, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -275,6 +423,49 @@ export async function putDocumentBlob( Body: body, ContentType: contentType, ServerSideEncryption: 'AES256', + ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), + }), + ); +} + +export async function putTempDocumentBlob( + token: string, + userId: string, + body: Buffer, + contentType: string, + namespace: string | null, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadKey(token, userId, namespace); + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: contentType, + ServerSideEncryption: 'AES256', + }), + ); +} + +export async function copyTempDocumentBlobToDocument( + token: string, + userId: string, + documentId: string, + namespace: string | null, + contentType: string, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send( + new CopyObjectCommand({ + Bucket: cfg.bucket, + Key: documentKey(documentId, namespace), + CopySource: `${cfg.bucket}/${tempDocumentUploadKey(token, userId, namespace)}`, + ContentType: contentType, + MetadataDirective: 'REPLACE', + ServerSideEncryption: 'AES256', }), ); } @@ -292,6 +483,18 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): await deleteDocumentPrefix(`${key}/`).catch(() => undefined); } +export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadKey(token, userId, namespace) })); +} + +export async function deleteTempDocumentFinalizeReceipt(token: string, userId: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadReceiptKey(token, userId, namespace) })); +} + export function isMissingBlobError(error: unknown): boolean { if (!error || typeof error !== 'object') return false; const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; @@ -339,3 +542,53 @@ export async function deleteDocumentPrefix(prefix: string): Promise { return deleted; } + +export async function deleteExpiredTempDocumentUploads( + userId: string, + namespace: string | null, + olderThanMs: number, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const prefix = tempDocumentUploadPrefix(userId, namespace); + let continuationToken: string | undefined; + const keys: string[] = []; + + do { + const listRes = await client.send( + new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + }), + ); + + for (const item of listRes.Contents ?? []) { + const key = item.Key; + const lastModified = item.LastModified?.getTime() ?? 0; + if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue; + keys.push(key); + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + + if (keys.length === 0) return 0; + + let deleted = 0; + for (let i = 0; i < keys.length; i += 1000) { + const batch = keys.slice(i, i + 1000); + const deleteRes = await client.send( + new DeleteObjectsCommand({ + Bucket: cfg.bucket, + Delete: { + Objects: batch.map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + deleted += deleteRes.Deleted?.length ?? 0; + } + + return deleted; +} diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts new file mode 100644 index 0000000..6fe134e --- /dev/null +++ b/src/lib/server/documents/register-upload.ts @@ -0,0 +1,104 @@ +import { randomUUID } from 'node:crypto'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { + enqueueDocumentPreview, +} from '@/lib/server/documents/previews'; +import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; +import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; +import { errorToLog, serverLogger } from '@/lib/server/logger'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; +import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import type { BaseDocument, DocumentType } from '@/types/documents'; + +type RegisterUploadedDocumentInput = { + documentId: string; + userId: string; + namespace: string | null; + name: string; + type: DocumentType; + size: number; + lastModified: number; +}; + +export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise { + const reusableParsedPdf = input.type === 'pdf' + ? await findReusableParsedPdfResult(input.documentId) + : null; + + await db + .insert(documents) + .values({ + id: input.documentId, + userId: input.userId, + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + filePath: input.documentId, + parseState: input.type === 'pdf' + ? stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, + ) + : null, + parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, + }) + .onConflictDoUpdate({ + target: [documents.id, documents.userId], + set: { + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + filePath: input.documentId, + parseState: input.type === 'pdf' + ? stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, + ) + : null, + parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, + }, + }); + + await enqueueDocumentPreview( + { + id: input.documentId, + type: input.type, + lastModified: input.lastModified, + }, + input.namespace, + ).catch((error) => { + serverLogger.warn({ + event: 'documents.preview.enqueue.failed', + degraded: true, + fallbackPath: 'skip_preview_enqueue', + documentId: input.documentId, + error: errorToLog(error), + }, 'Failed to enqueue document preview'); + }); + + if (input.type === 'pdf' && !reusableParsedPdf) { + const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + await recordJobEvent(input.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig); + enqueueParsePdfJob({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + }); + } + + return { + id: input.documentId, + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + scope: 'user', + }; +} diff --git a/tests/upload.spec.ts b/tests/upload.spec.ts index 2008f72..7a6ef51 100644 --- a/tests/upload.spec.ts +++ b/tests/upload.spec.ts @@ -30,6 +30,29 @@ test.describe('Document Upload Tests', () => { await expectDocumentListed(page, 'sample.txt'); }); + test('reuses the same canonical id for identical uploads', async ({ page }) => { + await uploadFile(page, 'sample.pdf'); + await uploadFile(page, 'sample.pdf'); + + const result = await page.evaluate(async () => { + const res = await fetch('/api/documents', { cache: 'no-store' }); + if (!res.ok) { + return { ok: false as const, reason: `status:${res.status}` }; + } + const data = await res.json() as { documents?: Array<{ id: string; name: string }> }; + const matching = (data.documents || []).filter((doc) => doc.name === 'sample.pdf'); + const uniqueIds = Array.from(new Set(matching.map((doc) => doc.id))); + return { ok: true as const, matchingCount: matching.length, uniqueIds }; + }); + + expect(result.ok).toBeTruthy(); + if (!result.ok) { + throw new Error(`Failed to inspect uploaded documents: ${result.reason}`); + } + expect(result.matchingCount).toBe(1); + expect(result.uniqueIds).toHaveLength(1); + }); + test('hashes text/HTML docs using UTF-8 encoded stored string', async ({ page }) => { await uploadFile(page, 'sample.txt'); await expectDocumentListed(page, 'sample.txt'); From cc868511d9c8908d04e017d9c61f0b08d1b06521 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 3 Jun 2026 20:50:44 -0600 Subject: [PATCH 4/9] fix(documents): handle precondition failures and optimize upload finalization Add detection for precondition failures during blob copy to prevent race conditions when finalizing uploads. Refactor upload finalization to use Promise.all for concurrent processing. Extract parse state and parsed JSON key logic to variables for clarity and to avoid duplication in document registration. --- .../documents/blob/upload/finalize/route.ts | 29 +++++++++++-------- src/lib/server/documents/blobstore.ts | 8 +++++ src/lib/server/documents/register-upload.ts | 28 ++++++++---------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/src/app/api/documents/blob/upload/finalize/route.ts b/src/app/api/documents/blob/upload/finalize/route.ts index ea82ce4..8ab877f 100644 --- a/src/app/api/documents/blob/upload/finalize/route.ts +++ b/src/app/api/documents/blob/upload/finalize/route.ts @@ -10,6 +10,7 @@ import { headDocumentBlob, headTempDocumentBlob, isMissingBlobError, + isPreconditionFailed, isValidTempUploadToken, putTempDocumentFinalizeReceipt, } from '@/lib/server/documents/blobstore'; @@ -138,13 +139,18 @@ async function finalizeOne(input: { await headDocumentBlob(documentId, input.namespace); } catch (error) { if (!isMissingBlobError(error)) throw error; - await copyTempDocumentBlobToDocument( - input.upload.token, - input.userId, - documentId, - input.namespace, - temp.contentType, - ); + try { + await copyTempDocumentBlobToDocument( + input.upload.token, + input.userId, + documentId, + input.namespace, + temp.contentType, + { ifNoneMatch: true }, + ); + } catch (copyError) { + if (!isPreconditionFailed(copyError)) throw copyError; + } } const canonicalHead = await headDocumentBlob(documentId, input.namespace); @@ -194,14 +200,13 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 }); } - const stored: BaseDocument[] = []; - for (const upload of uploads) { - stored.push(await finalizeOne({ + const stored = await Promise.all( + uploads.map((upload) => finalizeOne({ upload, userId, namespace, - })); - } + })), + ); return NextResponse.json({ stored }); } catch (error) { diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index b8fa988..36316c9 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -73,6 +73,12 @@ export function isValidDocumentId(id: string): boolean { return DOCUMENT_ID_REGEX.test(id); } +export function isPreconditionFailed(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } }; + return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed'; +} + export function isValidTempUploadToken(token: string): boolean { return TEMP_UPLOAD_TOKEN_REGEX.test(token); } @@ -455,6 +461,7 @@ export async function copyTempDocumentBlobToDocument( documentId: string, namespace: string | null, contentType: string, + options?: { ifNoneMatch?: boolean }, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -466,6 +473,7 @@ export async function copyTempDocumentBlobToDocument( ContentType: contentType, MetadataDirective: 'REPLACE', ServerSideEncryption: 'AES256', + ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), }), ); } diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts index 6fe134e..f63daea 100644 --- a/src/lib/server/documents/register-upload.ts +++ b/src/lib/server/documents/register-upload.ts @@ -27,6 +27,14 @@ export async function registerUploadedDocument(input: RegisterUploadedDocumentIn const reusableParsedPdf = input.type === 'pdf' ? await findReusableParsedPdfResult(input.documentId) : null; + const parsedJsonKey = reusableParsedPdf?.parsedJsonKey ?? null; + const parseState = input.type === 'pdf' + ? stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, + ) + : null; await db .insert(documents) @@ -38,14 +46,8 @@ export async function registerUploadedDocument(input: RegisterUploadedDocumentIn size: input.size, lastModified: input.lastModified, filePath: input.documentId, - parseState: input.type === 'pdf' - ? stringifyDocumentParseState( - reusableParsedPdf - ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } - : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, - ) - : null, - parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, + parseState, + parsedJsonKey, }) .onConflictDoUpdate({ target: [documents.id, documents.userId], @@ -55,14 +57,8 @@ export async function registerUploadedDocument(input: RegisterUploadedDocumentIn size: input.size, lastModified: input.lastModified, filePath: input.documentId, - parseState: input.type === 'pdf' - ? stringifyDocumentParseState( - reusableParsedPdf - ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } - : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, - ) - : null, - parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, + parseState, + parsedJsonKey, }, }); From d489356b0087ef551db642ce6cbefb0ef4a280f5 Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 3 Jun 2026 23:31:18 -0600 Subject: [PATCH 5/9] refactor(documents): unify document upload and deletion logic Replace type-specific document add/remove functions with generalized uploadDocuments and deleteDocument methods. Consolidate file upload handling in DocumentUploader for improved maintainability. Introduce evictCachedDocument to streamline cache eviction across document types. Update DocumentContext to support batch uploads and single-method deletion, simplifying context API and reducing duplication. --- src/components/doclist/DocumentList.tsx | 10 +- src/components/documents/DocumentUploader.tsx | 105 +++++++++-------- src/contexts/DocumentContext.tsx | 111 +++++++++--------- src/lib/client/cache/documents.ts | 4 + 4 files changed, 114 insertions(+), 116 deletions(-) diff --git a/src/components/doclist/DocumentList.tsx b/src/components/doclist/DocumentList.tsx index b854554..dfe1a0e 100644 --- a/src/components/doclist/DocumentList.tsx +++ b/src/components/doclist/DocumentList.tsx @@ -228,13 +228,11 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) { const { pdfDocs, - removePDFDocument, isPDFLoading, epubDocs, - removeEPUBDocument, isEPUBLoading, htmlDocs, - removeHTMLDocument, + deleteDocument, isHTMLLoading, } = useDocuments(); @@ -436,9 +434,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) { const handleDelete = useCallback(async () => { if (!documentToDelete) return; try { - if (documentToDelete.type === 'pdf') await removePDFDocument(documentToDelete.id); - else if (documentToDelete.type === 'epub') await removeEPUBDocument(documentToDelete.id); - else if (documentToDelete.type === 'html') await removeHTMLDocument(documentToDelete.id); + await deleteDocument(documentToDelete.id); setFolders((prev) => prev.map((f) => ({ ...f, @@ -451,7 +447,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) { } catch (err) { console.error('Failed to remove document:', err); } - }, [documentToDelete, removePDFDocument, removeEPUBDocument, removeHTMLDocument]); + }, [deleteDocument, documentToDelete]); const handleDeleteDoc = useCallback((doc: DocumentListDocument) => { setDocumentToDelete({ id: doc.id, name: doc.name, type: doc.type }); diff --git a/src/components/documents/DocumentUploader.tsx b/src/components/documents/DocumentUploader.tsx index 4f1a07c..54d8859 100644 --- a/src/components/documents/DocumentUploader.tsx +++ b/src/components/documents/DocumentUploader.tsx @@ -33,9 +33,7 @@ export function DocumentUploader({ const uploaderId = useId(); const enableDocx = useFeatureFlag('enableDocxConversion'); const { - addPDFDocument: addPDF, - addEPUBDocument: addEPUB, - addHTMLDocument: addHTML, + uploadDocuments, refreshDocuments, } = useDocuments(); const [isUploading, setIsUploading] = useState(false); @@ -63,57 +61,60 @@ export function DocumentUploader({ }); try { + const docxFiles: File[] = []; + const regularFiles: File[] = []; + for (const file of acceptedFiles) { - if (file.type === 'application/pdf') { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: file.name, - }); - await addPDF(file); - completedFiles += 1; - } else if (file.type === 'application/epub+zip') { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: file.name, - }); - await addEPUB(file); - completedFiles += 1; - } else if (file.type === 'text/plain' || file.type === 'text/markdown' || file.name.endsWith('.md')) { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: file.name, - }); - await addHTML(file); - completedFiles += 1; - } else if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { - // Preserve prior UX: show "Converting DOCX..." state rather than generic uploading. - setIsUploading(false); - setIsConverting(true); - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'converting', - currentFileName: file.name, - }); - // Convert+upload directly on the server. Use sha(docx) as stable ID to avoid duplicates. - await uploadDocxAsPdf(file); - await refreshDocuments(); - setIsConverting(false); - setIsUploading(true); - completedFiles += 1; - } else { + if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { + docxFiles.push(file); continue; } + if ( + file.type === 'application/pdf' + || file.type === 'application/epub+zip' + || file.type === 'text/plain' + || file.type === 'text/markdown' + || file.name.endsWith('.md') + ) { + regularFiles.push(file); + } + } + + if (regularFiles.length > 0) { + emitBatchState({ + isActive: true, + totalFiles, + completedFiles, + phase: 'uploading', + currentFileName: regularFiles[0]?.name ?? null, + }); + await uploadDocuments(regularFiles); + completedFiles += regularFiles.length; + emitBatchState({ + isActive: true, + totalFiles, + completedFiles, + phase: 'uploading', + currentFileName: null, + }); + } + + for (const file of docxFiles) { + // Preserve prior UX: show "Converting DOCX..." state rather than generic uploading. + setIsUploading(false); + setIsConverting(true); + emitBatchState({ + isActive: true, + totalFiles, + completedFiles, + phase: 'converting', + currentFileName: file.name, + }); + await uploadDocxAsPdf(file); + await refreshDocuments(); + setIsConverting(false); + setIsUploading(true); + completedFiles += 1; emitBatchState({ isActive: true, @@ -137,7 +138,7 @@ export function DocumentUploader({ currentFileName: null, }); } - }, [addHTML, addPDF, addEPUB, refreshDocuments, enableDocx, emitBatchState]); + }, [uploadDocuments, refreshDocuments, enableDocx, emitBatchState]); const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, diff --git a/src/contexts/DocumentContext.tsx b/src/contexts/DocumentContext.tsx index f45d8b1..860fa91 100644 --- a/src/contexts/DocumentContext.tsx +++ b/src/contexts/DocumentContext.tsx @@ -3,32 +3,53 @@ import { createContext, useCallback, useContext, useEffect, useMemo, ReactNode } from 'react'; import { useQuery, useQueryClient } from '@tanstack/react-query'; import type { BaseDocument } from '@/types/documents'; -import { listDocuments, uploadDocuments, deleteDocuments } from '@/lib/client/api/documents'; -import { putCachedEpub, putCachedHtml, putCachedPdf, evictCachedEpub, evictCachedHtml, evictCachedPdf } from '@/lib/client/cache/documents'; +import { + deleteDocuments as deleteServerDocuments, + listDocuments, + uploadDocuments as uploadServerDocuments, +} from '@/lib/client/api/documents'; +import { cacheStoredDocumentFromBytes, evictCachedDocument } from '@/lib/client/cache/documents'; import { useAuthSession } from '@/hooks/useAuthSession'; interface DocumentContextType { pdfDocs: Array; - addPDFDocument: (file: File) => Promise; - removePDFDocument: (id: string) => Promise; isPDFLoading: boolean; epubDocs: Array; - addEPUBDocument: (file: File) => Promise; - removeEPUBDocument: (id: string) => Promise; isEPUBLoading: boolean; htmlDocs: Array; - addHTMLDocument: (file: File) => Promise; - removeHTMLDocument: (id: string) => Promise; isHTMLLoading: boolean; + uploadDocuments: (files: File[]) => Promise; + deleteDocument: (id: string) => Promise; refreshDocuments: () => Promise; } const DocumentContext = createContext(undefined); const DOCUMENTS_QUERY_KEY = 'documents'; +type SupportedDocument = BaseDocument & { type: 'pdf' | 'epub' | 'html' }; + +function mergeStoredDocuments( + previous: SupportedDocument[] | undefined, + uploaded: BaseDocument[], +): SupportedDocument[] { + const next = [...(previous ?? [])]; + + for (let index = uploaded.length - 1; index >= 0; index -= 1) { + const stored = uploaded[index]; + if (stored.type !== 'pdf' && stored.type !== 'epub' && stored.type !== 'html') { + continue; + } + const supportedStored = stored as SupportedDocument; + const withoutExisting = next.filter((document) => document.id !== supportedStored.id); + next.splice(0, next.length, supportedStored, ...withoutExisting); + } + + return next; +} + export function DocumentProvider({ children }: { children: ReactNode }) { const queryClient = useQueryClient(); const { data: sessionData, isPending: isSessionPending } = useAuthSession(); @@ -81,67 +102,43 @@ export function DocumentProvider({ children }: { children: ReactNode }) { return { pdfDocs, epubDocs, htmlDocs }; }, [docs]); - const cacheUploaded = useCallback(async (stored: BaseDocument, file: File) => { - try { - if (stored.type === 'pdf') { - await putCachedPdf(stored, await file.arrayBuffer()); - } else if (stored.type === 'epub') { - await putCachedEpub(stored, await file.arrayBuffer()); - } else if (stored.type === 'html') { - const buf = await file.arrayBuffer(); - const decoded = new TextDecoder().decode(new Uint8Array(buf)); - await putCachedHtml(stored, decoded); - } - } catch (err) { - // Cache failures should not block uploads. - console.warn('Failed to cache uploaded document:', stored.id, err); - } - }, []); + const uploadDocuments = useCallback(async (files: File[]): Promise => { + if (files.length === 0) return []; - const addDocument = useCallback(async (file: File): Promise => { - const [stored] = await uploadDocuments([file]); - if (!stored) throw new Error('Upload succeeded but returned no document'); - await cacheUploaded(stored, file); - const isSupported = stored.type === 'pdf' || stored.type === 'epub' || stored.type === 'html'; - if (!isSupported) return stored.id; - const supportedStored = stored as BaseDocument & { type: 'pdf' | 'epub' | 'html' }; - queryClient.setQueryData>(documentsQueryKey, (prev = []) => { - const next = prev.filter((d) => d.id !== supportedStored.id); - return [supportedStored, ...next]; - }); - return stored.id; - }, [cacheUploaded, queryClient, documentsQueryKey]); - - const addPDFDocument = useCallback(async (file: File) => addDocument(file), [addDocument]); - const addEPUBDocument = useCallback(async (file: File) => addDocument(file), [addDocument]); - const addHTMLDocument = useCallback(async (file: File) => addDocument(file), [addDocument]); - - const removeById = useCallback(async (id: string) => { - await deleteDocuments({ ids: [id] }); - await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]); - queryClient.setQueryData>(documentsQueryKey, (prev = []) => - prev.filter((d) => d.id !== id), + const stored = await uploadServerDocuments(files); + await Promise.allSettled( + stored.map(async (document, index) => { + const file = files[index]; + if (!file) return; + await cacheStoredDocumentFromBytes(document, await file.arrayBuffer()); + }), ); - }, [queryClient, documentsQueryKey]); - const removePDFDocument = useCallback(async (id: string) => removeById(id), [removeById]); - const removeEPUBDocument = useCallback(async (id: string) => removeById(id), [removeById]); - const removeHTMLDocument = useCallback(async (id: string) => removeById(id), [removeById]); + queryClient.setQueryData(documentsQueryKey, (previous) => + mergeStoredDocuments(previous, stored), + ); + + return stored; + }, [documentsQueryKey, queryClient]); + + const deleteDocument = useCallback(async (id: string) => { + await deleteServerDocuments({ ids: [id] }); + await evictCachedDocument(id); + queryClient.setQueryData(documentsQueryKey, (previous = []) => + previous.filter((document) => document.id !== id), + ); + }, [documentsQueryKey, queryClient]); return ( diff --git a/src/lib/client/cache/documents.ts b/src/lib/client/cache/documents.ts index b552c0a..ef3aa06 100644 --- a/src/lib/client/cache/documents.ts +++ b/src/lib/client/cache/documents.ts @@ -107,6 +107,10 @@ export async function evictCachedHtml(id: string): Promise { await removeHtmlDocument(id); } +export async function evictCachedDocument(id: string): Promise { + await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]); +} + export async function clearDocumentCache(): Promise { const { clearPdfDocuments, clearEpubDocuments, clearHtmlDocuments } = await import('@/lib/client/dexie'); await Promise.all([clearPdfDocuments(), clearEpubDocuments(), clearHtmlDocuments()]); From 62c913e96dfaedc5d535f05c524269dad89b8c63 Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 4 Jun 2026 10:52:49 -0600 Subject: [PATCH 6/9] feat(pdf): refactor parse operation orchestration and client readiness flow Introduce startPdfParseOperation to encapsulate worker operation creation, job rate tracking, and parse state initialization for PDFs. Replace scattered operation setup logic in register-upload, docx-to-pdf upload, and parse-state-backfill with this unified helper. Update job enqueuing to propagate initial opId/jobId/status for accurate state tracking. Change getParsedPdfDocument to throw on non-ready states and remove polling logic from usePdfDocument, shifting readiness detection to event streams. Switch API responses for non-ready parses from 202 to 409 to clarify client expectations and simplify error handling. Add targeted tests to verify legacy backfill behavior and ensure no worker operation is created for pure data fetches of pending legacy PDFs. BREAKING CHANGE: API now returns 409 Conflict for non-ready parsed PDFs instead of 202, and clients must subscribe to event streams for parse progress. --- src/app/(app)/pdf/[id]/usePdfDocument.ts | 118 ++++----- .../api/documents/[id]/parsed/events/route.ts | 47 ++-- src/app/api/documents/[id]/parsed/route.ts | 233 ++---------------- .../api/documents/docx-to-pdf/upload/route.ts | 29 +-- src/lib/client/api/documents.ts | 24 +- src/lib/server/compute/worker-parse-state.ts | 2 +- .../server/documents/parse-state-backfill.ts | 23 +- .../server/documents/pdf-parse-operation.ts | 30 +++ src/lib/server/documents/register-upload.ts | 20 +- src/lib/server/jobs/user-pdf-layout-job.ts | 29 ++- ...vents-route-legacy-backfill.vitest.spec.ts | 156 ++++++++++++ ...arsed-route-legacy-backfill.vitest.spec.ts | 33 +-- 12 files changed, 357 insertions(+), 387 deletions(-) create mode 100644 src/lib/server/documents/pdf-parse-operation.ts create mode 100644 tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts diff --git a/src/app/(app)/pdf/[id]/usePdfDocument.ts b/src/app/(app)/pdf/[id]/usePdfDocument.ts index e0fbf79..ffa2be8 100644 --- a/src/app/(app)/pdf/[id]/usePdfDocument.ts +++ b/src/app/(app)/pdf/[id]/usePdfDocument.ts @@ -169,53 +169,29 @@ export function usePdfDocument(): PdfDocumentState { // Guards for setCurrentDocument to prevent stale loads from overwriting newer selections. const docLoadSeqRef = useRef(0); const docLoadAbortRef = useRef(null); - const parsePollAbortRef = useRef(null); + const parseStreamAbortRef = useRef(null); const parseSseCloseRef = useRef<(() => void) | null>(null); const lastPreparedPlaybackPageRef = useRef(null); - const fetchParsedDocument = useCallback(async ( + const loadParsedDocumentOnce = useCallback(async ( documentId: string, - initialStatus: PdfParseStatus | null, signal: AbortSignal, - initialOpId?: string | null, ): Promise => { - // Legacy PDFs may have null parseStatus; treat as pending so opening the - // document backfills parse output via the parsed endpoint polling path. - const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending'; - setParseStatus(effectiveInitialStatus); + if (signal.aborted) return; + const parsed = await getParsedPdfDocument(documentId, { signal }); + if (signal.aborted) return; + setParsedDocument(parsed); + setParseStatus('ready'); setParseProgress(null); - const delayMs = 1200; - const retryFailed = effectiveInitialStatus === 'failed'; - let attempt = 0; - let effectiveOpId = initialOpId?.trim() || null; - while (!signal.aborted) { - if (signal.aborted) return; - const result = await getParsedPdfDocument(documentId, { - signal, - retryFailed: retryFailed && attempt === 0, - ...(effectiveOpId ? { opId: effectiveOpId } : {}), - }); - if (result.status === 'ready') { - setParsedDocument(result.parsed); - setParseStatus('ready'); - setParseProgress(null); - setActiveParseOpId(null); - return; - } - if ('opId' in result && typeof result.opId === 'string' && result.opId.trim()) { - effectiveOpId = result.opId.trim(); - setActiveParseOpId(effectiveOpId); - } - setParseStatus(result.status); - setParseProgress(result.parseProgress ?? null); - if (result.status === 'failed') { - setParsedDocument(null); - setActiveParseOpId(null); - return; - } - attempt += 1; - await new Promise((resolve) => setTimeout(resolve, delayMs)); - } + setActiveParseOpId(null); + }, []); + + const resetParsedDocumentState = useCallback(() => { + setParsedDocument(null); + setCurrDocText(undefined); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; + pageTextCacheRef.current.clear(); }, []); const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise => { @@ -229,14 +205,14 @@ export function usePdfDocument(): PdfDocumentState { } }, []); - const startParsedPolling = useCallback((documentId: string, initialOpId?: string | null) => { - parsePollAbortRef.current?.abort(); + const startParsedEventStream = useCallback((documentId: string, initialOpId?: string | null) => { + parseStreamAbortRef.current?.abort(); parseSseCloseRef.current?.(); parseSseCloseRef.current = null; setParseProgress(null); setActiveParseOpId(initialOpId?.trim() || null); const controller = new AbortController(); - parsePollAbortRef.current = controller; + parseStreamAbortRef.current = controller; const closeSse = subscribeParsedPdfDocumentEvents(documentId, { opId: initialOpId?.trim() || null, @@ -248,25 +224,27 @@ export function usePdfDocument(): PdfDocumentState { } setParseStatus(snapshot.parseStatus); setParseProgress(snapshot.parseProgress); - if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') { - if (snapshot.parseStatus === 'failed') { - setParsedDocument(null); - setIsPlaybackReady(false); - lastPreparedPlaybackPageRef.current = null; - setActiveParseOpId(null); - } else { - void fetchParsedDocument( - documentId, - 'ready', - controller.signal, - typeof snapshot.opId === 'string' ? snapshot.opId : (initialOpId ?? null), - ); - } + if (snapshot.parseStatus === 'ready') { closeSse(); parseSseCloseRef.current = null; - if (parsePollAbortRef.current === controller) { - parsePollAbortRef.current = null; + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; } + void loadParsedDocumentOnce(documentId, controller.signal).catch((error) => { + if (error instanceof DOMException && error.name === 'AbortError') return; + console.error('Failed to load parsed PDF after ready status:', error); + resetParsedDocumentState(); + }); + return; + } + if (snapshot.parseStatus === 'failed') { + closeSse(); + parseSseCloseRef.current = null; + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; + } + resetParsedDocumentState(); + setActiveParseOpId(null); return; } }, @@ -286,11 +264,11 @@ export function usePdfDocument(): PdfDocumentState { if (parseSseCloseRef.current === closeSse) { parseSseCloseRef.current = null; } - if (parsePollAbortRef.current === controller) { - parsePollAbortRef.current = null; + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; } }, { once: true }); - }, [fetchParsedDocument, setActiveParseOpId]); + }, [loadParsedDocumentOnce, resetParsedDocumentState, setActiveParseOpId]); useEffect(() => { pdfDocumentRef.current = pdfDocument; @@ -472,8 +450,8 @@ export function usePdfDocument(): PdfDocumentState { // or fast refresh. pdfDocGenerationRef.current += 1; loadSeqRef.current += 1; - parsePollAbortRef.current?.abort(); - parsePollAbortRef.current = null; + parseStreamAbortRef.current?.abort(); + parseStreamAbortRef.current = null; parseSseCloseRef.current?.(); parseSseCloseRef.current = null; pageTextCacheRef.current.clear(); @@ -502,7 +480,7 @@ export function usePdfDocument(): PdfDocumentState { setParseStatus(initialParseStatus); setParseProgress(null); setActiveParseOpId(null); - startParsedPolling(id, null); + startParsedEventStream(id, null); void fetchDocumentSettings(id, controller.signal); } @@ -537,7 +515,7 @@ export function usePdfDocument(): PdfDocumentState { setCurrDocText, setPdfDocument, fetchDocumentSettings, - startParsedPolling, + startParsedEventStream, ]); const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise => { @@ -564,11 +542,11 @@ export function usePdfDocument(): PdfDocumentState { setParseStatus(forced.status); setParseProgress(null); setActiveParseOpId(forced.opId ?? null); - startParsedPolling(currDocId, forced.opId ?? null); + startParsedEventStream(currDocId, forced.opId ?? null); } catch (error) { console.error('Failed to force PDF reparse:', error); } - }, [currDocId, startParsedPolling]); + }, [currDocId, startParsedEventStream]); /** * Clears the current document state @@ -582,8 +560,8 @@ export function usePdfDocument(): PdfDocumentState { docLoadSeqRef.current += 1; docLoadAbortRef.current?.abort(); docLoadAbortRef.current = null; - parsePollAbortRef.current?.abort(); - parsePollAbortRef.current = null; + parseStreamAbortRef.current?.abort(); + parseStreamAbortRef.current = null; parseSseCloseRef.current?.(); parseSseCloseRef.current = null; setCurrDocId(undefined); diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index 79d6801..3406252 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -5,7 +5,11 @@ import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; import { isAbortLikeError } from '@/lib/server/compute/abort-like-error'; -import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/server/compute/worker-parse-state'; +import { + isWorkerOperationStateStale, + mergeNonReadyParseSnapshot, + snapshotFromWorkerState, +} from '@/lib/server/compute/worker-parse-state'; import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { @@ -83,13 +87,20 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr && workerState.opId === opId && !isWorkerOperationStateStale(workerState, opStaleMs) ) { + const merged = mergeNonReadyParseSnapshot({ + parseStatus, + parseProgress: state.progress ?? null, + workerState, + }); + const workerSnapshot = snapshotFromWorkerState(workerState); + const fromWorker = workerSnapshot.parseStatus === 'pending' || workerSnapshot.parseStatus === 'running'; return { snapshot: { - ...snapshotFromWorkerState(workerState), + ...merged, opId: workerState.opId, }, opId: workerState.opId, - fromWorker: true, + fromWorker, }; } } @@ -262,22 +273,19 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string let current = initialState.snapshot; let signature = JSON.stringify(current); let currentOpId = requestedOpId ?? initialState.opId; - let currentFromWorker = initialState.fromWorker; let lastEventId: number | null = null; let loggedMissingOpId = false; const pinnedRequestedOp = requestedOpId ?? null; - const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot, fromWorker: boolean): boolean => { + const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot): boolean => { const isTerminal = snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed'; if (!isTerminal) return false; - // If caller pinned an opId, keep streaming until worker confirms that - // op state. DB fallback can report stale terminal status for other rows. - if (pinnedRequestedOp && snapshot.opId === pinnedRequestedOp && !fromWorker) return false; + if (pinnedRequestedOp && snapshot.opId !== pinnedRequestedOp) return false; return true; }; writeSnapshot(current); - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } @@ -304,7 +312,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } current = next.snapshot; signature = next.signature; - currentFromWorker = next.fromWorker; if (!requestedOpId && next.opId !== currentOpId) { currentOpId = next.opId; lastEventId = null; @@ -313,7 +320,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string workerAbort = null; } } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); } }).catch((error) => { @@ -350,7 +357,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } current = next.snapshot; signature = next.signature; - currentFromWorker = next.fromWorker; currentOpId = requestedOpId ?? next.opId; if (!currentOpId) { if (!loggedMissingOpId) { @@ -358,7 +364,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string logger.warn({ event: 'documents.parsed.events.missing_opid_non_terminal', degraded: true, - step: 'poll_without_worker_op', + step: 'missing_opid_fallback', documentId: id, storageUserIdHash, parseStatus: current.parseStatus, @@ -368,7 +374,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } else if (loggedMissingOpId) { loggedMissingOpId = false; } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } @@ -472,19 +478,23 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string ); if (!workerSnapshot || workerSnapshot.opId !== currentOpId) continue; + const mergedSnapshot = mergeNonReadyParseSnapshot({ + parseStatus: current.parseStatus, + parseProgress: current.parseProgress, + workerState: workerSnapshot, + }); const nextSnapshot: ParsedSnapshot = { - ...snapshotFromWorkerState(workerSnapshot), + ...mergedSnapshot, opId: workerSnapshot.opId, }; const nextSignature = JSON.stringify(nextSnapshot); if (nextSignature !== signature) { current = nextSnapshot; signature = nextSignature; - currentFromWorker = true; writeSnapshot(current); } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } @@ -523,12 +533,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } current = next.snapshot; signature = next.signature; - currentFromWorker = next.fromWorker; if (!requestedOpId && next.opId !== currentOpId) { currentOpId = next.opId; lastEventId = null; } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 11158ea..9375ee2 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -4,41 +4,36 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; import { - documentParseStateFromWorkerState, isWorkerOperationStateStale, snapshotFromWorkerState, } from '@/lib/server/compute/worker-parse-state'; import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state'; import { - documentKey, getParsedDocumentBlob, getParsedDocumentBlobByKey, isMissingBlobError, isValidDocumentId, - putParsedDocumentBlob, } from '@/lib/server/documents/blobstore'; import { normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, - resolveParsedPdfParserVersion, stringifyDocumentParseState, } from '@/lib/server/documents/parse-state'; -import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; -import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { checkJobRate, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; -import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger'; +import { createRequestLogger, hashForLog } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; -import { logDegraded } from '@/lib/server/errors/logging'; import type { ParsedPdfDocument } from '@/types/parsed-pdf'; import { getComputeOpStaleMs } from '@openreader/compute-core'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutJobResult } from '@openreader/compute-core/api-contracts'; export const dynamic = 'force-dynamic'; @@ -102,105 +97,12 @@ async function writeParseRowState(input: { .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); } -async function finalizeFromWorkerState(input: { - workerState: WorkerOperationState; - row: ParseRow; - namespace: string | null; - logger: ServerLogger; -}): Promise { - const snapshot = snapshotFromWorkerState(input.workerState); - const workerParseState = documentParseStateFromWorkerState(input.workerState); - - if (snapshot.parseStatus === 'pending' || snapshot.parseStatus === 'running') { - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState(workerParseState), - }); - return NextResponse.json({ - parseStatus: snapshot.parseStatus, - parseProgress: snapshot.parseProgress, - opId: input.workerState.opId, - }, { status: 202 }); - } - - if (snapshot.parseStatus === 'failed') { - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState(workerParseState), - }); - - return NextResponse.json({ - parseStatus: 'failed', - parseProgress: null, - opId: input.workerState.opId, - error: input.workerState.error?.message ?? 'Worker parse failed', - }, { status: 202 }); - } - - let parsedJsonKey: string | null = null; - if (input.workerState.result && 'parsedObjectKey' in input.workerState.result) { - parsedJsonKey = typeof input.workerState.result.parsedObjectKey === 'string' - ? input.workerState.result.parsedObjectKey - : null; - } - - if (!parsedJsonKey && input.workerState.result && 'parsed' in input.workerState.result && input.workerState.result.parsed) { - const parsedJson = Buffer.from(JSON.stringify(input.workerState.result.parsed)); - parsedJsonKey = await putParsedDocumentBlob(input.row.id, parsedJson, input.namespace); - } - - if (!parsedJsonKey) { - return errorResponse(new Error('Worker completed without parsed output'), { - apiErrorMessage: 'Worker completed without parsed output', - normalize: { code: 'DOCUMENTS_PARSED_WORKER_OUTPUT_MISSING', errorClass: 'upstream' }, - }); - } - - const json = await getParsedDocumentBlobByKey(parsedJsonKey); - let parsedDoc: ParsedPdfDocument | null = null; - try { - parsedDoc = JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument; - } catch { - parsedDoc = null; - } - - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState({ - ...workerParseState, - parserVersion: resolveParsedPdfParserVersion(parsedDoc), - }), - parsedJsonKey, - }); - - if (!hasAnyParsedBlocks(parsedDoc)) { - input.logger.warn({ - event: 'documents.parsed.no_blocks_in_output', - documentId: input.row.id, - userIdHash: hashForLog(input.row.userId), - parsedJsonKey, - }, 'Worker output parsed successfully but contained no blocks'); - } - - return new NextResponse(new Uint8Array(json), { - status: 200, - headers: { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - }, - }); -} - export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { const { logger } = createRequestLogger({ route: '/api/documents/[id]/parsed', request: req, }); try { - const opStaleMs = getComputeOpStaleMs(); if (!isS3Configured()) return s3NotConfiguredResponse(); const authCtxOrRes = await requireAuthContext(req); @@ -209,8 +111,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string const params = await ctx.params; const id = (params.id || '').trim().toLowerCase(); - const retryFailed = req.nextUrl.searchParams.get('retry') === '1'; - const requestedOpId = normalizeOpId(req.nextUrl.searchParams.get('opId')); if (!isValidDocumentId(id)) { return NextResponse.json({ error: 'Invalid document id' }, { status: 400 }); } @@ -225,112 +125,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - if (requestedOpId) { - const workerState = await fetchWorkerOperationState(requestedOpId); - if (workerState && workerState.opId === requestedOpId) { - return finalizeFromWorkerState({ - workerState, - row, - namespace: testNamespace, - logger, - }); - } - logDegraded(logger, { - event: 'documents.parsed.requested_op_unavailable', - msg: 'Requested worker operation id was unavailable', - step: 'requested_op_lookup', - context: { - documentId: id, - userIdHash: hashForLog(row.userId), - opId: requestedOpId, - error: { - name: 'RequestedOperationUnavailable', - message: `Requested worker operation ${requestedOpId} was unavailable`, - }, - }, - }); - } - - let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); - state = await healStaleDocumentParseState({ - documentId: id, - userId: row.userId, - state, - }); - + const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); const effectiveStatus = normalizeParseStatus(state.status); const effectiveProgress = state.progress ?? null; const effectiveOpId = normalizeOpId(state.opId); - if (effectiveOpId && effectiveStatus !== 'ready') { - const workerState = await fetchWorkerOperationState(effectiveOpId); - if ( - workerState - && workerState.opId === effectiveOpId - && !isWorkerOperationStateStale(workerState, opStaleMs) - ) { - return finalizeFromWorkerState({ - workerState, - row, - namespace: testNamespace, - logger, - }); - } - } - - if (!effectiveOpId && (effectiveStatus === 'pending' || effectiveStatus === 'running')) { - const created = await backfillPendingPdfParseOperation({ - documentId: id, - userId: row.userId, - namespace: testNamespace, - state, - }); - if (created) { - const snapshot = snapshotFromWorkerState(created); - await writeParseRowState({ - documentId: row.id, - userId: row.userId, - parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)), - }); - return NextResponse.json({ - parseStatus: snapshot.parseStatus, - parseProgress: snapshot.parseProgress, - opId: created.opId, - }, { status: 202 }); - } - } - - if (effectiveStatus === 'failed' && retryFailed) { - const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig); - if (!rateDecision.allowed) { - return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); - } - const created = await createOrReusePdfWorkerOperation({ - documentId: id, - namespace: testNamespace, - documentObjectKey: documentKey(id, testNamespace), - }); - await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); - const snapshot = snapshotFromWorkerState(created); - await writeParseRowState({ - documentId: row.id, - userId: row.userId, - parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)), - }); - return NextResponse.json({ - parseStatus: snapshot.parseStatus, - parseProgress: snapshot.parseProgress, - opId: created.opId, - }, { status: 202 }); - } - if (effectiveStatus !== 'ready') { return NextResponse.json({ parseStatus: effectiveStatus, parseProgress: effectiveProgress, opId: effectiveOpId, - }, { status: 202 }); + }, { status: 409 }); } try { @@ -446,25 +251,33 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); } - const created = await createOrReusePdfWorkerOperation({ + const forceToken = randomUUID(); + const startedParse = await startPdfParseOperation({ documentId: id, + userId: authCtxOrRes.userId, namespace: testNamespace, - documentObjectKey: documentKey(id, testNamespace), - forceToken: randomUUID(), + forceToken, }); - await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); - - const snapshot = snapshotFromWorkerState(created); + const snapshot = snapshotFromWorkerState(startedParse.workerState); await writeParseRowState({ documentId: row.id, userId: row.userId, - parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)), + parseState: stringifyDocumentParseState(startedParse.parseState), + }); + enqueueParsePdfJob({ + documentId: id, + userId: row.userId, + namespace: testNamespace, + forceToken, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', }); return NextResponse.json({ parseStatus: snapshot.parseStatus, parseProgress: snapshot.parseProgress, - opId: created.opId, + opId: startedParse.workerState.opId, }, { status: 202 }); } catch (error) { return errorResponse(error, { diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 10d1f23..834b4ab 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -10,16 +10,14 @@ import { db } from '@/db'; import { documents } from '@/db/schema'; import { safeDocumentName } from '@/lib/server/documents/utils'; import { enqueueDocumentPreview } from '@/lib/server/documents/previews'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; -import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; import { putDocumentBlob } from '@/lib/server/documents/blobstore'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core'; const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); @@ -132,6 +130,11 @@ export async function POST(req: NextRequest) { const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`); const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now(); + const startedParse = await startPdfParseOperation({ + documentId: id, + userId: storageUserId, + namespace: testNamespace, + }); await db .insert(documents) @@ -143,12 +146,7 @@ export async function POST(req: NextRequest) { size: pdfContent.length, lastModified, filePath: id, - parseState: stringifyDocumentParseState({ - status: 'pending', - progress: null, - updatedAt: Date.now(), - parserVersion: PDF_PARSER_VERSION, - }), + parseState: stringifyDocumentParseState(startedParse.parseState), parsedJsonKey: null, }) .onConflictDoUpdate({ @@ -159,12 +157,7 @@ export async function POST(req: NextRequest) { size: pdfContent.length, lastModified, filePath: id, - parseState: stringifyDocumentParseState({ - status: 'pending', - progress: null, - updatedAt: Date.now(), - parserVersion: PDF_PARSER_VERSION, - }), + parseState: stringifyDocumentParseState(startedParse.parseState), parsedJsonKey: null, }, }); @@ -186,13 +179,13 @@ export async function POST(req: NextRequest) { }, 'Failed to enqueue preview for converted DOCX'); }); - // Record upload-driven parse load (see register route for rationale). - const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig); enqueueParsePdfJob({ documentId: id, userId: storageUserId, namespace: testNamespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', }); return NextResponse.json({ diff --git a/src/lib/client/api/documents.ts b/src/lib/client/api/documents.ts index 360914f..d6a80ce 100644 --- a/src/lib/client/api/documents.ts +++ b/src/lib/client/api/documents.ts @@ -82,31 +82,20 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort export async function getParsedPdfDocument( id: string, - options?: { signal?: AbortSignal; retryFailed?: boolean; opId?: string }, -): Promise< - | { status: 'ready'; parsed: ParsedPdfDocument } - | { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null; opId?: string | null } -> { - const params = new URLSearchParams(); - if (options?.retryFailed) params.set('retry', '1'); - if (options?.opId) params.set('opId', options.opId); - const query = params.size > 0 ? `?${params.toString()}` : ''; - const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, { + options?: { signal?: AbortSignal }, +): Promise { + const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed`, { signal: options?.signal, cache: 'no-store', }); - if (res.status === 202) { + if (res.status === 409) { const data = (await res.json().catch(() => null)) as { parseStatus?: string; parseProgress?: PdfParseProgress | null; opId?: string | null; } | null; - const parseStatus = data?.parseStatus; - if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') { - return { status: parseStatus, parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null }; - } - return { status: 'pending', parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null }; + throw new Error(data?.parseStatus ? `Parsed PDF is not ready (${data.parseStatus})` : 'Parsed PDF is not ready'); } if (!res.ok) { @@ -114,8 +103,7 @@ export async function getParsedPdfDocument( throw new Error(data?.error || 'Failed to load parsed PDF'); } - const parsed = (await res.json()) as ParsedPdfDocument; - return { status: 'ready', parsed }; + return (await res.json()) as ParsedPdfDocument; } export function subscribeParsedPdfDocumentEvents( diff --git a/src/lib/server/compute/worker-parse-state.ts b/src/lib/server/compute/worker-parse-state.ts index 6798a22..4b954b7 100644 --- a/src/lib/server/compute/worker-parse-state.ts +++ b/src/lib/server/compute/worker-parse-state.ts @@ -68,7 +68,7 @@ export function mergeNonReadyParseSnapshot(input: { workerState: WorkerOperationState; }): { parseStatus: PdfParseStatus; parseProgress: PdfParseProgress | null } { const workerSnapshot = snapshotFromWorkerState(input.workerState); - if (workerSnapshot.parseStatus === 'ready') { + if (workerSnapshot.parseStatus === 'ready' || workerSnapshot.parseStatus === 'failed') { return { parseStatus: input.parseStatus, parseProgress: input.parseProgress, diff --git a/src/lib/server/documents/parse-state-backfill.ts b/src/lib/server/documents/parse-state-backfill.ts index 97f9cae..b972553 100644 --- a/src/lib/server/documents/parse-state-backfill.ts +++ b/src/lib/server/documents/parse-state-backfill.ts @@ -1,7 +1,5 @@ -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; -import { documentKey } from '@/lib/server/documents/blobstore'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; -import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; import { normalizeParseStatus, type DocumentParseState } from '@/lib/server/documents/parse-state'; import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; @@ -20,13 +18,18 @@ export async function backfillPendingPdfParseOperation(input: { if (parseStatus === 'ready' || parseStatus === 'failed') return null; if (normalizeOpId(input.state.opId)) return null; - const created = await createOrReusePdfWorkerOperation({ + const startedParse = await startPdfParseOperation({ documentId: input.documentId, + userId: input.userId, namespace: input.namespace, - documentObjectKey: documentKey(input.documentId, input.namespace), }); - - const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - await recordJobEvent(input.userId, 'pdf_layout', created.opId, rateConfig); - return created; + enqueueParsePdfJob({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', + }); + return startedParse.workerState; } diff --git a/src/lib/server/documents/pdf-parse-operation.ts b/src/lib/server/documents/pdf-parse-operation.ts new file mode 100644 index 0000000..c356f86 --- /dev/null +++ b/src/lib/server/documents/pdf-parse-operation.ts @@ -0,0 +1,30 @@ +import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; +import { documentParseStateFromWorkerState } from '@/lib/server/compute/worker-parse-state'; +import { documentKey } from '@/lib/server/documents/blobstore'; +import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; +import type { DocumentParseState } from '@/lib/server/documents/parse-state'; +import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; + +export async function startPdfParseOperation(input: { + documentId: string; + userId: string; + namespace: string | null; + forceToken?: string; +}): Promise<{ + workerState: WorkerOperationState; + parseState: DocumentParseState; +}> { + const workerState = await createOrReusePdfWorkerOperation({ + documentId: input.documentId, + namespace: input.namespace, + documentObjectKey: documentKey(input.documentId, input.namespace), + ...(input.forceToken ? { forceToken: input.forceToken } : {}), + }); + const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + await recordJobEvent(input.userId, 'pdf_layout', workerState.opId, rateConfig); + return { + workerState, + parseState: documentParseStateFromWorkerState(workerState), + }; +} diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts index f63daea..06bafd2 100644 --- a/src/lib/server/documents/register-upload.ts +++ b/src/lib/server/documents/register-upload.ts @@ -1,4 +1,3 @@ -import { randomUUID } from 'node:crypto'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { @@ -6,10 +5,9 @@ import { } from '@/lib/server/documents/previews'; import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; import { errorToLog, serverLogger } from '@/lib/server/logger'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; -import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter'; import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { BaseDocument, DocumentType } from '@/types/documents'; @@ -27,12 +25,19 @@ export async function registerUploadedDocument(input: RegisterUploadedDocumentIn const reusableParsedPdf = input.type === 'pdf' ? await findReusableParsedPdfResult(input.documentId) : null; + const startedParse = input.type === 'pdf' && !reusableParsedPdf + ? await startPdfParseOperation({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + }) + : null; const parsedJsonKey = reusableParsedPdf?.parsedJsonKey ?? null; const parseState = input.type === 'pdf' ? stringifyDocumentParseState( reusableParsedPdf ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } - : { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }, + : (startedParse?.parseState ?? { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }), ) : null; @@ -79,13 +84,14 @@ export async function registerUploadedDocument(input: RegisterUploadedDocumentIn }, 'Failed to enqueue document preview'); }); - if (input.type === 'pdf' && !reusableParsedPdf) { - const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - await recordJobEvent(input.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig); + if (startedParse) { enqueueParsePdfJob({ documentId: input.documentId, userId: input.userId, namespace: input.namespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', }); } diff --git a/src/lib/server/jobs/user-pdf-layout-job.ts b/src/lib/server/jobs/user-pdf-layout-job.ts index 531dfd0..0f44043 100644 --- a/src/lib/server/jobs/user-pdf-layout-job.ts +++ b/src/lib/server/jobs/user-pdf-layout-job.ts @@ -27,6 +27,9 @@ import { PDF_PARSER_VERSION } from '@openreader/compute-core'; type UserPdfLayoutJobRequest = PdfLayoutJobBase & { userId: string; forceToken?: string; + initialOpId?: string; + initialJobId?: string; + initialStatus?: 'pending' | 'running'; }; const running = new Set(); @@ -49,6 +52,11 @@ function keyFor(input: UserPdfLayoutJobRequest): string { return `shared:${input.documentId}`; } +function normalizeOpId(value: string | undefined): string | undefined { + const normalized = value?.trim(); + return normalized ? normalized : undefined; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -212,6 +220,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise } running.add(key); + let activeOpId = normalizeOpId(input.initialOpId); + let activeJobId = normalizeOpId(input.initialJobId); try { const scopedRows = await loadScopedRows(input); const scopedUserIds = userIdsFromRows(scopedRows); @@ -257,18 +267,21 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise const coordinator = [...scopedRows].sort((a, b) => a.userId.localeCompare(b.userId))[0]; if (!coordinator) return; - const runningStateData: DocumentParseState = { - status: 'running', + const initialInflightStatus = input.initialStatus === 'running' ? 'running' : 'pending'; + const inflightStateData: DocumentParseState = { + status: initialInflightStatus, progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION, + ...(activeOpId ? { opId: activeOpId } : {}), + ...(activeJobId ? { jobId: activeJobId } : {}), }; - const runningState = stringifyDocumentParseState(runningStateData); + const inflightState = stringifyDocumentParseState(inflightStateData); const claimRows = (await db .update(documents) .set({ - parseState: runningState, + parseState: inflightState, }) .where( and( @@ -291,12 +304,10 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise await updateParseStateForUsers({ documentId: input.documentId, userIds: cohortUserIds, - parseState: runningState, + parseState: inflightState, }); const compute = await getCompute(); - let activeOpId: string | undefined; - let activeJobId: string | undefined; let lastProgressWriteAt = 0; let lastSnapshotWriteAt = 0; let lastSnapshotStatus: 'pending' | 'running' | null = null; @@ -395,6 +406,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise progress: null, updatedAt: Date.now(), parserVersion: resolveParsedPdfParserVersion(parsedDoc), + ...(activeOpId ? { opId: activeOpId } : {}), + ...(activeJobId ? { jobId: activeJobId } : {}), }; const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds; await updateParseStateForUsers({ @@ -448,6 +461,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise updatedAt: Date.now(), error: message, parserVersion: PDF_PARSER_VERSION, + ...(activeOpId ? { opId: activeOpId } : {}), + ...(activeJobId ? { jobId: activeJobId } : {}), }; const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId]; await updateParseStateForUsers({ diff --git a/tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts b/tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts new file mode 100644 index 0000000..d39d9f5 --- /dev/null +++ b/tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts @@ -0,0 +1,156 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const hoisted = vi.hoisted(() => ({ + db: null as { + select: ReturnType; + update: ReturnType; + } | null, + row: { + id: 'doc-1', + userId: 'user-1', + parseState: null as string | null, + }, + requireAuthContext: vi.fn(), + backfillPendingPdfParseOperation: vi.fn(), + healStaleDocumentParseState: vi.fn(async ({ state }) => state), +})); + +vi.mock('@/db', () => ({ + get db() { + return hoisted.db; + }, +})); + +vi.mock('@/lib/server/auth/auth', () => ({ + requireAuthContext: hoisted.requireAuthContext, +})); + +vi.mock('@/lib/server/documents/parse-state-backfill', () => ({ + backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation, +})); + +vi.mock('@/lib/server/documents/parse-state-healing', () => ({ + healStaleDocumentParseState: hoisted.healStaleDocumentParseState, +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + isValidDocumentId: vi.fn(() => true), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + isS3Configured: vi.fn(() => true), +})); + +vi.mock('@/lib/server/testing/test-namespace', () => ({ + getOpenReaderTestNamespace: vi.fn(() => null), +})); + +vi.mock('@/lib/server/logger', () => ({ + createRequestLogger: vi.fn(() => ({ + logger: { + warn: vi.fn(), + info: vi.fn(), + error: vi.fn(), + }, + requestId: 'req-test', + })), + hashForLog: vi.fn(() => 'user-hash'), +})); + +describe('GET /api/documents/[id]/parsed/events legacy backfill', () => { + const originalFetch = global.fetch; + + beforeEach(() => { + process.env.COMPUTE_WORKER_URL = 'http://localhost:4010'; + process.env.COMPUTE_WORKER_TOKEN = 'worker-test-token'; + + hoisted.row = { + id: 'doc-1', + userId: 'user-1', + parseState: null, + }; + hoisted.db = { + select: vi.fn(() => ({ + from: vi.fn(() => ({ + where: vi.fn(async () => ([{ ...hoisted.row }])), + })), + })), + update: vi.fn(() => ({ + set: vi.fn((values: { parseState?: string | null }) => ({ + where: vi.fn(async () => { + if (typeof values.parseState !== 'undefined') { + hoisted.row.parseState = values.parseState; + } + return []; + }), + })), + })), + }; + hoisted.requireAuthContext.mockReset(); + hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' }); + hoisted.backfillPendingPdfParseOperation.mockReset(); + hoisted.healStaleDocumentParseState.mockClear(); + + global.fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_, reject) => { + const signal = init?.signal; + if (signal) { + signal.addEventListener('abort', () => { + reject(new DOMException('Aborted', 'AbortError')); + }, { once: true }); + } + })) as typeof fetch; + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + test('creates a worker op for legacy pending PDFs without opId', async () => { + hoisted.backfillPendingPdfParseOperation.mockResolvedValue({ + opId: 'op-legacy-1', + opKey: 'pdf_layout|v1|doc-1||doc-1|', + jobId: 'job-legacy-1', + kind: 'pdf_layout', + status: 'queued', + queuedAt: Date.now(), + progress: null, + result: undefined, + error: undefined, + updatedAt: Date.now(), + }); + + const { GET } = await import('../../src/app/api/documents/[id]/parsed/events/route'); + const controller = new AbortController(); + const request = new NextRequest('http://localhost/api/documents/doc-1/parsed/events', { + signal: controller.signal, + }); + const response = await GET(request, { + params: Promise.resolve({ id: 'doc-1' }), + }); + + expect(response.status).toBe(200); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + + const first = await reader!.read(); + const chunk = new TextDecoder().decode(first.value); + expect(chunk).toContain('event: snapshot'); + expect(chunk).toContain('"parseStatus":"pending"'); + expect(chunk).toContain('"opId":"op-legacy-1"'); + + controller.abort(); + await reader!.cancel().catch(() => {}); + + expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({ + documentId: 'doc-1', + userId: 'user-1', + namespace: null, + state: expect.objectContaining({ status: 'pending' }), + })); + const parseState = String(hoisted.row.parseState ?? ''); + expect(parseState).toContain('"status":"pending"'); + expect(parseState).toContain('"opId":"op-legacy-1"'); + expect(parseState).toContain('"jobId":"job-legacy-1"'); + }); +}); diff --git a/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts b/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts index 15b4bf1..1c48cb6 100644 --- a/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts +++ b/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts @@ -69,7 +69,7 @@ vi.mock('@/lib/server/logger', () => ({ hashForLog: vi.fn(() => 'user-hash'), })); -describe('GET /api/documents/[id]/parsed legacy backfill', () => { +describe('GET /api/documents/[id]/parsed pure data fetch', () => { beforeEach(async () => { process.env.BASE_URL = 'http://localhost:3003'; process.env.AUTH_SECRET = 'test-secret'; @@ -108,40 +108,19 @@ describe('GET /api/documents/[id]/parsed legacy backfill', () => { hoisted.healStaleDocumentParseState.mockClear(); }); - test('creates a worker op for legacy pending PDFs without opId', async () => { - hoisted.backfillPendingPdfParseOperation.mockResolvedValue({ - opId: 'op-legacy-1', - opKey: 'pdf_layout|v1|doc-1||doc-1|', - jobId: 'job-legacy-1', - kind: 'pdf_layout', - status: 'queued', - queuedAt: Date.now(), - progress: null, - result: undefined, - error: undefined, - updatedAt: Date.now(), - }); - + test('returns non-ready status without creating a worker op for legacy pending PDFs without opId', async () => { const { GET } = await import('../../src/app/api/documents/[id]/parsed/route'); const request = new NextRequest('http://localhost/api/documents/doc-1/parsed'); const response = await GET(request, { params: Promise.resolve({ id: 'doc-1' }), }); - expect(response.status).toBe(202); + expect(response.status).toBe(409); await expect(response.json()).resolves.toMatchObject({ parseStatus: 'pending', - opId: 'op-legacy-1', + opId: null, }); - expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({ - documentId: 'doc-1', - userId: 'user-1', - namespace: null, - state: expect.objectContaining({ status: 'pending' }), - })); - const parseState = String(hoisted.row.parseState ?? ''); - expect(parseState).toContain('"status":"pending"'); - expect(parseState).toContain('"opId":"op-legacy-1"'); - expect(parseState).toContain('"jobId":"job-legacy-1"'); + expect(hoisted.backfillPendingPdfParseOperation).not.toHaveBeenCalled(); + expect(hoisted.row.parseState).toBeNull(); }); }); From 7344fcbc51d4b6b4f0f0a124dd7c08e30b232bef Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 4 Jun 2026 12:47:37 -0600 Subject: [PATCH 7/9] fix(pdf): prevent redundant parse jobs and improve docx-to-pdf upload reuse Avoid enqueuing duplicate parse jobs for PDFs that are already parsed or in terminal states. Update docx-to-pdf upload to reuse existing parsed PDF results when possible, skipping unnecessary parse operations and job enqueuing. Adjust parse-state-backfill to only enqueue jobs for pending or running states. Refine client upload logic to better infer document type from MIME when file names are missing. Enhance usePdfDocument to handle terminal parse states more robustly. This change ensures idempotent parse job creation and reduces unnecessary worker load, while improving document upload and parse state handling. --- src/app/(app)/pdf/[id]/usePdfDocument.ts | 31 +++++++++---- .../api/documents/docx-to-pdf/upload/route.ts | 46 ++++++++++++------- src/lib/client/api/documents.ts | 19 ++++++-- src/lib/server/compute/worker-parse-state.ts | 2 +- .../server/documents/parse-state-backfill.ts | 18 ++++---- ...arsed-route-legacy-backfill.vitest.spec.ts | 21 ++++++--- 6 files changed, 90 insertions(+), 47 deletions(-) diff --git a/src/app/(app)/pdf/[id]/usePdfDocument.ts b/src/app/(app)/pdf/[id]/usePdfDocument.ts index ffa2be8..c916bb1 100644 --- a/src/app/(app)/pdf/[id]/usePdfDocument.ts +++ b/src/app/(app)/pdf/[id]/usePdfDocument.ts @@ -213,31 +213,42 @@ export function usePdfDocument(): PdfDocumentState { setActiveParseOpId(initialOpId?.trim() || null); const controller = new AbortController(); parseStreamAbortRef.current = controller; + let isResolvingTerminalState = false; const closeSse = subscribeParsedPdfDocumentEvents(documentId, { opId: initialOpId?.trim() || null, }, { onSnapshot: (snapshot) => { if (controller.signal.aborted) return; + if (isResolvingTerminalState) return; if (typeof snapshot.opId === 'string' && snapshot.opId.trim()) { setActiveParseOpId(snapshot.opId.trim()); } setParseStatus(snapshot.parseStatus); setParseProgress(snapshot.parseProgress); if (snapshot.parseStatus === 'ready') { - closeSse(); - parseSseCloseRef.current = null; - if (parseStreamAbortRef.current === controller) { - parseStreamAbortRef.current = null; - } - void loadParsedDocumentOnce(documentId, controller.signal).catch((error) => { - if (error instanceof DOMException && error.name === 'AbortError') return; - console.error('Failed to load parsed PDF after ready status:', error); - resetParsedDocumentState(); - }); + isResolvingTerminalState = true; + void (async () => { + try { + await loadParsedDocumentOnce(documentId, controller.signal); + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') return; + console.error('Failed to load parsed PDF after ready status:', error); + resetParsedDocumentState(); + } finally { + if (parseSseCloseRef.current === closeSse) { + closeSse(); + parseSseCloseRef.current = null; + } + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; + } + } + })(); return; } if (snapshot.parseStatus === 'failed') { + isResolvingTerminalState = true; closeSse(); parseSseCloseRef.current = null; if (parseStreamAbortRef.current === controller) { diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 834b4ab..51d1bf1 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -8,6 +8,7 @@ import { pathToFileURL } from 'url'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { db } from '@/db'; import { documents } from '@/db/schema'; +import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; import { safeDocumentName } from '@/lib/server/documents/utils'; import { enqueueDocumentPreview } from '@/lib/server/documents/previews'; import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; @@ -18,6 +19,7 @@ import { isS3Configured } from '@/lib/server/storage/s3'; import { putDocumentBlob } from '@/lib/server/documents/blobstore'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); @@ -130,11 +132,19 @@ export async function POST(req: NextRequest) { const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`); const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now(); - const startedParse = await startPdfParseOperation({ - documentId: id, - userId: storageUserId, - namespace: testNamespace, - }); + const reusableParsedPdf = await findReusableParsedPdfResult(id); + const startedParse = reusableParsedPdf + ? null + : await startPdfParseOperation({ + documentId: id, + userId: storageUserId, + namespace: testNamespace, + }); + const parseState = stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : startedParse!.parseState, + ); await db .insert(documents) @@ -146,8 +156,8 @@ export async function POST(req: NextRequest) { size: pdfContent.length, lastModified, filePath: id, - parseState: stringifyDocumentParseState(startedParse.parseState), - parsedJsonKey: null, + parseState, + parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, }) .onConflictDoUpdate({ target: [documents.id, documents.userId], @@ -157,8 +167,8 @@ export async function POST(req: NextRequest) { size: pdfContent.length, lastModified, filePath: id, - parseState: stringifyDocumentParseState(startedParse.parseState), - parsedJsonKey: null, + parseState, + parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, }, }); @@ -179,14 +189,16 @@ export async function POST(req: NextRequest) { }, 'Failed to enqueue preview for converted DOCX'); }); - enqueueParsePdfJob({ - documentId: id, - userId: storageUserId, - namespace: testNamespace, - initialOpId: startedParse.workerState.opId, - initialJobId: startedParse.workerState.jobId, - initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', - }); + if (startedParse) { + enqueueParsePdfJob({ + documentId: id, + userId: storageUserId, + namespace: testNamespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', + }); + } return NextResponse.json({ stored: { diff --git a/src/lib/client/api/documents.ts b/src/lib/client/api/documents.ts index d6a80ce..9a70cf5 100644 --- a/src/lib/client/api/documents.ts +++ b/src/lib/client/api/documents.ts @@ -47,6 +47,14 @@ function documentTypeForName(name: string): DocumentType { return 'html'; } +function documentTypeForMime(contentType: string): DocumentType | null { + const normalized = contentType.trim().toLowerCase(); + if (normalized === 'application/pdf') return 'pdf'; + if (normalized === 'application/epub+zip') return 'epub'; + if (normalized === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') return 'docx'; + return null; +} + export function mimeTypeForDoc(doc: Pick): string { if (doc.type === 'pdf') return 'application/pdf'; if (doc.type === 'epub') return 'application/epub+zip'; @@ -314,11 +322,14 @@ export async function uploadDocuments(files: File[], options?: UploadOptions): P const sources: UploadSource[] = []; for (const file of files) { - const type = documentTypeForName(file.name); - const name = file.name || `upload.${type}`; - const contentType = file.type || mimeTypeForDoc({ name, type }); + const name = file.name || ''; + const type = name + ? documentTypeForName(name) + : (documentTypeForMime(file.type) ?? 'html'); + const resolvedName = name || `upload.${type}`; + const contentType = file.type || mimeTypeForDoc({ name: resolvedName, type }); sources.push({ - name, + name: resolvedName, type, size: file.size, lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(), diff --git a/src/lib/server/compute/worker-parse-state.ts b/src/lib/server/compute/worker-parse-state.ts index 4b954b7..6798a22 100644 --- a/src/lib/server/compute/worker-parse-state.ts +++ b/src/lib/server/compute/worker-parse-state.ts @@ -68,7 +68,7 @@ export function mergeNonReadyParseSnapshot(input: { workerState: WorkerOperationState; }): { parseStatus: PdfParseStatus; parseProgress: PdfParseProgress | null } { const workerSnapshot = snapshotFromWorkerState(input.workerState); - if (workerSnapshot.parseStatus === 'ready' || workerSnapshot.parseStatus === 'failed') { + if (workerSnapshot.parseStatus === 'ready') { return { parseStatus: input.parseStatus, parseProgress: input.parseProgress, diff --git a/src/lib/server/documents/parse-state-backfill.ts b/src/lib/server/documents/parse-state-backfill.ts index b972553..f058ba1 100644 --- a/src/lib/server/documents/parse-state-backfill.ts +++ b/src/lib/server/documents/parse-state-backfill.ts @@ -23,13 +23,15 @@ export async function backfillPendingPdfParseOperation(input: { userId: input.userId, namespace: input.namespace, }); - enqueueParsePdfJob({ - documentId: input.documentId, - userId: input.userId, - namespace: input.namespace, - initialOpId: startedParse.workerState.opId, - initialJobId: startedParse.workerState.jobId, - initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', - }); + if (startedParse.parseState.status === 'pending' || startedParse.parseState.status === 'running') { + enqueueParsePdfJob({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status, + }); + } return startedParse.workerState; } diff --git a/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts b/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts index 1c48cb6..75c5cd2 100644 --- a/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts +++ b/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts @@ -14,8 +14,9 @@ const hoisted = vi.hoisted(() => ({ }, requireAuthContext: vi.fn(), fetchWorkerOperationState: vi.fn(), - backfillPendingPdfParseOperation: vi.fn(), healStaleDocumentParseState: vi.fn(async ({ state }) => state), + startPdfParseOperation: vi.fn(), + enqueueParsePdfJob: vi.fn(), })); vi.mock('@/db', () => ({ @@ -32,14 +33,18 @@ vi.mock('@/lib/server/compute/worker-op-state', () => ({ fetchWorkerOperationState: hoisted.fetchWorkerOperationState, })); -vi.mock('@/lib/server/documents/parse-state-backfill', () => ({ - backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation, -})); - vi.mock('@/lib/server/documents/parse-state-healing', () => ({ healStaleDocumentParseState: hoisted.healStaleDocumentParseState, })); +vi.mock('@/lib/server/documents/pdf-parse-operation', () => ({ + startPdfParseOperation: hoisted.startPdfParseOperation, +})); + +vi.mock('@/lib/server/jobs/user-pdf-layout-job', () => ({ + enqueueParsePdfJob: hoisted.enqueueParsePdfJob, +})); + vi.mock('@/lib/server/documents/blobstore', () => ({ documentKey: vi.fn(), getParsedDocumentBlob: vi.fn(), @@ -104,8 +109,9 @@ describe('GET /api/documents/[id]/parsed pure data fetch', () => { hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' }); hoisted.fetchWorkerOperationState.mockReset(); hoisted.fetchWorkerOperationState.mockResolvedValue(null); - hoisted.backfillPendingPdfParseOperation.mockReset(); hoisted.healStaleDocumentParseState.mockClear(); + hoisted.startPdfParseOperation.mockReset(); + hoisted.enqueueParsePdfJob.mockReset(); }); test('returns non-ready status without creating a worker op for legacy pending PDFs without opId', async () => { @@ -120,7 +126,8 @@ describe('GET /api/documents/[id]/parsed pure data fetch', () => { parseStatus: 'pending', opId: null, }); - expect(hoisted.backfillPendingPdfParseOperation).not.toHaveBeenCalled(); + expect(hoisted.startPdfParseOperation).not.toHaveBeenCalled(); + expect(hoisted.enqueueParsePdfJob).not.toHaveBeenCalled(); expect(hoisted.row.parseState).toBeNull(); }); }); From 2ecd281f75f0c59a7d3c45faa3390be3d2bf77eb Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 4 Jun 2026 13:00:05 -0600 Subject: [PATCH 8/9] refactor(upload): unify docx-to-pdf conversion into finalize handler and remove legacy route Remove the separate docx-to-pdf upload API and integrate DOCX conversion directly into the blob upload finalize route. Update client and uploader logic to treat DOCX as a supported upload type and streamline batch state handling. Adjust document caching and test helpers to align with the new unified flow. Add server-side DOCX-to-PDF conversion utility and corresponding unit tests. This change simplifies the upload pipeline, consolidates conversion logic, and improves maintainability. --- .../documents/blob/upload/finalize/route.ts | 58 +++-- .../api/documents/docx-to-pdf/upload/route.ts | 225 ------------------ src/components/doclist/DocumentList.tsx | 14 +- src/components/documents/DocumentUploader.tsx | 88 ++----- src/contexts/DocumentContext.tsx | 20 ++ src/lib/client/api/documents.ts | 20 -- src/lib/server/documents/docx-convert.ts | 78 ++++++ src/types/documents.ts | 1 - tests/helpers.ts | 24 +- .../blob-upload-finalize-docx.vitest.spec.ts | 150 ++++++++++++ tests/upload.spec.ts | 8 +- 11 files changed, 316 insertions(+), 370 deletions(-) delete mode 100644 src/app/api/documents/docx-to-pdf/upload/route.ts create mode 100644 src/lib/server/documents/docx-convert.ts create mode 100644 tests/unit/blob-upload-finalize-docx.vitest.spec.ts diff --git a/src/app/api/documents/blob/upload/finalize/route.ts b/src/app/api/documents/blob/upload/finalize/route.ts index 8ab877f..b1f5f6e 100644 --- a/src/app/api/documents/blob/upload/finalize/route.ts +++ b/src/app/api/documents/blob/upload/finalize/route.ts @@ -1,3 +1,4 @@ +import path from 'path'; import { createHash } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; @@ -12,8 +13,10 @@ import { isMissingBlobError, isPreconditionFailed, isValidTempUploadToken, + putDocumentBlob, putTempDocumentFinalizeReceipt, } from '@/lib/server/documents/blobstore'; +import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert'; import { registerUploadedDocument } from '@/lib/server/documents/register-upload'; import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorResponse } from '@/lib/server/errors/next-response'; @@ -133,23 +136,48 @@ async function finalizeOne(input: { throw new Error('Temporary upload expired before finalize'); } - const documentId = createHash('sha256').update(temp.body).digest('hex'); + const isDocxUpload = input.upload.type === 'docx'; + const finalizedType: DocumentType = isDocxUpload ? 'pdf' : input.upload.type; + const finalizedBody = isDocxUpload + ? await convertDocxBufferToPdfBuffer(temp.body) + : temp.body; + const finalizedContentType = finalizedType === 'pdf' + ? 'application/pdf' + : temp.contentType; + const finalizedName = finalizedType === 'pdf' && input.upload.type === 'docx' + ? safeDocumentName(`${path.parse(input.upload.name).name}.pdf`, 'upload.pdf') + : input.upload.name; + const documentId = createHash('sha256').update(finalizedBody).digest('hex'); try { await headDocumentBlob(documentId, input.namespace); } catch (error) { if (!isMissingBlobError(error)) throw error; - try { - await copyTempDocumentBlobToDocument( - input.upload.token, - input.userId, - documentId, - input.namespace, - temp.contentType, - { ifNoneMatch: true }, - ); - } catch (copyError) { - if (!isPreconditionFailed(copyError)) throw copyError; + if (!isDocxUpload) { + try { + await copyTempDocumentBlobToDocument( + input.upload.token, + input.userId, + documentId, + input.namespace, + finalizedContentType, + { ifNoneMatch: true }, + ); + } catch (copyError) { + if (!isPreconditionFailed(copyError)) throw copyError; + } + } else { + try { + await putDocumentBlob( + documentId, + finalizedBody, + finalizedContentType, + input.namespace, + { ifNoneMatch: true }, + ); + } catch (putError) { + if (!isPreconditionFailed(putError)) throw putError; + } } } @@ -158,9 +186,9 @@ async function finalizeOne(input: { documentId, userId: input.userId, namespace: input.namespace, - name: input.upload.name, - type: input.upload.type, - size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : temp.size, + name: finalizedName, + type: finalizedType, + size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength, lastModified: input.upload.lastModified, }); diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts deleted file mode 100644 index 51d1bf1..0000000 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ /dev/null @@ -1,225 +0,0 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { writeFile, mkdir, readFile, readdir, rm, stat } from 'fs/promises'; -import { spawn } from 'child_process'; -import path from 'path'; -import { existsSync } from 'fs'; -import { randomUUID, createHash } from 'crypto'; -import { pathToFileURL } from 'url'; -import { requireAuthContext } from '@/lib/server/auth/auth'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; -import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; -import { safeDocumentName } from '@/lib/server/documents/utils'; -import { enqueueDocumentPreview } from '@/lib/server/documents/previews'; -import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; -import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; -import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; -import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; -import { isS3Configured } from '@/lib/server/storage/s3'; -import { putDocumentBlob } from '@/lib/server/documents/blobstore'; -import { errorToLog, serverLogger } from '@/lib/server/logger'; -import { errorResponse } from '@/lib/server/errors/next-response'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core'; - -const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); -const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); - -async function ensureTempDir() { - if (!existsSync(DOCSTORE_DIR)) { - await mkdir(DOCSTORE_DIR, { recursive: true }); - } - if (!existsSync(TEMP_DIR)) { - await mkdir(TEMP_DIR, { recursive: true }); - } -} - -async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise { - return new Promise((resolve, reject) => { - const args: string[] = []; - if (profileDir) { - args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`); - } - args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath); - const proc = spawn('soffice', args); - - proc.on('error', (error) => reject(error)); - proc.on('close', (code) => { - if (code === 0) resolve(); - else reject(new Error(`LibreOffice conversion failed with code ${code}`)); - }); - }); -} - -async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100): Promise { - const end = Date.now() + timeoutMs; - while (Date.now() < end) { - const files = await readdir(dir); - const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf')); - if (pdf) { - const pdfPath = path.join(dir, pdf); - try { - const first = await stat(pdfPath); - await new Promise((res) => setTimeout(res, intervalMs)); - const second = await stat(pdfPath); - if (second.size > 0 && second.size === first.size) { - return pdfPath; - } - } catch { - // ignore transient errors - } - } - await new Promise((res) => setTimeout(res, intervalMs)); - } - throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`); -} - -export async function POST(req: NextRequest) { - try { - if (!isS3Configured()) { - return NextResponse.json( - { error: 'Documents storage is not configured. Set S3_* environment variables.' }, - { status: 503 }, - ); - } - - const testNamespace = getOpenReaderTestNamespace(req.headers); - - const ctxOrRes = await requireAuthContext(req); - if (ctxOrRes instanceof Response) return ctxOrRes; - if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - const storageUserId = ctxOrRes.userId; - - await ensureTempDir(); - - const formData = await req.formData(); - const file = formData.get('file'); - if (!(file instanceof File)) { - return NextResponse.json({ error: 'No file provided' }, { status: 400 }); - } - - if (!file.name.toLowerCase().endsWith('.docx')) { - return NextResponse.json({ error: 'File must be a .docx document' }, { status: 400 }); - } - - const docxBytes = Buffer.from(await file.arrayBuffer()); - // Keep stable IDs tied to source bytes. - const id = createHash('sha256').update(docxBytes).digest('hex'); - - const tempId = randomUUID(); - const jobDir = path.join(TEMP_DIR, tempId); - await mkdir(jobDir, { recursive: true }); - const profileDir = path.join(jobDir, 'lo-profile'); - await mkdir(profileDir, { recursive: true }); - const inputPath = path.join(jobDir, 'input.docx'); - - await writeFile(inputPath, docxBytes); - - try { - await convertDocxToPdf(inputPath, jobDir, profileDir); - const pdfPath = await waitForPdfReady(jobDir); - const pdfContent = await readFile(pdfPath); - - try { - await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace); - } catch (error) { - // Idempotent behavior: if blob already exists for this sha, continue. - const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; - const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed'; - if (!isPreconditionFailed) { - throw error; - } - } - - const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`); - const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now(); - const reusableParsedPdf = await findReusableParsedPdfResult(id); - const startedParse = reusableParsedPdf - ? null - : await startPdfParseOperation({ - documentId: id, - userId: storageUserId, - namespace: testNamespace, - }); - const parseState = stringifyDocumentParseState( - reusableParsedPdf - ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } - : startedParse!.parseState, - ); - - await db - .insert(documents) - .values({ - id, - userId: storageUserId, - name: derivedName, - type: 'pdf', - size: pdfContent.length, - lastModified, - filePath: id, - parseState, - parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, - }) - .onConflictDoUpdate({ - target: [documents.id, documents.userId], - set: { - name: derivedName, - type: 'pdf', - size: pdfContent.length, - lastModified, - filePath: id, - parseState, - parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null, - }, - }); - - await enqueueDocumentPreview( - { - id, - type: 'pdf', - lastModified, - }, - testNamespace, - ).catch((error) => { - serverLogger.warn({ - event: 'documents.docx.preview_enqueue.failed', - degraded: true, - fallbackPath: 'skip_preview_enqueue', - documentId: id, - error: errorToLog(error), - }, 'Failed to enqueue preview for converted DOCX'); - }); - - if (startedParse) { - enqueueParsePdfJob({ - documentId: id, - userId: storageUserId, - namespace: testNamespace, - initialOpId: startedParse.workerState.opId, - initialJobId: startedParse.workerState.jobId, - initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', - }); - } - - return NextResponse.json({ - stored: { - id, - name: derivedName, - type: 'pdf', - size: pdfContent.length, - lastModified, - }, - }); - } finally { - await rm(jobDir, { recursive: true, force: true }).catch(() => {}); - } - } catch (error) { - serverLogger.error({ - event: 'documents.docx.convert_upload.failed', - error: errorToLog(error), - }, 'Failed converting/uploading DOCX'); - return errorResponse(error, { - apiErrorMessage: 'Failed to convert document', - normalize: { code: 'DOCUMENTS_DOCX_CONVERT_UPLOAD_FAILED', errorClass: 'upstream' }, - }); - } -} diff --git a/src/components/doclist/DocumentList.tsx b/src/components/doclist/DocumentList.tsx index dfe1a0e..7349150 100644 --- a/src/components/doclist/DocumentList.tsx +++ b/src/components/doclist/DocumentList.tsx @@ -122,16 +122,14 @@ interface DocumentListInnerProps { function SidebarUploadLoader({ totalFiles, completedFiles, - phase, currentFileName, }: { totalFiles: number; completedFiles: number; - phase: 'uploading' | 'converting'; + phase: 'uploading'; currentFileName: string | null; }) { const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0; - const label = phase === 'converting' ? 'Converting' : 'Uploading'; const radius = 7; const stroke = 2; const size = 18; @@ -143,12 +141,12 @@ function SidebarUploadLoader({
- {label} + Uploading {completedFiles}/{totalFiles}
{progress}% - + sum + batch.totalFiles, 0); const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0); - const convertingBatch = batches.find((batch) => batch.phase === 'converting'); - const phase: 'uploading' | 'converting' = convertingBatch ? 'converting' : 'uploading'; - const currentFileName = convertingBatch?.currentFileName ?? batches.find((batch) => batch.currentFileName)?.currentFileName ?? null; - return { totalFiles, completedFiles, phase, currentFileName }; + const currentFileName = batches.find((batch) => batch.currentFileName)?.currentFileName ?? null; + return { totalFiles, completedFiles, phase: 'uploading' as const, currentFileName }; }, [activeUploadBatches]); const fallbackViewMode: ViewMode = viewMode; diff --git a/src/components/documents/DocumentUploader.tsx b/src/components/documents/DocumentUploader.tsx index 54d8859..49465d5 100644 --- a/src/components/documents/DocumentUploader.tsx +++ b/src/components/documents/DocumentUploader.tsx @@ -4,7 +4,6 @@ import { useState, useCallback, useId, type ReactNode } from 'react'; import { useDropzone } from 'react-dropzone'; import { UploadIcon } from '@/components/icons/Icons'; import { useDocuments } from '@/contexts/DocumentContext'; -import { uploadDocxAsPdf } from '@/lib/client/api/documents'; import { useFeatureFlag } from '@/contexts/RuntimeConfigContext'; import { dropzoneSurfaceClass } from '@/components/ui'; @@ -20,7 +19,7 @@ export interface UploadBatchState { isActive: boolean; totalFiles: number; completedFiles: number; - phase: 'uploading' | 'converting'; + phase: 'uploading'; currentFileName: string | null; } @@ -34,10 +33,8 @@ export function DocumentUploader({ const enableDocx = useFeatureFlag('enableDocxConversion'); const { uploadDocuments, - refreshDocuments, } = useDocuments(); const [isUploading, setIsUploading] = useState(false); - const [isConverting, setIsConverting] = useState(false); const [error, setError] = useState(null); const emitBatchState = useCallback((state: Omit) => { @@ -61,75 +58,24 @@ export function DocumentUploader({ }); try { - const docxFiles: File[] = []; - const regularFiles: File[] = []; + const uploadableFiles = acceptedFiles.filter((file) => + file.type === 'application/pdf' + || file.type === 'application/epub+zip' + || file.type === 'text/plain' + || file.type === 'text/markdown' + || file.name.endsWith('.md') + || (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), + ); - for (const file of acceptedFiles) { - if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { - docxFiles.push(file); - continue; - } - if ( - file.type === 'application/pdf' - || file.type === 'application/epub+zip' - || file.type === 'text/plain' - || file.type === 'text/markdown' - || file.name.endsWith('.md') - ) { - regularFiles.push(file); - } - } - - if (regularFiles.length > 0) { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: regularFiles[0]?.name ?? null, - }); - await uploadDocuments(regularFiles); - completedFiles += regularFiles.length; - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: null, - }); - } - - for (const file of docxFiles) { - // Preserve prior UX: show "Converting DOCX..." state rather than generic uploading. - setIsUploading(false); - setIsConverting(true); - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'converting', - currentFileName: file.name, - }); - await uploadDocxAsPdf(file); - await refreshDocuments(); - setIsConverting(false); - setIsUploading(true); - completedFiles += 1; - - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: null, - }); + if (uploadableFiles.length > 0) { + await uploadDocuments(uploadableFiles); + completedFiles = uploadableFiles.length; } } catch (err) { setError('Failed to upload file. Please try again.'); console.error('Upload error:', err); } finally { setIsUploading(false); - setIsConverting(false); emitBatchState({ isActive: false, totalFiles, @@ -138,7 +84,7 @@ export function DocumentUploader({ currentFileName: null, }); } - }, [uploadDocuments, refreshDocuments, enableDocx, emitBatchState]); + }, [uploadDocuments, enableDocx, emitBatchState]); const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, @@ -152,12 +98,12 @@ export function DocumentUploader({ } : {}) }, multiple: true, - disabled: isUploading || isConverting, + disabled: isUploading, noClick: variant === 'overlay', noKeyboard: variant === 'overlay' }); - const isDisabled = isUploading || isConverting; + const isDisabled = isUploading; if (variant === 'overlay') { const rootProps = getRootProps(); @@ -210,8 +156,6 @@ export function DocumentUploader({ {isUploading ? (

Uploading…

- ) : isConverting ? ( -

Converting DOCX…

) : (

@@ -226,8 +170,6 @@ export function DocumentUploader({ {isUploading ? (

Uploading file...

- ) : isConverting ? ( -

Converting DOCX to PDF...

) : ( <>

diff --git a/src/contexts/DocumentContext.tsx b/src/contexts/DocumentContext.tsx index 860fa91..d96dd4d 100644 --- a/src/contexts/DocumentContext.tsx +++ b/src/contexts/DocumentContext.tsx @@ -110,6 +110,26 @@ export function DocumentProvider({ children }: { children: ReactNode }) { stored.map(async (document, index) => { const file = files[index]; if (!file) return; + const sourceType = file.name + ? ( + file.name.toLowerCase().endsWith('.pdf') + ? 'pdf' + : file.name.toLowerCase().endsWith('.epub') + ? 'epub' + : file.name.toLowerCase().endsWith('.docx') + ? 'docx' + : 'html' + ) + : ( + file.type === 'application/pdf' + ? 'pdf' + : file.type === 'application/epub+zip' + ? 'epub' + : file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ? 'docx' + : 'html' + ); + if (document.type !== sourceType) return; await cacheStoredDocumentFromBytes(document, await file.arrayBuffer()); }), ); diff --git a/src/lib/client/api/documents.ts b/src/lib/client/api/documents.ts index 9a70cf5..f13a235 100644 --- a/src/lib/client/api/documents.ts +++ b/src/lib/client/api/documents.ts @@ -504,23 +504,3 @@ export async function getDocumentPreviewStatus( throw new Error(`Failed to load preview status (status ${res.status})`); } - -export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise { - const form = new FormData(); - form.append('file', file); - - const res = await fetch('/api/documents/docx-to-pdf/upload', { - method: 'POST', - body: form, - signal: options?.signal, - }); - - if (!res.ok) { - const data = (await res.json().catch(() => null)) as { error?: string } | null; - throw new Error(data?.error || 'Failed to convert DOCX'); - } - - const data = (await res.json()) as { stored: BaseDocument }; - if (!data?.stored) throw new Error('DOCX conversion succeeded but returned no document'); - return data.stored; -} diff --git a/src/lib/server/documents/docx-convert.ts b/src/lib/server/documents/docx-convert.ts new file mode 100644 index 0000000..67cf5cd --- /dev/null +++ b/src/lib/server/documents/docx-convert.ts @@ -0,0 +1,78 @@ +import { spawn } from 'child_process'; +import { existsSync } from 'fs'; +import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises'; +import path from 'path'; +import { randomUUID } from 'node:crypto'; +import { pathToFileURL } from 'url'; + +const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); +const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); + +async function ensureTempDir(): Promise { + if (!existsSync(DOCSTORE_DIR)) { + await mkdir(DOCSTORE_DIR, { recursive: true }); + } + if (!existsSync(TEMP_DIR)) { + await mkdir(TEMP_DIR, { recursive: true }); + } +} + +async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise { + return new Promise((resolve, reject) => { + const args: string[] = []; + if (profileDir) { + args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`); + } + args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath); + const proc = spawn('soffice', args); + + proc.on('error', (error) => reject(error)); + proc.on('close', (code) => { + if (code === 0) resolve(); + else reject(new Error(`LibreOffice conversion failed with code ${code}`)); + }); + }); +} + +async function waitForPdfReady(dir: string, timeoutMs = 20_000, intervalMs = 100): Promise { + const end = Date.now() + timeoutMs; + while (Date.now() < end) { + const files = await readdir(dir); + const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf')); + if (pdf) { + const pdfPath = path.join(dir, pdf); + try { + const first = await stat(pdfPath); + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + const second = await stat(pdfPath); + if (second.size > 0 && second.size === first.size) { + return pdfPath; + } + } catch { + // Ignore transient filesystem races while LibreOffice is still writing. + } + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`); +} + +export async function convertDocxBufferToPdfBuffer(docxBytes: Buffer): Promise { + await ensureTempDir(); + + const tempId = randomUUID(); + const jobDir = path.join(TEMP_DIR, tempId); + await mkdir(jobDir, { recursive: true }); + const profileDir = path.join(jobDir, 'lo-profile'); + await mkdir(profileDir, { recursive: true }); + const inputPath = path.join(jobDir, 'input.docx'); + + try { + await writeFile(inputPath, docxBytes); + await convertDocxToPdf(inputPath, jobDir, profileDir); + const pdfPath = await waitForPdfReady(jobDir); + return await readFile(pdfPath); + } finally { + await rm(jobDir, { recursive: true, force: true }).catch(() => {}); + } +} diff --git a/src/types/documents.ts b/src/types/documents.ts index b50bd40..ac870d5 100644 --- a/src/types/documents.ts +++ b/src/types/documents.ts @@ -11,7 +11,6 @@ export interface BaseDocument { parsedJsonKey?: string | null; scope?: 'user'; folderId?: string; - isConverting?: boolean; } export interface PDFDocument extends BaseDocument { diff --git a/tests/helpers.ts b/tests/helpers.ts index 3a7835d..44c8bd8 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -1,6 +1,4 @@ import { Page, expect, type TestInfo, type Locator } from '@playwright/test'; -import fs from 'fs'; -import path from 'path'; import { createHash } from 'crypto'; const DIR = './tests/files/'; @@ -10,14 +8,6 @@ function escapeRegExp(input: string) { return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } -function fixturePath(fileName: string) { - return path.join(__dirname, 'files', fileName); -} - -function sha256HexOfFile(filePath: string) { - return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); -} - async function waitForPdfViewerReady(page: Page, timeout = 60000) { await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) }); const loader = page.getByTestId('pdf-status-loader').first(); @@ -40,7 +30,7 @@ async function waitForPdfViewerReady(page: Page, timeout = 60000) { } /** - * Upload a sample epub or pdf + * Upload a sample document fixture */ export async function uploadFile(page: Page, filePath: string) { const input = page.locator('input[type=file]').first(); @@ -50,7 +40,7 @@ export async function uploadFile(page: Page, filePath: string) { await input.setInputFiles(`${DIR}${filePath}`); // Wait for the uploader to finish processing. The input is disabled while - // uploading/converting via react-dropzone's `disabled` prop. + // uploading via react-dropzone's `disabled` prop. // Tolerate extremely fast operations where the disabled state may be missed. try { await expect(input).toBeDisabled({ timeout: 2000 }); @@ -69,14 +59,8 @@ export async function uploadAndDisplay(page: Page, fileName: string) { const lower = fileName.toLowerCase(); if (lower.endsWith('.docx')) { - // Best-effort: conversion can complete before we observe this UI state. - try { - await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 }); - } catch { - // ignore - } - const expectedId = sha256HexOfFile(fixturePath(fileName)); - const targetLink = page.locator(`a[href$="/pdf/${expectedId}"]`).first(); + const convertedName = `${fileName.replace(/\.[^.]+$/, '')}.pdf`; + const targetLink = page.getByRole('link', { name: new RegExp(escapeRegExp(convertedName), 'i') }).first(); await expect(targetLink).toBeVisible({ timeout: 15000 }); await dismissOnboardingModals(page); await targetLink.click(); diff --git a/tests/unit/blob-upload-finalize-docx.vitest.spec.ts b/tests/unit/blob-upload-finalize-docx.vitest.spec.ts new file mode 100644 index 0000000..b818d66 --- /dev/null +++ b/tests/unit/blob-upload-finalize-docx.vitest.spec.ts @@ -0,0 +1,150 @@ +import { createHash } from 'node:crypto'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const hoisted = vi.hoisted(() => ({ + requireAuthContext: vi.fn(), + registerUploadedDocument: vi.fn(), + convertDocxBufferToPdfBuffer: vi.fn(), + headTempDocumentBlob: vi.fn(), + getTempDocumentBlob: vi.fn(), + getTempDocumentFinalizeReceipt: vi.fn(), + putTempDocumentFinalizeReceipt: vi.fn(), + deleteTempDocumentUpload: vi.fn(), + headDocumentBlob: vi.fn(), + copyTempDocumentBlobToDocument: vi.fn(), + putDocumentBlob: vi.fn(), +})); + +vi.mock('@/lib/server/auth/auth', () => ({ + requireAuthContext: hoisted.requireAuthContext, +})); + +vi.mock('@/lib/server/documents/register-upload', () => ({ + registerUploadedDocument: hoisted.registerUploadedDocument, +})); + +vi.mock('@/lib/server/documents/docx-convert', () => ({ + convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer, +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + TEMP_DOCUMENT_UPLOAD_TTL_MS: 24 * 60 * 60 * 1000, + copyTempDocumentBlobToDocument: hoisted.copyTempDocumentBlobToDocument, + deleteTempDocumentUpload: hoisted.deleteTempDocumentUpload, + getTempDocumentBlob: hoisted.getTempDocumentBlob, + getTempDocumentFinalizeReceipt: hoisted.getTempDocumentFinalizeReceipt, + headDocumentBlob: hoisted.headDocumentBlob, + headTempDocumentBlob: hoisted.headTempDocumentBlob, + isMissingBlobError: vi.fn((error: unknown) => { + const maybe = error as { code?: string } | undefined; + return maybe?.code === 'NoSuchKey'; + }), + isPreconditionFailed: vi.fn(() => false), + isValidTempUploadToken: vi.fn(() => true), + putDocumentBlob: hoisted.putDocumentBlob, + putTempDocumentFinalizeReceipt: hoisted.putTempDocumentFinalizeReceipt, +})); + +vi.mock('@/lib/server/testing/test-namespace', () => ({ + getOpenReaderTestNamespace: vi.fn(() => null), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + isS3Configured: vi.fn(() => true), +})); + +vi.mock('@/lib/server/logger', () => ({ + errorToLog: vi.fn((error: unknown) => error), + serverLogger: { + warn: vi.fn(), + error: vi.fn(), + }, +})); + +describe('POST /api/documents/blob/upload/finalize DOCX flow', () => { + beforeEach(() => { + hoisted.requireAuthContext.mockReset(); + hoisted.registerUploadedDocument.mockReset(); + hoisted.convertDocxBufferToPdfBuffer.mockReset(); + hoisted.headTempDocumentBlob.mockReset(); + hoisted.getTempDocumentBlob.mockReset(); + hoisted.getTempDocumentFinalizeReceipt.mockReset(); + hoisted.putTempDocumentFinalizeReceipt.mockReset(); + hoisted.deleteTempDocumentUpload.mockReset(); + hoisted.headDocumentBlob.mockReset(); + hoisted.copyTempDocumentBlobToDocument.mockReset(); + hoisted.putDocumentBlob.mockReset(); + + hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' }); + hoisted.getTempDocumentFinalizeReceipt.mockResolvedValue(null); + hoisted.headTempDocumentBlob.mockResolvedValue({ + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + contentLength: 12, + lastModified: Date.now(), + }); + hoisted.getTempDocumentBlob.mockResolvedValue(Buffer.from('docx-bytes')); + hoisted.convertDocxBufferToPdfBuffer.mockResolvedValue(Buffer.from('pdf-bytes')); + hoisted.headDocumentBlob + .mockRejectedValueOnce({ code: 'NoSuchKey' }) + .mockResolvedValue({ + contentLength: Buffer.byteLength('pdf-bytes'), + contentType: 'application/pdf', + eTag: 'etag-1', + }); + hoisted.registerUploadedDocument.mockImplementation(async (input: { documentId: string; name: string; type: string; size: number; lastModified: number }) => ({ + id: input.documentId, + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + scope: 'user', + })); + hoisted.putTempDocumentFinalizeReceipt.mockResolvedValue(undefined); + hoisted.deleteTempDocumentUpload.mockResolvedValue(undefined); + }); + + test('converts raw DOCX during finalize and registers a PDF', async () => { + const { POST } = await import('../../src/app/api/documents/blob/upload/finalize/route'); + const request = new NextRequest('http://localhost/api/documents/blob/upload/finalize', { + method: 'POST', + body: JSON.stringify({ + uploads: [{ + token: '123e4567-e89b-12d3-a456-426614174000', + name: 'Report.docx', + type: 'docx', + lastModified: 1700000000000, + }], + }), + headers: { + 'Content-Type': 'application/json', + }, + }); + + const response = await POST(request); + const expectedId = createHash('sha256').update(Buffer.from('pdf-bytes')).digest('hex'); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + stored: [{ + id: expectedId, + name: 'Report.pdf', + type: 'pdf', + }], + }); + expect(hoisted.convertDocxBufferToPdfBuffer).toHaveBeenCalledTimes(1); + expect(hoisted.putDocumentBlob).toHaveBeenCalledWith( + expectedId, + Buffer.from('pdf-bytes'), + 'application/pdf', + null, + { ifNoneMatch: true }, + ); + expect(hoisted.copyTempDocumentBlobToDocument).not.toHaveBeenCalled(); + expect(hoisted.registerUploadedDocument).toHaveBeenCalledWith(expect.objectContaining({ + documentId: expectedId, + name: 'Report.pdf', + type: 'pdf', + })); + }); +}); diff --git a/tests/upload.spec.ts b/tests/upload.spec.ts index 7a6ef51..dad3ae0 100644 --- a/tests/upload.spec.ts +++ b/tests/upload.spec.ts @@ -100,13 +100,7 @@ test.describe('Document Upload Tests', () => { test('uploads and converts a DOCX document', async ({ page }) => { await uploadFile(page, 'sample.docx'); - // Should see the converting message (best-effort; conversion may complete extremely fast) - try { - await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 }); - } catch { - // ignore - } - // After conversion, should see the PDF with the same name + // DOCX uploads are normalized into stored PDFs with the same basename. await expectDocumentListed(page, 'sample.pdf'); }); From c59052a03e3d6abd459f634a7eff0cdc04e9673c Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 4 Jun 2026 13:20:47 -0600 Subject: [PATCH 9/9] fix(documents): allow all accepted files for upload and add docx conversion timeout Update DocumentUploader to upload all accepted files without filtering by type, enabling consistent handling of DOCX and other supported formats. Add a default timeout for DOCX-to-PDF conversion to prevent hanging LibreOffice processes. This improves reliability and ensures uploads do not silently stall. --- src/components/documents/DocumentUploader.tsx | 17 +++-------------- src/lib/server/documents/docx-convert.ts | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/components/documents/DocumentUploader.tsx b/src/components/documents/DocumentUploader.tsx index 49465d5..ffa1b27 100644 --- a/src/components/documents/DocumentUploader.tsx +++ b/src/components/documents/DocumentUploader.tsx @@ -58,19 +58,8 @@ export function DocumentUploader({ }); try { - const uploadableFiles = acceptedFiles.filter((file) => - file.type === 'application/pdf' - || file.type === 'application/epub+zip' - || file.type === 'text/plain' - || file.type === 'text/markdown' - || file.name.endsWith('.md') - || (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), - ); - - if (uploadableFiles.length > 0) { - await uploadDocuments(uploadableFiles); - completedFiles = uploadableFiles.length; - } + await uploadDocuments(acceptedFiles); + completedFiles = acceptedFiles.length; } catch (err) { setError('Failed to upload file. Please try again.'); console.error('Upload error:', err); @@ -84,7 +73,7 @@ export function DocumentUploader({ currentFileName: null, }); } - }, [uploadDocuments, enableDocx, emitBatchState]); + }, [uploadDocuments, emitBatchState]); const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, diff --git a/src/lib/server/documents/docx-convert.ts b/src/lib/server/documents/docx-convert.ts index 67cf5cd..1c940a1 100644 --- a/src/lib/server/documents/docx-convert.ts +++ b/src/lib/server/documents/docx-convert.ts @@ -7,6 +7,7 @@ import { pathToFileURL } from 'url'; const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); +const DEFAULT_DOCX_CONVERSION_TIMEOUT_MS = 60_000; async function ensureTempDir(): Promise { if (!existsSync(DOCSTORE_DIR)) { @@ -25,9 +26,24 @@ async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir } args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath); const proc = spawn('soffice', args); + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + settled = true; + proc.kill(); + reject(new Error(`LibreOffice conversion timed out after ${DEFAULT_DOCX_CONVERSION_TIMEOUT_MS}ms`)); + }, DEFAULT_DOCX_CONVERSION_TIMEOUT_MS); - proc.on('error', (error) => reject(error)); + proc.on('error', (error) => { + clearTimeout(timeout); + if (settled) return; + settled = true; + reject(error); + }); proc.on('close', (code) => { + clearTimeout(timeout); + if (settled) return; + settled = true; if (code === 0) resolve(); else reject(new Error(`LibreOffice conversion failed with code ${code}`)); });