diff --git a/compute/core/src/index.ts b/compute/core/src/index.ts index 4ba1e89..4360a58 100644 --- a/compute/core/src/index.ts +++ b/compute/core/src/index.ts @@ -16,6 +16,7 @@ export { } from './config/timeout'; export { renderPage } from './pdf/render'; export { mergeTextWithRegions } from './pdf/merge'; +export { PDF_PARSER_VERSION } from './pdf/parser-version'; export { stitchCrossPageBlocks } from './pdf/stitch'; export { normalizeTextItemsForLayout } from './pdf/normalize-text'; export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map'; diff --git a/compute/core/src/pdf/parse.ts b/compute/core/src/pdf/parse.ts index 4cf0cd3..f0bde04 100644 --- a/compute/core/src/pdf/parse.ts +++ b/compute/core/src/pdf/parse.ts @@ -4,6 +4,7 @@ import { ensureModel } from './model'; import { runLayoutModel } from './runLayoutModel'; import { mergeTextWithRegions } from './merge'; import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime'; +import { PDF_PARSER_VERSION } from './parser-version'; import { stitchCrossPageBlocks } from './stitch'; import { renderPage } from './render'; import { normalizeTextItemsForLayout } from './normalize-text'; @@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise const doc: ParsedPdfDocument = { schemaVersion: 1, documentId: input.documentId, - parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69', + parserVersion: PDF_PARSER_VERSION, parsedAt: Date.now(), pages, }; diff --git a/compute/core/src/pdf/parser-version.ts b/compute/core/src/pdf/parser-version.ts new file mode 100644 index 0000000..b5fa7d7 --- /dev/null +++ b/compute/core/src/pdf/parser-version.ts @@ -0,0 +1 @@ +export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69'; diff --git a/compute/core/src/pdf/pdfjs-runtime.ts b/compute/core/src/pdf/pdfjs-runtime.ts index 206e650..233d9c3 100644 --- a/compute/core/src/pdf/pdfjs-runtime.ts +++ b/compute/core/src/pdf/pdfjs-runtime.ts @@ -1,12 +1,15 @@ import fs from 'node:fs'; +import { createRequire } from 'node:module'; import path from 'node:path'; let cachedStandardFontDataUrl: string | null = null; +const require = createRequire(import.meta.url); export function resolvePdfjsStandardFontDataUrl(): string { if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl; - const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts'); + const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json')); + const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts'); if (!fs.existsSync(standardFontDir)) { throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`); diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index 4a25c2e..b08d4d4 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -31,6 +31,7 @@ export default function EPUBPage() { const { setCurrentDocument, currDocName, + isPlaybackReady, clearCurrDoc, createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter, @@ -242,7 +243,7 @@ export default function EPUBPage() { ) : ( - + )} Promise; clearCurrDoc: () => void; extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise; @@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [isPlaybackReady, setIsPlaybackReady] = useState(false); // Mirror state into a ref so resolveEpubLocator (registered once with // TTSContext via a stable callback) can always read the latest page text // without forcing re-registration on every page turn. @@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { setCurrDocData(undefined); setCurrDocName(undefined); setCurrDocText(undefined); + setIsPlaybackReady(false); setCurrDocPages(undefined); isEPUBSetOnce.current = false; shouldPauseRef.current = true; @@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { */ const setCurrentDocument = useCallback(async (id: string): Promise => { try { + setIsPlaybackReady(false); const meta = await getDocumentMetadata(id); if (!meta) { clearCurrDoc(); @@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { */ const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise => { try { + setIsPlaybackReady(false); const location = rendition?.location; if (!location) return ''; const { start, end } = location; @@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { nextText: continuationPreview }); setCurrDocText(textContent); + setIsPlaybackReady(true); return textContent; } catch (error) { @@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, clearCurrDoc, extractPageText, walkUpcomingRenderedLocations, @@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, clearCurrDoc, extractPageText, walkUpcomingRenderedLocations, diff --git a/src/app/(app)/html/[id]/page.tsx b/src/app/(app)/html/[id]/page.tsx index f0193a3..09a0b45 100644 --- a/src/app/(app)/html/[id]/page.tsx +++ b/src/app/(app)/html/[id]/page.tsx @@ -31,6 +31,7 @@ export default function HTMLPage() { setCurrentDocument, currDocData, currDocName, + isPlaybackReady, blocks, isTxt, clearCurrDoc, @@ -233,7 +234,7 @@ export default function HTMLPage() { ) : ( - + )} Promise; @@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState { const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); + const [isPlaybackReady, setIsPlaybackReady] = useState(false); const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]); const blocks = useMemo( @@ -91,22 +93,43 @@ export function useHtmlDocument(): HtmlDocumentState { // sentence splitting + sequential advancement from there. const lastFedDocRef = useRef(null); useEffect(() => { - if (!currDocText) return; - const key = `${currDocData ?? ''}::${currDocText.length}`; - if (lastFedDocRef.current === key) return; + if (currDocData === undefined) { + lastFedDocRef.current = null; + setTTSText(''); + setIsPlaybackReady(false); + return; + } + if (!currDocText) { + lastFedDocRef.current = null; + setTTSText(''); + setIsPlaybackReady(true); + return; + } + const key = `${currDocName ?? ''}::${currDocData ?? ''}::${currDocText.length}`; + if (lastFedDocRef.current === key) { + setIsPlaybackReady(true); + return; + } + setIsPlaybackReady(false); lastFedDocRef.current = key; setTTSText(currDocText); - }, [currDocText, currDocData, setTTSText]); + setIsPlaybackReady(true); + }, [currDocName, currDocText, currDocData, setTTSText]); const clearCurrDoc = useCallback(() => { setCurrDocData(undefined); setCurrDocName(undefined); + setIsPlaybackReady(false); lastFedDocRef.current = null; + setTTSText(''); stop(); - }, [stop]); + }, [setTTSText, stop]); const setCurrentDocument = useCallback(async (id: string): Promise => { try { + setIsPlaybackReady(false); + lastFedDocRef.current = null; + setTTSText(''); const meta = await getDocumentMetadata(id); if (!meta) { console.error('Document not found on server'); @@ -125,7 +148,7 @@ export function useHtmlDocument(): HtmlDocumentState { console.error('Failed to get HTML document:', error); clearCurrDoc(); } - }, [clearCurrDoc]); + }, [clearCurrDoc, setTTSText]); const audiobookAdapter = useMemo( () => @@ -207,6 +230,7 @@ export function useHtmlDocument(): HtmlDocumentState { currDocData, currDocName, currDocText, + isPlaybackReady, blocks, isTxt, setCurrentDocument, @@ -218,6 +242,7 @@ export function useHtmlDocument(): HtmlDocumentState { currDocData, currDocName, currDocText, + isPlaybackReady, blocks, isTxt, setCurrentDocument, diff --git a/src/app/(app)/pdf/[id]/page.tsx b/src/app/(app)/pdf/[id]/page.tsx index 79d4060..6cfe8cf 100644 --- a/src/app/(app)/pdf/[id]/page.tsx +++ b/src/app/(app)/pdf/[id]/page.tsx @@ -52,6 +52,7 @@ export default function PDFViewerPage() { clearCurrDoc, currDocPage, currDocPages, + isPlaybackReady, parseStatus, parseProgress, documentSettings, @@ -487,7 +488,7 @@ export default function PDFViewerPage() { ) : isParseReady ? ( - + ) : null} (); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [isPlaybackReady, setIsPlaybackReady] = useState(false); const [pdfDocument, setPdfDocument] = useState(); const [parsedDocument, setParsedDocument] = useState(null); const [parseStatus, setParseStatus] = useState(null); @@ -167,52 +169,29 @@ export function usePdfDocument(): PdfDocumentState { // Guards for setCurrentDocument to prevent stale loads from overwriting newer selections. const docLoadSeqRef = useRef(0); const docLoadAbortRef = useRef(null); - const parsePollAbortRef = useRef(null); + const parseStreamAbortRef = useRef(null); const parseSseCloseRef = useRef<(() => void) | null>(null); + const lastPreparedPlaybackPageRef = useRef(null); - const fetchParsedDocument = useCallback(async ( + const loadParsedDocumentOnce = useCallback(async ( documentId: string, - initialStatus: PdfParseStatus | null, signal: AbortSignal, - initialOpId?: string | null, ): Promise => { - // Legacy PDFs may have null parseStatus; treat as pending so opening the - // document backfills parse output via the parsed endpoint polling path. - const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending'; - setParseStatus(effectiveInitialStatus); + if (signal.aborted) return; + const parsed = await getParsedPdfDocument(documentId, { signal }); + if (signal.aborted) return; + setParsedDocument(parsed); + setParseStatus('ready'); setParseProgress(null); - const delayMs = 1200; - const retryFailed = effectiveInitialStatus === 'failed'; - let attempt = 0; - let effectiveOpId = initialOpId?.trim() || null; - while (!signal.aborted) { - if (signal.aborted) return; - const result = await getParsedPdfDocument(documentId, { - signal, - retryFailed: retryFailed && attempt === 0, - ...(effectiveOpId ? { opId: effectiveOpId } : {}), - }); - if (result.status === 'ready') { - setParsedDocument(result.parsed); - setParseStatus('ready'); - setParseProgress(null); - setActiveParseOpId(null); - return; - } - if ('opId' in result && typeof result.opId === 'string' && result.opId.trim()) { - effectiveOpId = result.opId.trim(); - setActiveParseOpId(effectiveOpId); - } - setParseStatus(result.status); - setParseProgress(result.parseProgress ?? null); - if (result.status === 'failed') { - setParsedDocument(null); - setActiveParseOpId(null); - return; - } - attempt += 1; - await new Promise((resolve) => setTimeout(resolve, delayMs)); - } + setActiveParseOpId(null); + }, []); + + const resetParsedDocumentState = useCallback(() => { + setParsedDocument(null); + setCurrDocText(undefined); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; + pageTextCacheRef.current.clear(); }, []); const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise => { @@ -226,42 +205,57 @@ export function usePdfDocument(): PdfDocumentState { } }, []); - const startParsedPolling = useCallback((documentId: string, initialOpId?: string | null) => { - parsePollAbortRef.current?.abort(); + const startParsedEventStream = useCallback((documentId: string, initialOpId?: string | null) => { + parseStreamAbortRef.current?.abort(); parseSseCloseRef.current?.(); parseSseCloseRef.current = null; setParseProgress(null); setActiveParseOpId(initialOpId?.trim() || null); const controller = new AbortController(); - parsePollAbortRef.current = controller; + parseStreamAbortRef.current = controller; + let isResolvingTerminalState = false; const closeSse = subscribeParsedPdfDocumentEvents(documentId, { opId: initialOpId?.trim() || null, }, { onSnapshot: (snapshot) => { if (controller.signal.aborted) return; + if (isResolvingTerminalState) return; if (typeof snapshot.opId === 'string' && snapshot.opId.trim()) { setActiveParseOpId(snapshot.opId.trim()); } setParseStatus(snapshot.parseStatus); setParseProgress(snapshot.parseProgress); - if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') { - if (snapshot.parseStatus === 'failed') { - setParsedDocument(null); - setActiveParseOpId(null); - } else { - void fetchParsedDocument( - documentId, - 'ready', - controller.signal, - typeof snapshot.opId === 'string' ? snapshot.opId : (initialOpId ?? null), - ); - } + if (snapshot.parseStatus === 'ready') { + isResolvingTerminalState = true; + void (async () => { + try { + await loadParsedDocumentOnce(documentId, controller.signal); + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') return; + console.error('Failed to load parsed PDF after ready status:', error); + resetParsedDocumentState(); + } finally { + if (parseSseCloseRef.current === closeSse) { + closeSse(); + parseSseCloseRef.current = null; + } + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; + } + } + })(); + return; + } + if (snapshot.parseStatus === 'failed') { + isResolvingTerminalState = true; closeSse(); parseSseCloseRef.current = null; - if (parsePollAbortRef.current === controller) { - parsePollAbortRef.current = null; + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; } + resetParsedDocumentState(); + setActiveParseOpId(null); return; } }, @@ -281,11 +275,11 @@ export function usePdfDocument(): PdfDocumentState { if (parseSseCloseRef.current === closeSse) { parseSseCloseRef.current = null; } - if (parsePollAbortRef.current === controller) { - parsePollAbortRef.current = null; + if (parseStreamAbortRef.current === controller) { + parseStreamAbortRef.current = null; } }, { once: true }); - }, [fetchParsedDocument, setActiveParseOpId]); + }, [loadParsedDocumentOnce, resetParsedDocumentState, setActiveParseOpId]); useEffect(() => { pdfDocumentRef.current = pdfDocument; @@ -297,6 +291,7 @@ export function usePdfDocument(): PdfDocumentState { useEffect(() => { setCurrDocPage(currDocPageNumber); + setIsPlaybackReady(false); }, [currDocPageNumber]); /** @@ -324,11 +319,13 @@ export function usePdfDocument(): PdfDocumentState { if (!currentPdf) return; const seq = ++loadSeqRef.current; const pageNumber = currDocPageNumber; + setIsPlaybackReady(false); const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined => parsedDocument?.pages.find((page) => page.pageNumber === pageNum); if (parseStatus !== 'ready' || !parsedDocument) { + lastPreparedPlaybackPageRef.current = null; setCurrDocText(undefined); setTTSText('', { location: currDocPageNumber }); return; @@ -401,7 +398,8 @@ export function usePdfDocument(): PdfDocumentState { return; } - if (text !== currDocText || text === '') { + const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber; + if (shouldPreparePlayback) { setCurrDocText(text); const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber); setTTSText(text, { @@ -414,6 +412,8 @@ export function usePdfDocument(): PdfDocumentState { ...(sourceUnits.length > 0 ? { sourceUnits } : {}), }); } + lastPreparedPlaybackPageRef.current = currDocPageNumber; + setIsPlaybackReady(true); } catch (error) { if (error instanceof DOMException && error.name === 'AbortError') { return; @@ -461,14 +461,16 @@ export function usePdfDocument(): PdfDocumentState { // or fast refresh. pdfDocGenerationRef.current += 1; loadSeqRef.current += 1; - parsePollAbortRef.current?.abort(); - parsePollAbortRef.current = null; + parseStreamAbortRef.current?.abort(); + parseStreamAbortRef.current = null; parseSseCloseRef.current?.(); parseSseCloseRef.current = null; pageTextCacheRef.current.clear(); setPdfDocument(undefined); setCurrDocPages(undefined); setCurrDocText(undefined); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; setCurrDocId(id); setCurrDocName(undefined); setCurrDocData(undefined); @@ -489,7 +491,7 @@ export function usePdfDocument(): PdfDocumentState { setParseStatus(initialParseStatus); setParseProgress(null); setActiveParseOpId(null); - startParsedPolling(id, null); + startParsedEventStream(id, null); void fetchDocumentSettings(id, controller.signal); } @@ -524,7 +526,7 @@ export function usePdfDocument(): PdfDocumentState { setCurrDocText, setPdfDocument, fetchDocumentSettings, - startParsedPolling, + startParsedEventStream, ]); const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise => { @@ -546,14 +548,16 @@ export function usePdfDocument(): PdfDocumentState { pageTextCacheRef.current.clear(); setParsedDocument(null); setCurrDocText(undefined); + setIsPlaybackReady(false); + lastPreparedPlaybackPageRef.current = null; setParseStatus(forced.status); setParseProgress(null); setActiveParseOpId(forced.opId ?? null); - startParsedPolling(currDocId, forced.opId ?? null); + startParsedEventStream(currDocId, forced.opId ?? null); } catch (error) { console.error('Failed to force PDF reparse:', error); } - }, [currDocId, startParsedPolling]); + }, [currDocId, startParsedEventStream]); /** * Clears the current document state @@ -567,14 +571,15 @@ export function usePdfDocument(): PdfDocumentState { docLoadSeqRef.current += 1; docLoadAbortRef.current?.abort(); docLoadAbortRef.current = null; - parsePollAbortRef.current?.abort(); - parsePollAbortRef.current = null; + parseStreamAbortRef.current?.abort(); + parseStreamAbortRef.current = null; parseSseCloseRef.current?.(); parseSseCloseRef.current = null; setCurrDocId(undefined); setCurrDocName(undefined); setCurrDocData(undefined); setCurrDocText(undefined); + setIsPlaybackReady(false); setCurrDocPages(undefined); setPdfDocument(undefined); setParsedDocument(null); @@ -582,6 +587,7 @@ export function usePdfDocument(): PdfDocumentState { setParseProgress(null); setActiveParseOpId(null); setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS); + lastPreparedPlaybackPageRef.current = null; pageTextCacheRef.current.clear(); stop(); }, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]); @@ -681,6 +687,7 @@ export function usePdfDocument(): PdfDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, parsedDocument, parseStatus, parseProgress, @@ -708,6 +715,7 @@ export function usePdfDocument(): PdfDocumentState { currDocPages, currDocPage, currDocText, + isPlaybackReady, parsedDocument, parseStatus, parseProgress, diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index e1adc9d..3406252 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -5,10 +5,15 @@ import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; import { isAbortLikeError } from '@/lib/server/compute/abort-like-error'; -import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/server/compute/worker-parse-state'; +import { + isWorkerOperationStateStale, + mergeNonReadyParseSnapshot, + snapshotFromWorkerState, +} from '@/lib/server/compute/worker-parse-state'; import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { + normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, stringifyDocumentParseState, @@ -67,7 +72,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr const state = await healStaleDocumentParseState({ documentId: row.id, userId: row.userId, - state: parseDocumentParseState(row.parseState), + state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)), }); const parseStatus = normalizeParseStatus(state.status); const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null; @@ -82,13 +87,20 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr && workerState.opId === opId && !isWorkerOperationStateStale(workerState, opStaleMs) ) { + const merged = mergeNonReadyParseSnapshot({ + parseStatus, + parseProgress: state.progress ?? null, + workerState, + }); + const workerSnapshot = snapshotFromWorkerState(workerState); + const fromWorker = workerSnapshot.parseStatus === 'pending' || workerSnapshot.parseStatus === 'running'; return { snapshot: { - ...snapshotFromWorkerState(workerState), + ...merged, opId: workerState.opId, }, opId: workerState.opId, - fromWorker: true, + fromWorker, }; } } @@ -199,7 +211,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string let initialState = await toSnapshotState(row, requestedOpId); if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') { - const state = parseDocumentParseState(row.parseState); + const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); const created = await backfillPendingPdfParseOperation({ documentId: id, userId: row.userId, @@ -261,22 +273,19 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string let current = initialState.snapshot; let signature = JSON.stringify(current); let currentOpId = requestedOpId ?? initialState.opId; - let currentFromWorker = initialState.fromWorker; let lastEventId: number | null = null; let loggedMissingOpId = false; const pinnedRequestedOp = requestedOpId ?? null; - const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot, fromWorker: boolean): boolean => { + const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot): boolean => { const isTerminal = snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed'; if (!isTerminal) return false; - // If caller pinned an opId, keep streaming until worker confirms that - // op state. DB fallback can report stale terminal status for other rows. - if (pinnedRequestedOp && snapshot.opId === pinnedRequestedOp && !fromWorker) return false; + if (pinnedRequestedOp && snapshot.opId !== pinnedRequestedOp) return false; return true; }; writeSnapshot(current); - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } @@ -303,7 +312,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } current = next.snapshot; signature = next.signature; - currentFromWorker = next.fromWorker; if (!requestedOpId && next.opId !== currentOpId) { currentOpId = next.opId; lastEventId = null; @@ -312,7 +320,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string workerAbort = null; } } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); } }).catch((error) => { @@ -349,7 +357,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } current = next.snapshot; signature = next.signature; - currentFromWorker = next.fromWorker; currentOpId = requestedOpId ?? next.opId; if (!currentOpId) { if (!loggedMissingOpId) { @@ -357,7 +364,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string logger.warn({ event: 'documents.parsed.events.missing_opid_non_terminal', degraded: true, - step: 'poll_without_worker_op', + step: 'missing_opid_fallback', documentId: id, storageUserIdHash, parseStatus: current.parseStatus, @@ -367,7 +374,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } else if (loggedMissingOpId) { loggedMissingOpId = false; } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } @@ -471,19 +478,23 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string ); if (!workerSnapshot || workerSnapshot.opId !== currentOpId) continue; + const mergedSnapshot = mergeNonReadyParseSnapshot({ + parseStatus: current.parseStatus, + parseProgress: current.parseProgress, + workerState: workerSnapshot, + }); const nextSnapshot: ParsedSnapshot = { - ...snapshotFromWorkerState(workerSnapshot), + ...mergedSnapshot, opId: workerSnapshot.opId, }; const nextSignature = JSON.stringify(nextSnapshot); if (nextSignature !== signature) { current = nextSnapshot; signature = nextSignature; - currentFromWorker = true; writeSnapshot(current); } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } @@ -522,12 +533,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } current = next.snapshot; signature = next.signature; - currentFromWorker = next.fromWorker; if (!requestedOpId && next.opId !== currentOpId) { currentOpId = next.opId; lastEventId = null; } - if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) { + if (shouldCloseForTerminalSnapshot(current)) { closeStream(); return; } diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 650760c..9375ee2 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -4,39 +4,36 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; import { - documentParseStateFromWorkerState, isWorkerOperationStateStale, snapshotFromWorkerState, } from '@/lib/server/compute/worker-parse-state'; import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state'; import { - documentKey, getParsedDocumentBlob, getParsedDocumentBlobByKey, isMissingBlobError, isValidDocumentId, - putParsedDocumentBlob, } from '@/lib/server/documents/blobstore'; import { + normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, stringifyDocumentParseState, } from '@/lib/server/documents/parse-state'; -import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; -import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { checkJobRate, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; -import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger'; +import { createRequestLogger, hashForLog } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; -import { logDegraded } from '@/lib/server/errors/logging'; import type { ParsedPdfDocument } from '@/types/parsed-pdf'; import { getComputeOpStaleMs } from '@openreader/compute-core'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutJobResult } from '@openreader/compute-core/api-contracts'; export const dynamic = 'force-dynamic'; @@ -100,102 +97,12 @@ async function writeParseRowState(input: { .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); } -async function finalizeFromWorkerState(input: { - workerState: WorkerOperationState; - row: ParseRow; - namespace: string | null; - logger: ServerLogger; -}): Promise { - const snapshot = snapshotFromWorkerState(input.workerState); - const workerParseState = documentParseStateFromWorkerState(input.workerState); - - if (snapshot.parseStatus === 'pending' || snapshot.parseStatus === 'running') { - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState(workerParseState), - }); - return NextResponse.json({ - parseStatus: snapshot.parseStatus, - parseProgress: snapshot.parseProgress, - opId: input.workerState.opId, - }, { status: 202 }); - } - - if (snapshot.parseStatus === 'failed') { - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState(workerParseState), - }); - - return NextResponse.json({ - parseStatus: 'failed', - parseProgress: null, - opId: input.workerState.opId, - error: input.workerState.error?.message ?? 'Worker parse failed', - }, { status: 202 }); - } - - let parsedJsonKey: string | null = null; - if (input.workerState.result && 'parsedObjectKey' in input.workerState.result) { - parsedJsonKey = typeof input.workerState.result.parsedObjectKey === 'string' - ? input.workerState.result.parsedObjectKey - : null; - } - - if (!parsedJsonKey && input.workerState.result && 'parsed' in input.workerState.result && input.workerState.result.parsed) { - const parsedJson = Buffer.from(JSON.stringify(input.workerState.result.parsed)); - parsedJsonKey = await putParsedDocumentBlob(input.row.id, parsedJson, input.namespace); - } - - if (!parsedJsonKey) { - return errorResponse(new Error('Worker completed without parsed output'), { - apiErrorMessage: 'Worker completed without parsed output', - normalize: { code: 'DOCUMENTS_PARSED_WORKER_OUTPUT_MISSING', errorClass: 'upstream' }, - }); - } - - await writeParseRowState({ - documentId: input.row.id, - userId: input.row.userId, - parseState: stringifyDocumentParseState(workerParseState), - parsedJsonKey, - }); - - const json = await getParsedDocumentBlobByKey(parsedJsonKey); - let parsedDoc: ParsedPdfDocument | null = null; - try { - parsedDoc = JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument; - } catch { - parsedDoc = null; - } - - if (!hasAnyParsedBlocks(parsedDoc)) { - input.logger.warn({ - event: 'documents.parsed.no_blocks_in_output', - documentId: input.row.id, - userIdHash: hashForLog(input.row.userId), - parsedJsonKey, - }, 'Worker output parsed successfully but contained no blocks'); - } - - return new NextResponse(new Uint8Array(json), { - status: 200, - headers: { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - }, - }); -} - export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { const { logger } = createRequestLogger({ route: '/api/documents/[id]/parsed', request: req, }); try { - const opStaleMs = getComputeOpStaleMs(); if (!isS3Configured()) return s3NotConfiguredResponse(); const authCtxOrRes = await requireAuthContext(req); @@ -204,8 +111,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string const params = await ctx.params; const id = (params.id || '').trim().toLowerCase(); - const retryFailed = req.nextUrl.searchParams.get('retry') === '1'; - const requestedOpId = normalizeOpId(req.nextUrl.searchParams.get('opId')); if (!isValidDocumentId(id)) { return NextResponse.json({ error: 'Invalid document id' }, { status: 400 }); } @@ -220,112 +125,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - if (requestedOpId) { - const workerState = await fetchWorkerOperationState(requestedOpId); - if (workerState && workerState.opId === requestedOpId) { - return finalizeFromWorkerState({ - workerState, - row, - namespace: testNamespace, - logger, - }); - } - logDegraded(logger, { - event: 'documents.parsed.requested_op_unavailable', - msg: 'Requested worker operation id was unavailable', - step: 'requested_op_lookup', - context: { - documentId: id, - userIdHash: hashForLog(row.userId), - opId: requestedOpId, - error: { - name: 'RequestedOperationUnavailable', - message: `Requested worker operation ${requestedOpId} was unavailable`, - }, - }, - }); - } - - let state = parseDocumentParseState(row.parseState); - state = await healStaleDocumentParseState({ - documentId: id, - userId: row.userId, - state, - }); - + const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); const effectiveStatus = normalizeParseStatus(state.status); const effectiveProgress = state.progress ?? null; const effectiveOpId = normalizeOpId(state.opId); - if (effectiveOpId && effectiveStatus !== 'ready') { - const workerState = await fetchWorkerOperationState(effectiveOpId); - if ( - workerState - && workerState.opId === effectiveOpId - && !isWorkerOperationStateStale(workerState, opStaleMs) - ) { - return finalizeFromWorkerState({ - workerState, - row, - namespace: testNamespace, - logger, - }); - } - } - - if (!effectiveOpId && (effectiveStatus === 'pending' || effectiveStatus === 'running')) { - const created = await backfillPendingPdfParseOperation({ - documentId: id, - userId: row.userId, - namespace: testNamespace, - state, - }); - if (created) { - const snapshot = snapshotFromWorkerState(created); - await writeParseRowState({ - documentId: row.id, - userId: row.userId, - parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)), - }); - return NextResponse.json({ - parseStatus: snapshot.parseStatus, - parseProgress: snapshot.parseProgress, - opId: created.opId, - }, { status: 202 }); - } - } - - if (effectiveStatus === 'failed' && retryFailed) { - const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig); - if (!rateDecision.allowed) { - return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); - } - const created = await createOrReusePdfWorkerOperation({ - documentId: id, - namespace: testNamespace, - documentObjectKey: documentKey(id, testNamespace), - }); - await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); - const snapshot = snapshotFromWorkerState(created); - await writeParseRowState({ - documentId: row.id, - userId: row.userId, - parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)), - }); - return NextResponse.json({ - parseStatus: snapshot.parseStatus, - parseProgress: snapshot.parseProgress, - opId: created.opId, - }, { status: 202 }); - } - if (effectiveStatus !== 'ready') { return NextResponse.json({ parseStatus: effectiveStatus, parseProgress: effectiveProgress, opId: effectiveOpId, - }, { status: 202 }); + }, { status: 409 }); } try { @@ -409,7 +219,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - let state = parseDocumentParseState(row.parseState); + let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); state = await healStaleDocumentParseState({ documentId: id, userId: row.userId, @@ -441,25 +251,33 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); } - const created = await createOrReusePdfWorkerOperation({ + const forceToken = randomUUID(); + const startedParse = await startPdfParseOperation({ documentId: id, + userId: authCtxOrRes.userId, namespace: testNamespace, - documentObjectKey: documentKey(id, testNamespace), - forceToken: randomUUID(), + forceToken, }); - await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); - - const snapshot = snapshotFromWorkerState(created); + const snapshot = snapshotFromWorkerState(startedParse.workerState); await writeParseRowState({ documentId: row.id, userId: row.userId, - parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)), + parseState: stringifyDocumentParseState(startedParse.parseState), + }); + enqueueParsePdfJob({ + documentId: id, + userId: row.userId, + namespace: testNamespace, + forceToken, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', }); return NextResponse.json({ parseStatus: snapshot.parseStatus, parseProgress: snapshot.parseProgress, - opId: created.opId, + opId: startedParse.workerState.opId, }, { status: 202 }); } catch (error) { return errorResponse(error, { diff --git a/src/app/api/documents/blob/upload/fallback/route.ts b/src/app/api/documents/blob/upload/fallback/route.ts index 7fc8208..f6d7f15 100644 --- a/src/app/api/documents/blob/upload/fallback/route.ts +++ b/src/app/api/documents/blob/upload/fallback/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore'; +import { isValidTempUploadToken, putTempDocumentBlob } from '@/lib/server/documents/blobstore'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; @@ -26,11 +26,12 @@ export async function PUT(req: NextRequest) { const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); const url = new URL(req.url); - const id = (url.searchParams.get('id') || '').trim().toLowerCase(); - if (!isValidDocumentId(id)) { - return NextResponse.json({ error: 'Invalid document id' }, { status: 400 }); + const token = (url.searchParams.get('token') || '').trim().toLowerCase(); + if (!isValidTempUploadToken(token)) { + return NextResponse.json({ error: 'Invalid upload token' }, { status: 400 }); } const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream'; @@ -86,7 +87,7 @@ export async function PUT(req: NextRequest) { const namespace = getOpenReaderTestNamespace(req.headers); try { - await putDocumentBlob(id, body, contentType, namespace); + await putTempDocumentBlob(token, ctxOrRes.userId, body, contentType, namespace); } catch (error) { if (!isPreconditionFailed(error)) { throw error; @@ -97,12 +98,12 @@ export async function PUT(req: NextRequest) { event: 'documents.blob.upload.fallback.proxy_used', degraded: true, fallbackPath: 'upload_proxy', - documentId: id, + uploadToken: token, contentType, bytes: body.byteLength, }, 'Document upload fallback proxy used'); - return NextResponse.json({ success: true, id }); + return NextResponse.json({ success: true, token }); } catch (error) { serverLogger.error({ event: 'documents.blob.upload.fallback.failed', diff --git a/src/app/api/documents/blob/upload/finalize/route.ts b/src/app/api/documents/blob/upload/finalize/route.ts new file mode 100644 index 0000000..b1f5f6e --- /dev/null +++ b/src/app/api/documents/blob/upload/finalize/route.ts @@ -0,0 +1,257 @@ +import path from 'path'; +import { createHash } from 'node:crypto'; +import { NextRequest, NextResponse } from 'next/server'; +import { requireAuthContext } from '@/lib/server/auth/auth'; +import { + TEMP_DOCUMENT_UPLOAD_TTL_MS, + copyTempDocumentBlobToDocument, + deleteTempDocumentUpload, + getTempDocumentBlob, + getTempDocumentFinalizeReceipt, + headDocumentBlob, + headTempDocumentBlob, + isMissingBlobError, + isPreconditionFailed, + isValidTempUploadToken, + putDocumentBlob, + putTempDocumentFinalizeReceipt, +} from '@/lib/server/documents/blobstore'; +import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert'; +import { registerUploadedDocument } from '@/lib/server/documents/register-upload'; +import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils'; +import { errorResponse } from '@/lib/server/errors/next-response'; +import { errorToLog, serverLogger } from '@/lib/server/logger'; +import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import type { BaseDocument, DocumentType } from '@/types/documents'; + +export const dynamic = 'force-dynamic'; + +type FinalizeUpload = { + token: string; + name: string; + type: DocumentType; + lastModified: number; +}; + +type FinalizeReceipt = { + stored: BaseDocument; +}; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType { + if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') { + return rawType; + } + return toDocumentTypeFromName(safeName); +} + +function normalizeLastModified(value: unknown): number { + return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now(); +} + +function parseFinalizePayload(body: unknown): FinalizeUpload[] { + if (!body || typeof body !== 'object') return []; + const rawUploads = (body as { uploads?: unknown }).uploads; + if (!Array.isArray(rawUploads)) return []; + + const uploads: FinalizeUpload[] = []; + for (const rawUpload of rawUploads) { + if (!rawUpload || typeof rawUpload !== 'object') continue; + const rec = rawUpload as Record; + const token = typeof rec.token === 'string' ? rec.token.trim().toLowerCase() : ''; + if (!isValidTempUploadToken(token)) continue; + const fallbackName = `upload-${token}.txt`; + const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName); + uploads.push({ + token, + name, + type: normalizeDocumentType(rec.type, name), + lastModified: normalizeLastModified(rec.lastModified), + }); + } + return uploads; +} + +async function loadTempUpload(input: { + token: string; + userId: string; + namespace: string | null; +}): Promise<{ contentType: string; size: number; lastModified: number; body: Buffer }> { + const RETRIES = 3; + const RETRY_DELAY_MS = 500; + + let lastError: unknown = null; + for (let attempt = 0; attempt < RETRIES; attempt += 1) { + try { + const head = await headTempDocumentBlob(input.token, input.userId, input.namespace); + const body = await getTempDocumentBlob(input.token, input.userId, input.namespace); + return { + contentType: head.contentType ?? 'application/octet-stream', + size: head.contentLength > 0 ? head.contentLength : body.byteLength, + lastModified: head.lastModified ?? Date.now(), + body, + }; + } catch (error) { + lastError = error; + if (isMissingBlobError(error) && attempt < RETRIES - 1) { + await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS)); + continue; + } + throw error; + } + } + + throw lastError instanceof Error ? lastError : new Error('Temporary upload is unavailable'); +} + +async function finalizeOne(input: { + upload: FinalizeUpload; + userId: string; + namespace: string | null; +}): Promise { + const existingReceipt = await getTempDocumentFinalizeReceipt( + input.upload.token, + input.userId, + input.namespace, + ); + if (existingReceipt?.stored) { + return existingReceipt.stored; + } + + const temp = await loadTempUpload({ + token: input.upload.token, + userId: input.userId, + namespace: input.namespace, + }); + + if (Date.now() - temp.lastModified > TEMP_DOCUMENT_UPLOAD_TTL_MS) { + await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch(() => undefined); + throw new Error('Temporary upload expired before finalize'); + } + + const isDocxUpload = input.upload.type === 'docx'; + const finalizedType: DocumentType = isDocxUpload ? 'pdf' : input.upload.type; + const finalizedBody = isDocxUpload + ? await convertDocxBufferToPdfBuffer(temp.body) + : temp.body; + const finalizedContentType = finalizedType === 'pdf' + ? 'application/pdf' + : temp.contentType; + const finalizedName = finalizedType === 'pdf' && input.upload.type === 'docx' + ? safeDocumentName(`${path.parse(input.upload.name).name}.pdf`, 'upload.pdf') + : input.upload.name; + const documentId = createHash('sha256').update(finalizedBody).digest('hex'); + + try { + await headDocumentBlob(documentId, input.namespace); + } catch (error) { + if (!isMissingBlobError(error)) throw error; + if (!isDocxUpload) { + try { + await copyTempDocumentBlobToDocument( + input.upload.token, + input.userId, + documentId, + input.namespace, + finalizedContentType, + { ifNoneMatch: true }, + ); + } catch (copyError) { + if (!isPreconditionFailed(copyError)) throw copyError; + } + } else { + try { + await putDocumentBlob( + documentId, + finalizedBody, + finalizedContentType, + input.namespace, + { ifNoneMatch: true }, + ); + } catch (putError) { + if (!isPreconditionFailed(putError)) throw putError; + } + } + } + + const canonicalHead = await headDocumentBlob(documentId, input.namespace); + const stored = await registerUploadedDocument({ + documentId, + userId: input.userId, + namespace: input.namespace, + name: finalizedName, + type: finalizedType, + size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength, + lastModified: input.upload.lastModified, + }); + + await putTempDocumentFinalizeReceipt( + input.upload.token, + input.userId, + input.namespace, + Buffer.from(JSON.stringify({ stored }), 'utf8'), + ); + + await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch((error) => { + serverLogger.warn({ + event: 'documents.blob.upload.finalize.temp_delete_failed', + degraded: true, + fallbackPath: 'leave_temp_upload', + documentId, + token: input.upload.token, + error: errorToLog(error), + }, 'Failed to delete temp upload after finalize'); + }); + + return stored; +} + +export async function POST(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + const userId = ctxOrRes.userId; + + const namespace = getOpenReaderTestNamespace(req.headers); + const uploads = parseFinalizePayload(await req.json().catch(() => null)); + if (uploads.length === 0) { + return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 }); + } + + const stored = await Promise.all( + uploads.map((upload) => finalizeOne({ + upload, + userId, + namespace, + })), + ); + + return NextResponse.json({ stored }); + } catch (error) { + if (error instanceof Error && error.message === 'Temporary upload expired before finalize') { + return NextResponse.json({ error: error.message }, { status: 410 }); + } + if (isMissingBlobError(error)) { + return NextResponse.json({ error: 'Temporary upload missing. Upload bytes again and retry finalize.' }, { status: 409 }); + } + + serverLogger.error({ + event: 'documents.blob.upload.finalize.failed', + error: errorToLog(error), + }, 'Failed to finalize uploaded documents'); + return errorResponse(error, { + apiErrorMessage: 'Failed to finalize uploaded documents', + normalize: { code: 'DOCUMENTS_BLOB_UPLOAD_FINALIZE_FAILED', errorClass: 'storage' }, + }); + } +} diff --git a/src/app/api/documents/blob/upload/presign/route.ts b/src/app/api/documents/blob/upload/presign/route.ts index 6d91ee4..183074c 100644 --- a/src/app/api/documents/blob/upload/presign/route.ts +++ b/src/app/api/documents/blob/upload/presign/route.ts @@ -1,6 +1,11 @@ +import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore'; +import { + TEMP_DOCUMENT_UPLOAD_TTL_MS, + deleteExpiredTempDocumentUploads, + presignTempPut, +} from '@/lib/server/documents/blobstore'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; @@ -10,7 +15,6 @@ import { errorResponse } from '@/lib/server/errors/next-response'; export const dynamic = 'force-dynamic'; type PresignUpload = { - id: string; contentType: string; size: number; }; @@ -24,14 +28,12 @@ function parseUploads(body: unknown): PresignUpload[] { for (const raw of rawUploads) { if (!raw || typeof raw !== 'object') continue; const rec = raw as Record; - const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : ''; - if (!isValidDocumentId(id)) continue; const contentType = typeof rec.contentType === 'string' && rec.contentType.trim() ? rec.contentType.trim() : 'application/octet-stream'; const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0; - uploads.push({ id, contentType, size }); + uploads.push({ contentType, size }); } return uploads; } @@ -47,6 +49,8 @@ export async function POST(req: NextRequest) { const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + const userId = ctxOrRes.userId; const body = await req.json().catch(() => null); const uploads = parseUploads(body); @@ -68,13 +72,16 @@ export async function POST(req: NextRequest) { } const namespace = getOpenReaderTestNamespace(req.headers); + await deleteExpiredTempDocumentUploads(userId, namespace, Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS) + .catch(() => undefined); const signed = await Promise.all( uploads.map(async (upload) => { - const res = await presignPut(upload.id, upload.contentType, namespace, { + const token = randomUUID(); + const res = await presignTempPut(token, userId, upload.contentType, namespace, { contentLength: upload.size, }); return { - id: upload.id, + token, url: res.url, headers: res.headers, }; diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts deleted file mode 100644 index 7ba5623..0000000 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { writeFile, mkdir, readFile, readdir, rm, stat } from 'fs/promises'; -import { spawn } from 'child_process'; -import path from 'path'; -import { existsSync } from 'fs'; -import { randomUUID, createHash } from 'crypto'; -import { pathToFileURL } from 'url'; -import { requireAuthContext } from '@/lib/server/auth/auth'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; -import { safeDocumentName } from '@/lib/server/documents/utils'; -import { enqueueDocumentPreview } from '@/lib/server/documents/previews'; -import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; -import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; -import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; -import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; -import { isS3Configured } from '@/lib/server/storage/s3'; -import { putDocumentBlob } from '@/lib/server/documents/blobstore'; -import { errorToLog, serverLogger } from '@/lib/server/logger'; -import { errorResponse } from '@/lib/server/errors/next-response'; - -const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); -const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); - -async function ensureTempDir() { - if (!existsSync(DOCSTORE_DIR)) { - await mkdir(DOCSTORE_DIR, { recursive: true }); - } - if (!existsSync(TEMP_DIR)) { - await mkdir(TEMP_DIR, { recursive: true }); - } -} - -async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise { - return new Promise((resolve, reject) => { - const args: string[] = []; - if (profileDir) { - args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`); - } - args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath); - const proc = spawn('soffice', args); - - proc.on('error', (error) => reject(error)); - proc.on('close', (code) => { - if (code === 0) resolve(); - else reject(new Error(`LibreOffice conversion failed with code ${code}`)); - }); - }); -} - -async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100): Promise { - const end = Date.now() + timeoutMs; - while (Date.now() < end) { - const files = await readdir(dir); - const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf')); - if (pdf) { - const pdfPath = path.join(dir, pdf); - try { - const first = await stat(pdfPath); - await new Promise((res) => setTimeout(res, intervalMs)); - const second = await stat(pdfPath); - if (second.size > 0 && second.size === first.size) { - return pdfPath; - } - } catch { - // ignore transient errors - } - } - await new Promise((res) => setTimeout(res, intervalMs)); - } - throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`); -} - -export async function POST(req: NextRequest) { - try { - if (!isS3Configured()) { - return NextResponse.json( - { error: 'Documents storage is not configured. Set S3_* environment variables.' }, - { status: 503 }, - ); - } - - const testNamespace = getOpenReaderTestNamespace(req.headers); - - const ctxOrRes = await requireAuthContext(req); - if (ctxOrRes instanceof Response) return ctxOrRes; - if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - const storageUserId = ctxOrRes.userId; - - await ensureTempDir(); - - const formData = await req.formData(); - const file = formData.get('file'); - if (!(file instanceof File)) { - return NextResponse.json({ error: 'No file provided' }, { status: 400 }); - } - - if (!file.name.toLowerCase().endsWith('.docx')) { - return NextResponse.json({ error: 'File must be a .docx document' }, { status: 400 }); - } - - const docxBytes = Buffer.from(await file.arrayBuffer()); - // Keep stable IDs tied to source bytes. - const id = createHash('sha256').update(docxBytes).digest('hex'); - - const tempId = randomUUID(); - const jobDir = path.join(TEMP_DIR, tempId); - await mkdir(jobDir, { recursive: true }); - const profileDir = path.join(jobDir, 'lo-profile'); - await mkdir(profileDir, { recursive: true }); - const inputPath = path.join(jobDir, 'input.docx'); - - await writeFile(inputPath, docxBytes); - - try { - await convertDocxToPdf(inputPath, jobDir, profileDir); - const pdfPath = await waitForPdfReady(jobDir); - const pdfContent = await readFile(pdfPath); - - try { - await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace); - } catch (error) { - // Idempotent behavior: if blob already exists for this sha, continue. - const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; - const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed'; - if (!isPreconditionFailed) { - throw error; - } - } - - const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`); - const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now(); - - await db - .insert(documents) - .values({ - id, - userId: storageUserId, - name: derivedName, - type: 'pdf', - size: pdfContent.length, - lastModified, - filePath: id, - parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }), - parsedJsonKey: null, - }) - .onConflictDoUpdate({ - target: [documents.id, documents.userId], - set: { - name: derivedName, - type: 'pdf', - size: pdfContent.length, - lastModified, - filePath: id, - parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }), - parsedJsonKey: null, - }, - }); - - await enqueueDocumentPreview( - { - id, - type: 'pdf', - lastModified, - }, - testNamespace, - ).catch((error) => { - serverLogger.warn({ - event: 'documents.docx.preview_enqueue.failed', - degraded: true, - fallbackPath: 'skip_preview_enqueue', - documentId: id, - error: errorToLog(error), - }, 'Failed to enqueue preview for converted DOCX'); - }); - - // Record upload-driven parse load (see register route for rationale). - const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig); - enqueueParsePdfJob({ - documentId: id, - userId: storageUserId, - namespace: testNamespace, - }); - - return NextResponse.json({ - stored: { - id, - name: derivedName, - type: 'pdf', - size: pdfContent.length, - lastModified, - }, - }); - } finally { - await rm(jobDir, { recursive: true, force: true }).catch(() => {}); - } - } catch (error) { - serverLogger.error({ - event: 'documents.docx.convert_upload.failed', - error: errorToLog(error), - }, 'Failed converting/uploading DOCX'); - return errorResponse(error, { - apiErrorMessage: 'Failed to convert document', - normalize: { code: 'DOCUMENTS_DOCX_CONVERT_UPLOAD_FAILED', errorClass: 'upstream' }, - }); - } -} diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 903ab96..8002f7a 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,25 +1,20 @@ -import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { and, count, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils'; +import { toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; import { cleanupDocumentPreviewArtifacts, deleteDocumentPreviewRows, - enqueueDocumentPreview, } from '@/lib/server/documents/previews'; -import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; -import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; -import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; +import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; import { + normalizeDocumentParseStateForCurrentParserVersion, normalizeParseStatus, parseDocumentParseState, - stringifyDocumentParseState, } from '@/lib/server/documents/parse-state'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; @@ -27,14 +22,6 @@ import type { BaseDocument, DocumentType } from '@/types/documents'; export const dynamic = 'force-dynamic'; -type RegisterDocument = { - id: string; - name: string; - type: DocumentType; - size: number; - lastModified: number; -}; - function s3NotConfiguredResponse(): NextResponse { return NextResponse.json( { error: 'Documents storage is not configured. Set S3_* environment variables.' }, @@ -49,175 +36,6 @@ function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType return toDocumentTypeFromName(safeName); } -function normalizeLastModified(value: unknown): number { - return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now(); -} - -function parseDocumentPayload(body: unknown): RegisterDocument[] { - if (!body || typeof body !== 'object') return []; - const rawDocs = (body as { documents?: unknown }).documents; - if (!Array.isArray(rawDocs)) return []; - - const docs: RegisterDocument[] = []; - for (const rawDoc of rawDocs) { - if (!rawDoc || typeof rawDoc !== 'object') continue; - const rec = rawDoc as Record; - const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : ''; - if (!isValidDocumentId(id)) continue; - const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`; - const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName); - const type = normalizeDocumentType(rec.type, name); - const lastModified = normalizeLastModified(rec.lastModified); - const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0; - docs.push({ id, name, type, size, lastModified }); - } - return docs; -} - -export async function POST(req: NextRequest) { - try { - if (!isS3Configured()) return s3NotConfiguredResponse(); - - const ctxOrRes = await requireAuthContext(req); - if (ctxOrRes instanceof Response) return ctxOrRes; - if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - - const testNamespace = getOpenReaderTestNamespace(req.headers); - const storageUserId = ctxOrRes.userId; - - const body = await req.json().catch(() => null); - const documentsData = parseDocumentPayload(body); - if (documentsData.length === 0) { - return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 }); - } - - const stored: BaseDocument[] = []; - - // Resolve the parse rate-limit config once (only when a PDF is present). - const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf') - ? getPdfLayoutRateConfig(await getResolvedRuntimeConfig()) - : null; - - for (const doc of documentsData) { - let headSize = doc.size; - - // Retry HEAD check to handle S3 read-after-write propagation delays. - // The client uploads bytes directly to S3 via presigned URL, then - // immediately calls this endpoint. On serverless platforms the HEAD - // request may reach S3 before the PUT is visible. - const HEAD_RETRIES = 3; - const HEAD_RETRY_DELAY_MS = 500; - let headError: unknown = null; - for (let attempt = 0; attempt < HEAD_RETRIES; attempt++) { - headError = null; - try { - const head = await headDocumentBlob(doc.id, testNamespace); - if (head.contentLength > 0) headSize = head.contentLength; - break; - } catch (error) { - if (isMissingBlobError(error)) { - headError = error; - if (attempt < HEAD_RETRIES - 1) { - await new Promise((r) => setTimeout(r, HEAD_RETRY_DELAY_MS)); - continue; - } - } else { - throw error; - } - } - } - if (headError && isMissingBlobError(headError)) { - return NextResponse.json( - { - error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`, - }, - { status: 409 }, - ); - } - - await db - .insert(documents) - .values({ - id: doc.id, - userId: storageUserId, - name: doc.name, - type: doc.type, - size: headSize, - lastModified: doc.lastModified, - filePath: doc.id, - parseState: doc.type === 'pdf' - ? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }) - : null, - parsedJsonKey: null, - }) - .onConflictDoUpdate({ - target: [documents.id, documents.userId], - set: { - name: doc.name, - type: doc.type, - size: headSize, - lastModified: doc.lastModified, - filePath: doc.id, - parseState: doc.type === 'pdf' - ? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }) - : null, - parsedJsonKey: null, - }, - }); - - stored.push({ - id: doc.id, - name: doc.name, - type: doc.type, - size: headSize, - lastModified: doc.lastModified, - scope: 'user', - }); - - await enqueueDocumentPreview( - { - id: doc.id, - type: doc.type, - lastModified: doc.lastModified, - }, - testNamespace, - ).catch((error) => { - serverLogger.warn({ - event: 'documents.preview.enqueue.failed', - degraded: true, - fallbackPath: 'skip_preview_enqueue', - documentId: doc.id, - error: errorToLog(error), - }, 'Failed to enqueue document preview'); - }); - - if (doc.type === 'pdf') { - // Account for upload-driven parse load in the same ledger the explicit - // re-parse limiter reads. We record (not reject) here so a legitimate - // bulk upload always parses; the recorded load still throttles - // subsequent loopable re-parse spam via /parsed. - await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] }); - enqueueParsePdfJob({ - documentId: doc.id, - userId: storageUserId, - namespace: testNamespace, - }); - } - } - - return NextResponse.json({ success: true, stored }); - } catch (error) { - serverLogger.error({ - event: 'documents.register.failed', - error: errorToLog(error), - }, 'Failed to register documents'); - return errorResponse(error, { - apiErrorMessage: 'Failed to register documents', - normalize: { code: 'DOCUMENTS_REGISTER_FAILED', errorClass: 'db' }, - }); - } -} - export async function GET(req: NextRequest) { try { if (!isS3Configured()) return s3NotConfiguredResponse(); @@ -260,13 +78,16 @@ export async function GET(req: NextRequest) { const results: BaseDocument[] = rows.map((doc) => { const type = normalizeDocumentType(doc.type, doc.name); + const parseState = type === 'pdf' + ? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState)) + : null; return { id: doc.id, name: doc.name, size: Number(doc.size), lastModified: Number(doc.lastModified), type, - parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null, + parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null, parsedJsonKey: doc.parsedJsonKey, scope: 'user', }; diff --git a/src/components/doclist/DocumentList.tsx b/src/components/doclist/DocumentList.tsx index b854554..7349150 100644 --- a/src/components/doclist/DocumentList.tsx +++ b/src/components/doclist/DocumentList.tsx @@ -122,16 +122,14 @@ interface DocumentListInnerProps { function SidebarUploadLoader({ totalFiles, completedFiles, - phase, currentFileName, }: { totalFiles: number; completedFiles: number; - phase: 'uploading' | 'converting'; + phase: 'uploading'; currentFileName: string | null; }) { const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0; - const label = phase === 'converting' ? 'Converting' : 'Uploading'; const radius = 7; const stroke = 2; const size = 18; @@ -143,12 +141,12 @@ function SidebarUploadLoader({
- {label} + Uploading {completedFiles}/{totalFiles}
{progress}% - + { if (!documentToDelete) return; try { - if (documentToDelete.type === 'pdf') await removePDFDocument(documentToDelete.id); - else if (documentToDelete.type === 'epub') await removeEPUBDocument(documentToDelete.id); - else if (documentToDelete.type === 'html') await removeHTMLDocument(documentToDelete.id); + await deleteDocument(documentToDelete.id); setFolders((prev) => prev.map((f) => ({ ...f, @@ -451,7 +445,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) { } catch (err) { console.error('Failed to remove document:', err); } - }, [documentToDelete, removePDFDocument, removeEPUBDocument, removeHTMLDocument]); + }, [deleteDocument, documentToDelete]); const handleDeleteDoc = useCallback((doc: DocumentListDocument) => { setDocumentToDelete({ id: doc.id, name: doc.name, type: doc.type }); @@ -577,10 +571,8 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) { if (batches.length === 0) return null; const totalFiles = batches.reduce((sum, batch) => sum + batch.totalFiles, 0); const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0); - const convertingBatch = batches.find((batch) => batch.phase === 'converting'); - const phase: 'uploading' | 'converting' = convertingBatch ? 'converting' : 'uploading'; - const currentFileName = convertingBatch?.currentFileName ?? batches.find((batch) => batch.currentFileName)?.currentFileName ?? null; - return { totalFiles, completedFiles, phase, currentFileName }; + const currentFileName = batches.find((batch) => batch.currentFileName)?.currentFileName ?? null; + return { totalFiles, completedFiles, phase: 'uploading' as const, currentFileName }; }, [activeUploadBatches]); const fallbackViewMode: ViewMode = viewMode; diff --git a/src/components/documents/DocumentUploader.tsx b/src/components/documents/DocumentUploader.tsx index 4f1a07c..ffa1b27 100644 --- a/src/components/documents/DocumentUploader.tsx +++ b/src/components/documents/DocumentUploader.tsx @@ -4,7 +4,6 @@ import { useState, useCallback, useId, type ReactNode } from 'react'; import { useDropzone } from 'react-dropzone'; import { UploadIcon } from '@/components/icons/Icons'; import { useDocuments } from '@/contexts/DocumentContext'; -import { uploadDocxAsPdf } from '@/lib/client/api/documents'; import { useFeatureFlag } from '@/contexts/RuntimeConfigContext'; import { dropzoneSurfaceClass } from '@/components/ui'; @@ -20,7 +19,7 @@ export interface UploadBatchState { isActive: boolean; totalFiles: number; completedFiles: number; - phase: 'uploading' | 'converting'; + phase: 'uploading'; currentFileName: string | null; } @@ -33,13 +32,9 @@ export function DocumentUploader({ const uploaderId = useId(); const enableDocx = useFeatureFlag('enableDocxConversion'); const { - addPDFDocument: addPDF, - addEPUBDocument: addEPUB, - addHTMLDocument: addHTML, - refreshDocuments, + uploadDocuments, } = useDocuments(); const [isUploading, setIsUploading] = useState(false); - const [isConverting, setIsConverting] = useState(false); const [error, setError] = useState(null); const emitBatchState = useCallback((state: Omit) => { @@ -63,72 +58,13 @@ export function DocumentUploader({ }); try { - for (const file of acceptedFiles) { - if (file.type === 'application/pdf') { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: file.name, - }); - await addPDF(file); - completedFiles += 1; - } else if (file.type === 'application/epub+zip') { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: file.name, - }); - await addEPUB(file); - completedFiles += 1; - } else if (file.type === 'text/plain' || file.type === 'text/markdown' || file.name.endsWith('.md')) { - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: file.name, - }); - await addHTML(file); - completedFiles += 1; - } else if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') { - // Preserve prior UX: show "Converting DOCX..." state rather than generic uploading. - setIsUploading(false); - setIsConverting(true); - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'converting', - currentFileName: file.name, - }); - // Convert+upload directly on the server. Use sha(docx) as stable ID to avoid duplicates. - await uploadDocxAsPdf(file); - await refreshDocuments(); - setIsConverting(false); - setIsUploading(true); - completedFiles += 1; - } else { - continue; - } - - emitBatchState({ - isActive: true, - totalFiles, - completedFiles, - phase: 'uploading', - currentFileName: null, - }); - } + await uploadDocuments(acceptedFiles); + completedFiles = acceptedFiles.length; } catch (err) { setError('Failed to upload file. Please try again.'); console.error('Upload error:', err); } finally { setIsUploading(false); - setIsConverting(false); emitBatchState({ isActive: false, totalFiles, @@ -137,7 +73,7 @@ export function DocumentUploader({ currentFileName: null, }); } - }, [addHTML, addPDF, addEPUB, refreshDocuments, enableDocx, emitBatchState]); + }, [uploadDocuments, emitBatchState]); const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, @@ -151,12 +87,12 @@ export function DocumentUploader({ } : {}) }, multiple: true, - disabled: isUploading || isConverting, + disabled: isUploading, noClick: variant === 'overlay', noKeyboard: variant === 'overlay' }); - const isDisabled = isUploading || isConverting; + const isDisabled = isUploading; if (variant === 'overlay') { const rootProps = getRootProps(); @@ -209,8 +145,6 @@ export function DocumentUploader({ {isUploading ? (

Uploading…

- ) : isConverting ? ( -

Converting DOCX…

) : (

@@ -225,8 +159,6 @@ export function DocumentUploader({ {isUploading ? (

Uploading file...

- ) : isConverting ? ( -

Converting DOCX to PDF...

) : ( <>

diff --git a/src/components/player/TTSPlayer.tsx b/src/components/player/TTSPlayer.tsx index 1badd9b..a93ddbd 100644 --- a/src/components/player/TTSPlayer.tsx +++ b/src/components/player/TTSPlayer.tsx @@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl'; import { Navigator } from '@/components/player/Navigator'; import { IconButton } from '@/components/ui'; -export default function TTSPlayer({ currentPage, numPages }: { +export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: { currentPage?: number; numPages?: number | undefined; + isPlaybackReady?: boolean; }) { const { isPlaying, @@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: { {isProcessing ? : } @@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: { - {isPlaying ? : } + {!isPlaying && !isPlaybackReady + ? + : (isPlaying ? : )} {isProcessing ? : } diff --git a/src/contexts/DocumentContext.tsx b/src/contexts/DocumentContext.tsx index f45d8b1..d96dd4d 100644 --- a/src/contexts/DocumentContext.tsx +++ b/src/contexts/DocumentContext.tsx @@ -3,32 +3,53 @@ import { createContext, useCallback, useContext, useEffect, useMemo, ReactNode } from 'react'; import { useQuery, useQueryClient } from '@tanstack/react-query'; import type { BaseDocument } from '@/types/documents'; -import { listDocuments, uploadDocuments, deleteDocuments } from '@/lib/client/api/documents'; -import { putCachedEpub, putCachedHtml, putCachedPdf, evictCachedEpub, evictCachedHtml, evictCachedPdf } from '@/lib/client/cache/documents'; +import { + deleteDocuments as deleteServerDocuments, + listDocuments, + uploadDocuments as uploadServerDocuments, +} from '@/lib/client/api/documents'; +import { cacheStoredDocumentFromBytes, evictCachedDocument } from '@/lib/client/cache/documents'; import { useAuthSession } from '@/hooks/useAuthSession'; interface DocumentContextType { pdfDocs: Array; - addPDFDocument: (file: File) => Promise; - removePDFDocument: (id: string) => Promise; isPDFLoading: boolean; epubDocs: Array; - addEPUBDocument: (file: File) => Promise; - removeEPUBDocument: (id: string) => Promise; isEPUBLoading: boolean; htmlDocs: Array; - addHTMLDocument: (file: File) => Promise; - removeHTMLDocument: (id: string) => Promise; isHTMLLoading: boolean; + uploadDocuments: (files: File[]) => Promise; + deleteDocument: (id: string) => Promise; refreshDocuments: () => Promise; } const DocumentContext = createContext(undefined); const DOCUMENTS_QUERY_KEY = 'documents'; +type SupportedDocument = BaseDocument & { type: 'pdf' | 'epub' | 'html' }; + +function mergeStoredDocuments( + previous: SupportedDocument[] | undefined, + uploaded: BaseDocument[], +): SupportedDocument[] { + const next = [...(previous ?? [])]; + + for (let index = uploaded.length - 1; index >= 0; index -= 1) { + const stored = uploaded[index]; + if (stored.type !== 'pdf' && stored.type !== 'epub' && stored.type !== 'html') { + continue; + } + const supportedStored = stored as SupportedDocument; + const withoutExisting = next.filter((document) => document.id !== supportedStored.id); + next.splice(0, next.length, supportedStored, ...withoutExisting); + } + + return next; +} + export function DocumentProvider({ children }: { children: ReactNode }) { const queryClient = useQueryClient(); const { data: sessionData, isPending: isSessionPending } = useAuthSession(); @@ -81,67 +102,63 @@ export function DocumentProvider({ children }: { children: ReactNode }) { return { pdfDocs, epubDocs, htmlDocs }; }, [docs]); - const cacheUploaded = useCallback(async (stored: BaseDocument, file: File) => { - try { - if (stored.type === 'pdf') { - await putCachedPdf(stored, await file.arrayBuffer()); - } else if (stored.type === 'epub') { - await putCachedEpub(stored, await file.arrayBuffer()); - } else if (stored.type === 'html') { - const buf = await file.arrayBuffer(); - const decoded = new TextDecoder().decode(new Uint8Array(buf)); - await putCachedHtml(stored, decoded); - } - } catch (err) { - // Cache failures should not block uploads. - console.warn('Failed to cache uploaded document:', stored.id, err); - } - }, []); + const uploadDocuments = useCallback(async (files: File[]): Promise => { + if (files.length === 0) return []; - const addDocument = useCallback(async (file: File): Promise => { - const [stored] = await uploadDocuments([file]); - if (!stored) throw new Error('Upload succeeded but returned no document'); - await cacheUploaded(stored, file); - const isSupported = stored.type === 'pdf' || stored.type === 'epub' || stored.type === 'html'; - if (!isSupported) return stored.id; - const supportedStored = stored as BaseDocument & { type: 'pdf' | 'epub' | 'html' }; - queryClient.setQueryData>(documentsQueryKey, (prev = []) => { - const next = prev.filter((d) => d.id !== supportedStored.id); - return [supportedStored, ...next]; - }); - return stored.id; - }, [cacheUploaded, queryClient, documentsQueryKey]); - - const addPDFDocument = useCallback(async (file: File) => addDocument(file), [addDocument]); - const addEPUBDocument = useCallback(async (file: File) => addDocument(file), [addDocument]); - const addHTMLDocument = useCallback(async (file: File) => addDocument(file), [addDocument]); - - const removeById = useCallback(async (id: string) => { - await deleteDocuments({ ids: [id] }); - await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]); - queryClient.setQueryData>(documentsQueryKey, (prev = []) => - prev.filter((d) => d.id !== id), + const stored = await uploadServerDocuments(files); + await Promise.allSettled( + stored.map(async (document, index) => { + const file = files[index]; + if (!file) return; + const sourceType = file.name + ? ( + file.name.toLowerCase().endsWith('.pdf') + ? 'pdf' + : file.name.toLowerCase().endsWith('.epub') + ? 'epub' + : file.name.toLowerCase().endsWith('.docx') + ? 'docx' + : 'html' + ) + : ( + file.type === 'application/pdf' + ? 'pdf' + : file.type === 'application/epub+zip' + ? 'epub' + : file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ? 'docx' + : 'html' + ); + if (document.type !== sourceType) return; + await cacheStoredDocumentFromBytes(document, await file.arrayBuffer()); + }), ); - }, [queryClient, documentsQueryKey]); - const removePDFDocument = useCallback(async (id: string) => removeById(id), [removeById]); - const removeEPUBDocument = useCallback(async (id: string) => removeById(id), [removeById]); - const removeHTMLDocument = useCallback(async (id: string) => removeById(id), [removeById]); + queryClient.setQueryData(documentsQueryKey, (previous) => + mergeStoredDocuments(previous, stored), + ); + + return stored; + }, [documentsQueryKey, queryClient]); + + const deleteDocument = useCallback(async (id: string) => { + await deleteServerDocuments({ ids: [id] }); + await evictCachedDocument(id); + queryClient.setQueryData(documentsQueryKey, (previous = []) => + previous.filter((document) => document.id !== id), + ); + }, [documentsQueryKey, queryClient]); return ( diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index f1e0e9e..b346baf 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const [currentWordIndex, setCurrentWordIndex] = useState(null); const isPlayingRef = useRef(false); const pauseEpochRef = useRef(0); + const pendingPlaybackStartRef = useRef(false); const sentencesRef = useRef([]); const currentIndexRef = useRef(0); const plannedSegmentsByLocationRef = useRef>(new Map()); @@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const recordManualPause = useCallback(() => { // Cancel any queued auto-resume intent and mark an explicit user pause. resumeAfterLocationChangeRef.current = false; + pendingPlaybackStartRef.current = false; pauseEpochRef.current += 1; }, []); @@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement * Used for external control of playback state */ const pause = useCallback(() => { + pendingPlaybackStartRef.current = false; recordManualPause(); clearPendingEpubJump(); abortPendingTtsRequests(true); @@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const shouldPause = normalizedOptions.shouldPause ?? false; const pauseEpochAtStart = pauseEpochRef.current; const pendingAutoResume = resumeAfterLocationChangeRef.current; - const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume); - if (shouldPause || pendingAutoResume) { - resumeAfterLocationChangeRef.current = false; - } + const pendingPlaybackStart = pendingPlaybackStartRef.current; + const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart); // Keep track of previous state and pause playback invalidatePlaybackRun(); @@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement try { if (newSentences.length === 0) { console.warn('No sentences found in text'); + if (shouldPause || pendingAutoResume) { + resumeAfterLocationChangeRef.current = false; + } + if (pendingPlaybackStart) { + pendingPlaybackStartRef.current = false; + } setIsProcessing(false); return; } + if (shouldPause || pendingAutoResume) { + resumeAfterLocationChangeRef.current = false; + } + if (pendingPlaybackStart) { + pendingPlaybackStartRef.current = false; + } + if (!isEPUB && typeof resolvedLocation === 'number') { const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || ''); if (firstFingerprint) { @@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Ensure audio is unlocked while we're still in the click/tap handler. unlockPlaybackOnUserGesture(); + const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]); + if (!activeHowl && !hasPreparedSentence) { + pendingPlaybackStartRef.current = true; + setIsProcessing(true); + return; + } + // Resume current sentence if we already have a paused Howl. if (activeHowl) { applyPlaybackRateToHowl(activeHowl); @@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setIsPlaying(true); }, [ activeHowl, + currentIndex, applyPlaybackRateToHowl, isPlaying, pauseActiveHowl, + playbackSegments, recordManualPause, clearPendingEpubJump, abortPendingTtsRequests, + sentences, unlockPlaybackOnUserGesture, ]); @@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement abortAudio(); clearWarmAudioCache(); playbackInFlightRef.current = false; + pendingPlaybackStartRef.current = false; pendingJumpTargetRef.current = null; clearPendingEpubJump(); bumpEpubPreloadGeneration(); diff --git a/src/lib/client/api/documents.ts b/src/lib/client/api/documents.ts index b943354..f13a235 100644 --- a/src/lib/client/api/documents.ts +++ b/src/lib/client/api/documents.ts @@ -1,10 +1,8 @@ -import { sha256HexFromArrayBuffer } from '@/lib/client/sha256'; import type { BaseDocument, DocumentType } from '@/types/documents'; import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; import type { DocumentSettings } from '@/types/document-settings'; export type UploadSource = { - id: string; name: string; type: DocumentType; size: number; @@ -23,8 +21,12 @@ function toUploadBody(body: UploadSource['body']): BodyInit { return body as unknown as BodyInit; } -async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise { - const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, { +async function uploadDocumentSourceViaProxy( + source: UploadSource, + token: string, + options?: UploadOptions, +): Promise { + const res = await fetch(`/api/documents/blob/upload/fallback?token=${encodeURIComponent(token)}`, { method: 'PUT', headers: { 'Content-Type': source.contentType || 'application/octet-stream' }, body: toUploadBody(source.body), @@ -45,6 +47,14 @@ function documentTypeForName(name: string): DocumentType { return 'html'; } +function documentTypeForMime(contentType: string): DocumentType | null { + const normalized = contentType.trim().toLowerCase(); + if (normalized === 'application/pdf') return 'pdf'; + if (normalized === 'application/epub+zip') return 'epub'; + if (normalized === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') return 'docx'; + return null; +} + export function mimeTypeForDoc(doc: Pick): string { if (doc.type === 'pdf') return 'application/pdf'; if (doc.type === 'epub') return 'application/epub+zip'; @@ -80,31 +90,20 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort export async function getParsedPdfDocument( id: string, - options?: { signal?: AbortSignal; retryFailed?: boolean; opId?: string }, -): Promise< - | { status: 'ready'; parsed: ParsedPdfDocument } - | { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null; opId?: string | null } -> { - const params = new URLSearchParams(); - if (options?.retryFailed) params.set('retry', '1'); - if (options?.opId) params.set('opId', options.opId); - const query = params.size > 0 ? `?${params.toString()}` : ''; - const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, { + options?: { signal?: AbortSignal }, +): Promise { + const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed`, { signal: options?.signal, cache: 'no-store', }); - if (res.status === 202) { + if (res.status === 409) { const data = (await res.json().catch(() => null)) as { parseStatus?: string; parseProgress?: PdfParseProgress | null; opId?: string | null; } | null; - const parseStatus = data?.parseStatus; - if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') { - return { status: parseStatus, parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null }; - } - return { status: 'pending', parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null }; + throw new Error(data?.parseStatus ? `Parsed PDF is not ready (${data.parseStatus})` : 'Parsed PDF is not ready'); } if (!res.ok) { @@ -112,8 +111,7 @@ export async function getParsedPdfDocument( throw new Error(data?.error || 'Failed to load parsed PDF'); } - const parsed = (await res.json()) as ParsedPdfDocument; - return { status: 'ready', parsed }; + return (await res.json()) as ParsedPdfDocument; } export function subscribeParsedPdfDocumentEvents( @@ -241,7 +239,6 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ uploads: sources.map((source) => ({ - id: source.id, contentType: source.contentType, size: source.size, })), @@ -255,14 +252,18 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U } const presigned = (await presignRes.json()) as { - uploads?: Array<{ id: string; url: string; headers?: Record }>; + uploads?: Array<{ token: string; url: string; headers?: Record }>; }; - const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload])); + const uploads = presigned.uploads || []; + if (uploads.length !== sources.length) { + throw new Error('Upload preparation returned an unexpected number of temp uploads'); + } - for (const source of sources) { - const upload = byId.get(source.id); - if (!upload?.url) { - throw new Error(`Missing presigned upload for document ${source.id}`); + for (let index = 0; index < sources.length; index += 1) { + const source = sources[index]; + const upload = uploads[index]; + if (!upload?.url || !upload.token) { + throw new Error(`Missing presigned upload for document ${source.name}`); } let putError: unknown = null; @@ -285,7 +286,7 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U } try { - await uploadDocumentSourceViaProxy(source, options); + await uploadDocumentSourceViaProxy(source, upload.token, options); } catch (proxyError) { const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error'; const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error'; @@ -293,27 +294,26 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U } } - const registerRes = await fetch('/api/documents', { + const finalizeRes = await fetch('/api/documents/blob/upload/finalize', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - documents: sources.map((source) => ({ - id: source.id, + uploads: sources.map((source, index) => ({ + token: uploads[index]?.token, name: source.name, type: source.type, - size: source.size, lastModified: source.lastModified, })), }), signal: options?.signal, }); - if (!registerRes.ok) { - const data = (await registerRes.json().catch(() => null)) as { error?: string } | null; - throw new Error(data?.error || 'Failed to register uploaded documents'); + if (!finalizeRes.ok) { + const data = (await finalizeRes.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || 'Failed to finalize uploaded documents'); } - const data = (await registerRes.json()) as { stored: BaseDocument[] }; + const data = (await finalizeRes.json()) as { stored: BaseDocument[] }; return data.stored || []; } @@ -322,14 +322,14 @@ export async function uploadDocuments(files: File[], options?: UploadOptions): P const sources: UploadSource[] = []; for (const file of files) { - const bytes = await file.arrayBuffer(); - const id = await sha256HexFromArrayBuffer(bytes); - const type = documentTypeForName(file.name); - const name = file.name || `${id}.${type}`; - const contentType = file.type || mimeTypeForDoc({ name, type }); + const name = file.name || ''; + const type = name + ? documentTypeForName(name) + : (documentTypeForMime(file.type) ?? 'html'); + const resolvedName = name || `upload.${type}`; + const contentType = file.type || mimeTypeForDoc({ name: resolvedName, type }); sources.push({ - id, - name, + name: resolvedName, type, size: file.size, lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(), @@ -504,23 +504,3 @@ export async function getDocumentPreviewStatus( throw new Error(`Failed to load preview status (status ${res.status})`); } - -export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise { - const form = new FormData(); - form.append('file', file); - - const res = await fetch('/api/documents/docx-to-pdf/upload', { - method: 'POST', - body: form, - signal: options?.signal, - }); - - if (!res.ok) { - const data = (await res.json().catch(() => null)) as { error?: string } | null; - throw new Error(data?.error || 'Failed to convert DOCX'); - } - - const data = (await res.json()) as { stored: BaseDocument }; - if (!data?.stored) throw new Error('DOCX conversion succeeded but returned no document'); - return data.stored; -} diff --git a/src/lib/client/cache/documents.ts b/src/lib/client/cache/documents.ts index b552c0a..ef3aa06 100644 --- a/src/lib/client/cache/documents.ts +++ b/src/lib/client/cache/documents.ts @@ -107,6 +107,10 @@ export async function evictCachedHtml(id: string): Promise { await removeHtmlDocument(id); } +export async function evictCachedDocument(id: string): Promise { + await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]); +} + export async function clearDocumentCache(): Promise { const { clearPdfDocuments, clearEpubDocuments, clearHtmlDocuments } = await import('@/lib/client/dexie'); await Promise.all([clearPdfDocuments(), clearEpubDocuments(), clearHtmlDocuments()]); diff --git a/src/lib/client/dexie.ts b/src/lib/client/dexie.ts index 40e99f6..19bc09b 100644 --- a/src/lib/client/dexie.ts +++ b/src/lib/client/dexie.ts @@ -971,11 +971,9 @@ export async function syncDocumentsToServer( for (const doc of pdfDocs) { const bytes = new Uint8Array(doc.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: doc.id, source: { - id, name: doc.name, type: 'pdf', size: bytes.byteLength, @@ -992,11 +990,9 @@ export async function syncDocumentsToServer( for (const doc of epubDocs) { const bytes = new Uint8Array(doc.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: doc.id, source: { - id, name: doc.name, type: 'epub', size: bytes.byteLength, @@ -1013,11 +1009,9 @@ export async function syncDocumentsToServer( for (const doc of htmlDocs) { const encoded = textEncoder.encode(doc.data); - const id = await sha256HexFromBytes(encoded); uploads.push({ oldId: doc.id, source: { - id, name: doc.name, type: 'html', size: encoded.byteLength, @@ -1036,11 +1030,13 @@ export async function syncDocumentsToServer( onProgress(50, 'Uploading to server...'); } - await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); + const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); - for (const entry of uploads) { - if (entry.oldId === entry.source.id) continue; - await applyDocumentIdMapping(entry.oldId, entry.source.id); + for (let index = 0; index < uploads.length; index += 1) { + const entry = uploads[index]; + const nextId = stored[index]?.id; + if (!nextId || entry.oldId === nextId) continue; + await applyDocumentIdMapping(entry.oldId, nextId); } if (onProgress) { @@ -1064,11 +1060,9 @@ export async function syncSelectedDocumentsToServer( const data = await getPdfDocument(doc.id); if (data) { const bytes = new Uint8Array(data.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: data.id, source: { - id, name: data.name, type: 'pdf', size: bytes.byteLength, @@ -1082,11 +1076,9 @@ export async function syncSelectedDocumentsToServer( const data = await getEpubDocument(doc.id); if (data) { const bytes = new Uint8Array(data.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: data.id, source: { - id, name: data.name, type: 'epub', size: bytes.byteLength, @@ -1100,11 +1092,9 @@ export async function syncSelectedDocumentsToServer( const data = await getHtmlDocument(doc.id); if (data) { const bytes = textEncoder.encode(data.data); - const id = await sha256HexFromBytes(bytes); uploads.push({ oldId: data.id, source: { - id, name: data.name, type: 'html', size: bytes.byteLength, @@ -1121,11 +1111,13 @@ export async function syncSelectedDocumentsToServer( } if (onProgress) onProgress(50, 'Uploading to server...'); - await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); + const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); - for (const entry of uploads) { - if (entry.oldId === entry.source.id) continue; - await applyDocumentIdMapping(entry.oldId, entry.source.id); + for (let index = 0; index < uploads.length; index += 1) { + const entry = uploads[index]; + const nextId = stored[index]?.id; + if (!nextId || entry.oldId === nextId) continue; + await applyDocumentIdMapping(entry.oldId, nextId); } if (onProgress) { diff --git a/src/lib/server/compute/worker-parse-state.ts b/src/lib/server/compute/worker-parse-state.ts index cce75a4..6798a22 100644 --- a/src/lib/server/compute/worker-parse-state.ts +++ b/src/lib/server/compute/worker-parse-state.ts @@ -1,3 +1,4 @@ +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; import type { DocumentParseState } from '@/lib/server/documents/parse-state'; @@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState( ? parseProgress : null, updatedAt: nowMs, + parserVersion: PDF_PARSER_VERSION, ...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}), ...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}), ...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}), diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts index 6f94501..cdc79cc 100644 --- a/src/lib/server/compute/worker.ts +++ b/src/lib/server/compute/worker.ts @@ -1,7 +1,7 @@ import { createHash, randomUUID } from 'node:crypto'; import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; import { parseSseEventId, parseSsePayload } from '@openreader/compute-core'; -import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core'; +import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { logDegraded, logServerError } from '@/lib/server/errors/logging'; import type { @@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string { return [ 'pdf_layout', 'v1', + PDF_PARSER_VERSION, input.documentId, input.namespace ?? '', input.documentObjectKey ?? '', diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index fb84f00..36316c9 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -1,4 +1,5 @@ import { + CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, @@ -11,6 +12,9 @@ import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const TEMP_UPLOAD_TOKEN_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const TEMP_UPLOAD_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/; +export const TEMP_DOCUMENT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000; function sanitizeNamespace(namespace: string | null): string | null { if (!namespace) return null; @@ -69,6 +73,23 @@ export function isValidDocumentId(id: string): boolean { return DOCUMENT_ID_REGEX.test(id); } +export function isPreconditionFailed(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } }; + return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed'; +} + +export function isValidTempUploadToken(token: string): boolean { + return TEMP_UPLOAD_TOKEN_REGEX.test(token); +} + +function sanitizeTempUploadUserId(userId: string): string { + if (!TEMP_UPLOAD_USER_ID_REGEX.test(userId)) { + throw new Error(`Invalid temp upload user id: ${userId}`); + } + return encodeURIComponent(userId); +} + export function documentKey(id: string, namespace: string | null): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); @@ -90,6 +111,27 @@ export function documentParsedKey(id: string, namespace: string | null): string return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; } +export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string { + const cfg = getS3Config(); + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + return `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}users/${sanitizeTempUploadUserId(userId)}/`; +} + +export function tempDocumentUploadKey(token: string, userId: string, namespace: string | null): string { + if (!isValidTempUploadToken(token)) { + throw new Error(`Invalid temp upload token: ${token}`); + } + return `${tempDocumentUploadPrefix(userId, namespace)}${token}.bin`; +} + +export function tempDocumentUploadReceiptKey(token: string, userId: string, namespace: string | null): string { + if (!isValidTempUploadToken(token)) { + throw new Error(`Invalid temp upload token: ${token}`); + } + return `${tempDocumentUploadPrefix(userId, namespace)}${token}.receipt.json`; +} + function legacyDocumentParsedKey(id: string, namespace: string | null): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); @@ -138,6 +180,40 @@ export async function presignPut( }; } +export async function presignTempPut( + token: string, + userId: string, + contentType: string, + namespace: string | null, + options?: { contentLength?: number }, +): Promise<{ url: string; headers: Record }> { + const cfg = getS3Config(); + const client = getS3Client(); + const key = tempDocumentUploadKey(token, userId, namespace); + const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; + const contentLength = + typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0 + ? Math.floor(options.contentLength) + : undefined; + + const command = new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + ContentType: normalizedType, + ServerSideEncryption: 'AES256', + ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), + }); + const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); + + return { + url, + headers: { + 'Content-Type': normalizedType, + 'x-amz-server-side-encryption': 'AES256', + }, + }; +} + export async function headDocumentBlob( id: string, namespace: string | null, @@ -153,6 +229,23 @@ export async function headDocumentBlob( }; } +export async function headTempDocumentBlob( + token: string, + userId: string, + namespace: string | null, +): Promise<{ contentLength: number; contentType: string | null; eTag: string | null; lastModified: number | null }> { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadKey(token, userId, namespace); + const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key })); + return { + contentLength: Number(res.ContentLength ?? 0), + contentType: res.ContentType ?? null, + eTag: res.ETag ?? null, + lastModified: res.LastModified?.getTime() ?? null, + }; +} + export async function getDocumentRange( id: string, start: number, @@ -185,6 +278,23 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro return bodyToBuffer(res.Body); } +export async function getTempDocumentBlob( + token: string, + userId: string, + namespace: string | null, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadKey(token, userId, namespace); + const res = await client.send( + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + }), + ); + return bodyToBuffer(res.Body); +} + export async function getDocumentBlobStream(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -198,6 +308,29 @@ export async function getDocumentBlobStream(id: string, namespace: string | null return res.Body as DocumentBlobBody; } +export async function getTempDocumentFinalizeReceipt( + token: string, + userId: string, + namespace: string | null, +): Promise { + try { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadReceiptKey(token, userId, namespace); + const res = await client.send( + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + }), + ); + const body = await bodyToBuffer(res.Body); + return JSON.parse(body.toString('utf8')) as T; + } catch (error) { + if (isMissingBlobError(error)) return null; + throw error; + } +} + export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -241,6 +374,26 @@ export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: return key; } +export async function putTempDocumentFinalizeReceipt( + token: string, + userId: string, + namespace: string | null, + body: Buffer, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadReceiptKey(token, userId, namespace); + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + }), + ); +} + export async function presignGet( id: string, namespace: string | null, @@ -264,6 +417,7 @@ export async function putDocumentBlob( body: Buffer, contentType: string, namespace: string | null, + options?: { ifNoneMatch?: boolean }, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -275,6 +429,51 @@ export async function putDocumentBlob( Body: body, ContentType: contentType, ServerSideEncryption: 'AES256', + ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), + }), + ); +} + +export async function putTempDocumentBlob( + token: string, + userId: string, + body: Buffer, + contentType: string, + namespace: string | null, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const key = tempDocumentUploadKey(token, userId, namespace); + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: contentType, + ServerSideEncryption: 'AES256', + }), + ); +} + +export async function copyTempDocumentBlobToDocument( + token: string, + userId: string, + documentId: string, + namespace: string | null, + contentType: string, + options?: { ifNoneMatch?: boolean }, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send( + new CopyObjectCommand({ + Bucket: cfg.bucket, + Key: documentKey(documentId, namespace), + CopySource: `${cfg.bucket}/${tempDocumentUploadKey(token, userId, namespace)}`, + ContentType: contentType, + MetadataDirective: 'REPLACE', + ServerSideEncryption: 'AES256', + ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), }), ); } @@ -292,6 +491,18 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): await deleteDocumentPrefix(`${key}/`).catch(() => undefined); } +export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadKey(token, userId, namespace) })); +} + +export async function deleteTempDocumentFinalizeReceipt(token: string, userId: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadReceiptKey(token, userId, namespace) })); +} + export function isMissingBlobError(error: unknown): boolean { if (!error || typeof error !== 'object') return false; const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; @@ -339,3 +550,53 @@ export async function deleteDocumentPrefix(prefix: string): Promise { return deleted; } + +export async function deleteExpiredTempDocumentUploads( + userId: string, + namespace: string | null, + olderThanMs: number, +): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const prefix = tempDocumentUploadPrefix(userId, namespace); + let continuationToken: string | undefined; + const keys: string[] = []; + + do { + const listRes = await client.send( + new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + }), + ); + + for (const item of listRes.Contents ?? []) { + const key = item.Key; + const lastModified = item.LastModified?.getTime() ?? 0; + if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue; + keys.push(key); + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + + if (keys.length === 0) return 0; + + let deleted = 0; + for (let i = 0; i < keys.length; i += 1000) { + const batch = keys.slice(i, i + 1000); + const deleteRes = await client.send( + new DeleteObjectsCommand({ + Bucket: cfg.bucket, + Delete: { + Objects: batch.map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + deleted += deleteRes.Deleted?.length ?? 0; + } + + return deleted; +} diff --git a/src/lib/server/documents/docx-convert.ts b/src/lib/server/documents/docx-convert.ts new file mode 100644 index 0000000..1c940a1 --- /dev/null +++ b/src/lib/server/documents/docx-convert.ts @@ -0,0 +1,94 @@ +import { spawn } from 'child_process'; +import { existsSync } from 'fs'; +import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises'; +import path from 'path'; +import { randomUUID } from 'node:crypto'; +import { pathToFileURL } from 'url'; + +const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); +const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); +const DEFAULT_DOCX_CONVERSION_TIMEOUT_MS = 60_000; + +async function ensureTempDir(): Promise { + if (!existsSync(DOCSTORE_DIR)) { + await mkdir(DOCSTORE_DIR, { recursive: true }); + } + if (!existsSync(TEMP_DIR)) { + await mkdir(TEMP_DIR, { recursive: true }); + } +} + +async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise { + return new Promise((resolve, reject) => { + const args: string[] = []; + if (profileDir) { + args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`); + } + args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath); + const proc = spawn('soffice', args); + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + settled = true; + proc.kill(); + reject(new Error(`LibreOffice conversion timed out after ${DEFAULT_DOCX_CONVERSION_TIMEOUT_MS}ms`)); + }, DEFAULT_DOCX_CONVERSION_TIMEOUT_MS); + + proc.on('error', (error) => { + clearTimeout(timeout); + if (settled) return; + settled = true; + reject(error); + }); + proc.on('close', (code) => { + clearTimeout(timeout); + if (settled) return; + settled = true; + if (code === 0) resolve(); + else reject(new Error(`LibreOffice conversion failed with code ${code}`)); + }); + }); +} + +async function waitForPdfReady(dir: string, timeoutMs = 20_000, intervalMs = 100): Promise { + const end = Date.now() + timeoutMs; + while (Date.now() < end) { + const files = await readdir(dir); + const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf')); + if (pdf) { + const pdfPath = path.join(dir, pdf); + try { + const first = await stat(pdfPath); + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + const second = await stat(pdfPath); + if (second.size > 0 && second.size === first.size) { + return pdfPath; + } + } catch { + // Ignore transient filesystem races while LibreOffice is still writing. + } + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`); +} + +export async function convertDocxBufferToPdfBuffer(docxBytes: Buffer): Promise { + await ensureTempDir(); + + const tempId = randomUUID(); + const jobDir = path.join(TEMP_DIR, tempId); + await mkdir(jobDir, { recursive: true }); + const profileDir = path.join(jobDir, 'lo-profile'); + await mkdir(profileDir, { recursive: true }); + const inputPath = path.join(jobDir, 'input.docx'); + + try { + await writeFile(inputPath, docxBytes); + await convertDocxToPdf(inputPath, jobDir, profileDir); + const pdfPath = await waitForPdfReady(jobDir); + return await readFile(pdfPath); + } finally { + await rm(jobDir, { recursive: true, force: true }).catch(() => {}); + } +} diff --git a/src/lib/server/documents/parse-state-backfill.ts b/src/lib/server/documents/parse-state-backfill.ts index 97f9cae..f058ba1 100644 --- a/src/lib/server/documents/parse-state-backfill.ts +++ b/src/lib/server/documents/parse-state-backfill.ts @@ -1,7 +1,5 @@ -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; -import { documentKey } from '@/lib/server/documents/blobstore'; -import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; -import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; import { normalizeParseStatus, type DocumentParseState } from '@/lib/server/documents/parse-state'; import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; @@ -20,13 +18,20 @@ export async function backfillPendingPdfParseOperation(input: { if (parseStatus === 'ready' || parseStatus === 'failed') return null; if (normalizeOpId(input.state.opId)) return null; - const created = await createOrReusePdfWorkerOperation({ + const startedParse = await startPdfParseOperation({ documentId: input.documentId, + userId: input.userId, namespace: input.namespace, - documentObjectKey: documentKey(input.documentId, input.namespace), }); - - const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - await recordJobEvent(input.userId, 'pdf_layout', created.opId, rateConfig); - return created; + if (startedParse.parseState.status === 'pending' || startedParse.parseState.status === 'running') { + enqueueParsePdfJob({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status, + }); + } + return startedParse.workerState; } diff --git a/src/lib/server/documents/parse-state-healing.ts b/src/lib/server/documents/parse-state-healing.ts index 82ca5dc..d8fcbd4 100644 --- a/src/lib/server/documents/parse-state-healing.ts +++ b/src/lib/server/documents/parse-state-healing.ts @@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: { error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`, ...(input.state.opId ? { opId: input.state.opId } : {}), ...(input.state.jobId ? { jobId: input.state.jobId } : {}), + ...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}), })); await db diff --git a/src/lib/server/documents/parse-state.ts b/src/lib/server/documents/parse-state.ts index 431e20c..22a94ad 100644 --- a/src/lib/server/documents/parse-state.ts +++ b/src/lib/server/documents/parse-state.ts @@ -1,4 +1,5 @@ -import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; export interface DocumentParseState { status: PdfParseStatus; @@ -7,6 +8,7 @@ export interface DocumentParseState { error?: string | null; opId?: string; jobId?: string; + parserVersion?: string; } export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' { @@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars return 'pending'; } +export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean { + return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION; +} + +export function normalizeDocumentParseStateForCurrentParserVersion( + state: DocumentParseState, + nowMs = Date.now(), +): DocumentParseState { + const status = normalizeParseStatus(state.status); + if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) { + return state; + } + + return { + status: 'pending', + progress: null, + updatedAt: nowMs, + }; +} + +export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string { + const parserVersion = parsed?.parserVersion?.trim(); + return parserVersion || PDF_PARSER_VERSION; +} + function normalizeProgress(progress: unknown): PdfParseProgress | null { if (!progress || typeof progress !== 'object') return null; const rec = progress as Record; @@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat const error = typeof parsed.error === 'string' ? parsed.error : null; const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined; const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined; + const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined; return { status, progress, @@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat ...(error ? { error } : {}), ...(opId ? { opId } : {}), ...(jobId ? { jobId } : {}), + ...(parserVersion ? { parserVersion } : {}), }; } catch { return { status: 'pending', progress: null }; @@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string { ...(state.error ? { error: state.error } : {}), ...(state.opId ? { opId: state.opId } : {}), ...(state.jobId ? { jobId: state.jobId } : {}), + ...(state.parserVersion ? { parserVersion: state.parserVersion } : {}), }); } diff --git a/src/lib/server/documents/parsed-pdf-reuse.ts b/src/lib/server/documents/parsed-pdf-reuse.ts new file mode 100644 index 0000000..1bbede4 --- /dev/null +++ b/src/lib/server/documents/parsed-pdf-reuse.ts @@ -0,0 +1,33 @@ +import { eq } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { + normalizeDocumentParseStateForCurrentParserVersion, + parseDocumentParseState, +} from '@/lib/server/documents/parse-state'; + +type ReadyParsedPdfResult = { + parsedJsonKey: string; +}; + +export async function findReusableParsedPdfResult( + documentId: string, +): Promise { + const rows = await db + .select({ + parseState: documents.parseState, + parsedJsonKey: documents.parsedJsonKey, + }) + .from(documents) + .where(eq(documents.id, documentId)); + + for (const row of rows) { + const parsedJsonKey = row.parsedJsonKey?.trim(); + if (!parsedJsonKey) continue; + const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)); + if (parseState.status !== 'ready') continue; + return { parsedJsonKey }; + } + + return null; +} diff --git a/src/lib/server/documents/pdf-parse-operation.ts b/src/lib/server/documents/pdf-parse-operation.ts new file mode 100644 index 0000000..c356f86 --- /dev/null +++ b/src/lib/server/documents/pdf-parse-operation.ts @@ -0,0 +1,30 @@ +import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; +import { documentParseStateFromWorkerState } from '@/lib/server/compute/worker-parse-state'; +import { documentKey } from '@/lib/server/documents/blobstore'; +import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; +import type { DocumentParseState } from '@/lib/server/documents/parse-state'; +import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; + +export async function startPdfParseOperation(input: { + documentId: string; + userId: string; + namespace: string | null; + forceToken?: string; +}): Promise<{ + workerState: WorkerOperationState; + parseState: DocumentParseState; +}> { + const workerState = await createOrReusePdfWorkerOperation({ + documentId: input.documentId, + namespace: input.namespace, + documentObjectKey: documentKey(input.documentId, input.namespace), + ...(input.forceToken ? { forceToken: input.forceToken } : {}), + }); + const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + await recordJobEvent(input.userId, 'pdf_layout', workerState.opId, rateConfig); + return { + workerState, + parseState: documentParseStateFromWorkerState(workerState), + }; +} diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts new file mode 100644 index 0000000..06bafd2 --- /dev/null +++ b/src/lib/server/documents/register-upload.ts @@ -0,0 +1,106 @@ +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { + enqueueDocumentPreview, +} from '@/lib/server/documents/previews'; +import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; +import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; +import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; +import { errorToLog, serverLogger } from '@/lib/server/logger'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import type { BaseDocument, DocumentType } from '@/types/documents'; + +type RegisterUploadedDocumentInput = { + documentId: string; + userId: string; + namespace: string | null; + name: string; + type: DocumentType; + size: number; + lastModified: number; +}; + +export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise { + const reusableParsedPdf = input.type === 'pdf' + ? await findReusableParsedPdfResult(input.documentId) + : null; + const startedParse = input.type === 'pdf' && !reusableParsedPdf + ? await startPdfParseOperation({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + }) + : null; + const parsedJsonKey = reusableParsedPdf?.parsedJsonKey ?? null; + const parseState = input.type === 'pdf' + ? stringifyDocumentParseState( + reusableParsedPdf + ? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION } + : (startedParse?.parseState ?? { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }), + ) + : null; + + await db + .insert(documents) + .values({ + id: input.documentId, + userId: input.userId, + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + filePath: input.documentId, + parseState, + parsedJsonKey, + }) + .onConflictDoUpdate({ + target: [documents.id, documents.userId], + set: { + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + filePath: input.documentId, + parseState, + parsedJsonKey, + }, + }); + + await enqueueDocumentPreview( + { + id: input.documentId, + type: input.type, + lastModified: input.lastModified, + }, + input.namespace, + ).catch((error) => { + serverLogger.warn({ + event: 'documents.preview.enqueue.failed', + degraded: true, + fallbackPath: 'skip_preview_enqueue', + documentId: input.documentId, + error: errorToLog(error), + }, 'Failed to enqueue document preview'); + }); + + if (startedParse) { + enqueueParsePdfJob({ + documentId: input.documentId, + userId: input.userId, + namespace: input.namespace, + initialOpId: startedParse.workerState.opId, + initialJobId: startedParse.workerState.jobId, + initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending', + }); + } + + return { + id: input.documentId, + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + scope: 'user', + }; +} diff --git a/src/lib/server/jobs/user-pdf-layout-job.ts b/src/lib/server/jobs/user-pdf-layout-job.ts index 341fd55..0f44043 100644 --- a/src/lib/server/jobs/user-pdf-layout-job.ts +++ b/src/lib/server/jobs/user-pdf-layout-job.ts @@ -1,11 +1,14 @@ import { and, eq, inArray, isNull } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; -import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; +import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; +import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded, logServerError } from '@/lib/server/errors/logging'; import { + normalizeDocumentParseStateForCurrentParserVersion, parseDocumentParseState, + resolveParsedPdfParserVersion, stringifyDocumentParseState, type DocumentParseState, } from '@/lib/server/documents/parse-state'; @@ -13,15 +16,20 @@ import { getCompute } from '@/lib/server/compute'; import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache'; import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy'; import type { + ParsedPdfDocument, PdfLayoutJobBase, PdfLayoutJobResult, PdfLayoutProgress, WorkerOperationState, } from '@openreader/compute-core/api-contracts'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; type UserPdfLayoutJobRequest = PdfLayoutJobBase & { userId: string; forceToken?: string; + initialOpId?: string; + initialJobId?: string; + initialStatus?: 'pending' | 'running'; }; const running = new Set(); @@ -41,7 +49,12 @@ function keyFor(input: UserPdfLayoutJobRequest): string { if (forceToken) { return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`; } - return `shared:${input.documentId}:${input.namespace || ''}`; + return `shared:${input.documentId}`; +} + +function normalizeOpId(value: string | undefined): string | undefined { + const normalized = value?.trim(); + return normalized ? normalized : undefined; } function sleep(ms: number): Promise { @@ -84,7 +97,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise { + if (parsed) return parsed; + + try { + const json = await getParsedDocumentBlobByKey(parsedJsonKey); + return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument; + } catch { + return null; + } +} + async function updateParseStateForUsers(input: { documentId: string; userIds: string[]; @@ -125,6 +152,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS; while (Date.now() < deadline) { + const reusableParsed = await findReusableParsedPdfResult(input.documentId); + if (reusableParsed) { + const readyState: DocumentParseState = { + status: 'ready', + progress: null, + updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + }; + await updateParseStateForUsers({ + documentId: input.documentId, + userIds: [input.userId], + parseState: stringifyDocumentParseState(readyState), + parsedJsonKey: reusableParsed.parsedJsonKey, + }); + return; + } + const rows = await loadScopedRows(input); const ready = rows.find(isReadyRow); if (ready) { @@ -132,6 +176,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise status: 'ready', progress: null, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, }; await updateParseStateForUsers({ documentId: input.documentId, @@ -142,7 +187,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise return; } - const statuses = rows.map((row) => parseDocumentParseState(row.parseState)); + const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState))); const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running'); const failedState = statuses.find((state) => state.status === 'failed'); if (failedState && !hasInFlight) { @@ -151,6 +196,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise progress: null, updatedAt: Date.now(), ...(failedState.error ? { error: failedState.error } : {}), + ...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}), }; await updateParseStateForUsers({ documentId: input.documentId, @@ -174,6 +220,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise } running.add(key); + let activeOpId = normalizeOpId(input.initialOpId); + let activeJobId = normalizeOpId(input.initialJobId); try { const scopedRows = await loadScopedRows(input); const scopedUserIds = userIdsFromRows(scopedRows); @@ -181,12 +229,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise // Non-force jobs can short-circuit by reusing existing ready output from // any row in the same document scope. if (!input.forceToken?.trim()) { + const reusableParsed = await findReusableParsedPdfResult(input.documentId); + if (reusableParsed) { + const readyState: DocumentParseState = { + status: 'ready', + progress: null, + updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + }; + await updateParseStateForUsers({ + documentId: input.documentId, + userIds: [input.userId], + parseState: stringifyDocumentParseState(readyState), + parsedJsonKey: reusableParsed.parsedJsonKey, + }); + return; + } + const ready = scopedRows.find(isReadyRow); if (ready) { const readyState: DocumentParseState = { status: 'ready', progress: null, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, }; await updateParseStateForUsers({ documentId: input.documentId, @@ -201,17 +267,21 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise const coordinator = [...scopedRows].sort((a, b) => a.userId.localeCompare(b.userId))[0]; if (!coordinator) return; - const runningStateData: DocumentParseState = { - status: 'running', + const initialInflightStatus = input.initialStatus === 'running' ? 'running' : 'pending'; + const inflightStateData: DocumentParseState = { + status: initialInflightStatus, progress: null, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, + ...(activeOpId ? { opId: activeOpId } : {}), + ...(activeJobId ? { jobId: activeJobId } : {}), }; - const runningState = stringifyDocumentParseState(runningStateData); + const inflightState = stringifyDocumentParseState(inflightStateData); const claimRows = (await db .update(documents) .set({ - parseState: runningState, + parseState: inflightState, }) .where( and( @@ -234,12 +304,10 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise await updateParseStateForUsers({ documentId: input.documentId, userIds: cohortUserIds, - parseState: runningState, + parseState: inflightState, }); const compute = await getCompute(); - let activeOpId: string | undefined; - let activeJobId: string | undefined; let lastProgressWriteAt = 0; let lastSnapshotWriteAt = 0; let lastSnapshotStatus: 'pending' | 'running' | null = null; @@ -279,6 +347,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise status: mappedStatus, progress: snapshot.progress ?? null, updatedAt: now, + parserVersion: PDF_PARSER_VERSION, ...(activeOpId ? { opId: activeOpId } : {}), ...(activeJobId ? { jobId: activeJobId } : {}), }; @@ -305,6 +374,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise phase: progress.phase, }, updatedAt: Date.now(), + parserVersion: PDF_PARSER_VERSION, ...(activeOpId ? { opId: activeOpId } : {}), ...(activeJobId ? { jobId: activeJobId } : {}), }; @@ -327,12 +397,17 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace); } + const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey); + const finalScopedRows = await loadScopedRows(input); const finalUserIds = userIdsFromRows(finalScopedRows); const readyState: DocumentParseState = { status: 'ready', progress: null, updatedAt: Date.now(), + parserVersion: resolveParsedPdfParserVersion(parsedDoc), + ...(activeOpId ? { opId: activeOpId } : {}), + ...(activeJobId ? { jobId: activeJobId } : {}), }; const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds; await updateParseStateForUsers({ @@ -385,6 +460,9 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise progress: null, updatedAt: Date.now(), error: message, + parserVersion: PDF_PARSER_VERSION, + ...(activeOpId ? { opId: activeOpId } : {}), + ...(activeJobId ? { jobId: activeJobId } : {}), }; const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId]; await updateParseStateForUsers({ diff --git a/src/types/documents.ts b/src/types/documents.ts index b50bd40..ac870d5 100644 --- a/src/types/documents.ts +++ b/src/types/documents.ts @@ -11,7 +11,6 @@ export interface BaseDocument { parsedJsonKey?: string | null; scope?: 'user'; folderId?: string; - isConverting?: boolean; } export interface PDFDocument extends BaseDocument { diff --git a/tests/helpers.ts b/tests/helpers.ts index cfc9472..44c8bd8 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -1,6 +1,4 @@ import { Page, expect, type TestInfo, type Locator } from '@playwright/test'; -import fs from 'fs'; -import path from 'path'; import { createHash } from 'crypto'; const DIR = './tests/files/'; @@ -10,14 +8,6 @@ function escapeRegExp(input: string) { return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } -function fixturePath(fileName: string) { - return path.join(__dirname, 'files', fileName); -} - -function sha256HexOfFile(filePath: string) { - return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); -} - async function waitForPdfViewerReady(page: Page, timeout = 60000) { await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) }); const loader = page.getByTestId('pdf-status-loader').first(); @@ -40,7 +30,7 @@ async function waitForPdfViewerReady(page: Page, timeout = 60000) { } /** - * Upload a sample epub or pdf + * Upload a sample document fixture */ export async function uploadFile(page: Page, filePath: string) { const input = page.locator('input[type=file]').first(); @@ -50,7 +40,7 @@ export async function uploadFile(page: Page, filePath: string) { await input.setInputFiles(`${DIR}${filePath}`); // Wait for the uploader to finish processing. The input is disabled while - // uploading/converting via react-dropzone's `disabled` prop. + // uploading via react-dropzone's `disabled` prop. // Tolerate extremely fast operations where the disabled state may be missed. try { await expect(input).toBeDisabled({ timeout: 2000 }); @@ -69,14 +59,8 @@ export async function uploadAndDisplay(page: Page, fileName: string) { const lower = fileName.toLowerCase(); if (lower.endsWith('.docx')) { - // Best-effort: conversion can complete before we observe this UI state. - try { - await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 }); - } catch { - // ignore - } - const expectedId = sha256HexOfFile(fixturePath(fileName)); - const targetLink = page.locator(`a[href$="/pdf/${expectedId}"]`).first(); + const convertedName = `${fileName.replace(/\.[^.]+$/, '')}.pdf`; + const targetLink = page.getByRole('link', { name: new RegExp(escapeRegExp(convertedName), 'i') }).first(); await expect(targetLink).toBeVisible({ timeout: 15000 }); await dismissOnboardingModals(page); await targetLink.click(); @@ -188,10 +172,11 @@ async function dismissOnboardingModals(page: Page): Promise { * Wait for the play button to be clickable and click it */ export async function waitAndClickPlay(page: Page) { - // Wait for play button selector without disabled attribute - await expect(page.getByRole('button', { name: 'Play' })).toBeVisible(); + const playButton = page.getByRole('button', { name: 'Play' }); + await expect(playButton).toBeVisible(); + await expect(playButton).toBeEnabled({ timeout: 15000 }); // Play the TTS by clicking the button - await page.getByRole('button', { name: 'Play' }).click(); + await playButton.click(); // Use resilient processing transition helper (tolerates fast completion) await expectProcessingTransition(page); } diff --git a/tests/navigation.spec.ts b/tests/navigation.spec.ts index b642b20..ea193f8 100644 --- a/tests/navigation.spec.ts +++ b/tests/navigation.spec.ts @@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => { await page.waitForTimeout(750); // After resize, playback should have paused (Play button visible) - await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 }); + const playButton = page.getByRole('button', { name: 'Play' }); + await expect(playButton).toBeVisible({ timeout: 15000 }); + await expect(playButton).toBeEnabled({ timeout: 15000 }); // Resume playback and ensure processing -> playing - await page.getByRole('button', { name: 'Play' }).click(); + await playButton.click(); await expectProcessingTransition(page); }); }); diff --git a/tests/unit/blob-upload-finalize-docx.vitest.spec.ts b/tests/unit/blob-upload-finalize-docx.vitest.spec.ts new file mode 100644 index 0000000..b818d66 --- /dev/null +++ b/tests/unit/blob-upload-finalize-docx.vitest.spec.ts @@ -0,0 +1,150 @@ +import { createHash } from 'node:crypto'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const hoisted = vi.hoisted(() => ({ + requireAuthContext: vi.fn(), + registerUploadedDocument: vi.fn(), + convertDocxBufferToPdfBuffer: vi.fn(), + headTempDocumentBlob: vi.fn(), + getTempDocumentBlob: vi.fn(), + getTempDocumentFinalizeReceipt: vi.fn(), + putTempDocumentFinalizeReceipt: vi.fn(), + deleteTempDocumentUpload: vi.fn(), + headDocumentBlob: vi.fn(), + copyTempDocumentBlobToDocument: vi.fn(), + putDocumentBlob: vi.fn(), +})); + +vi.mock('@/lib/server/auth/auth', () => ({ + requireAuthContext: hoisted.requireAuthContext, +})); + +vi.mock('@/lib/server/documents/register-upload', () => ({ + registerUploadedDocument: hoisted.registerUploadedDocument, +})); + +vi.mock('@/lib/server/documents/docx-convert', () => ({ + convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer, +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + TEMP_DOCUMENT_UPLOAD_TTL_MS: 24 * 60 * 60 * 1000, + copyTempDocumentBlobToDocument: hoisted.copyTempDocumentBlobToDocument, + deleteTempDocumentUpload: hoisted.deleteTempDocumentUpload, + getTempDocumentBlob: hoisted.getTempDocumentBlob, + getTempDocumentFinalizeReceipt: hoisted.getTempDocumentFinalizeReceipt, + headDocumentBlob: hoisted.headDocumentBlob, + headTempDocumentBlob: hoisted.headTempDocumentBlob, + isMissingBlobError: vi.fn((error: unknown) => { + const maybe = error as { code?: string } | undefined; + return maybe?.code === 'NoSuchKey'; + }), + isPreconditionFailed: vi.fn(() => false), + isValidTempUploadToken: vi.fn(() => true), + putDocumentBlob: hoisted.putDocumentBlob, + putTempDocumentFinalizeReceipt: hoisted.putTempDocumentFinalizeReceipt, +})); + +vi.mock('@/lib/server/testing/test-namespace', () => ({ + getOpenReaderTestNamespace: vi.fn(() => null), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + isS3Configured: vi.fn(() => true), +})); + +vi.mock('@/lib/server/logger', () => ({ + errorToLog: vi.fn((error: unknown) => error), + serverLogger: { + warn: vi.fn(), + error: vi.fn(), + }, +})); + +describe('POST /api/documents/blob/upload/finalize DOCX flow', () => { + beforeEach(() => { + hoisted.requireAuthContext.mockReset(); + hoisted.registerUploadedDocument.mockReset(); + hoisted.convertDocxBufferToPdfBuffer.mockReset(); + hoisted.headTempDocumentBlob.mockReset(); + hoisted.getTempDocumentBlob.mockReset(); + hoisted.getTempDocumentFinalizeReceipt.mockReset(); + hoisted.putTempDocumentFinalizeReceipt.mockReset(); + hoisted.deleteTempDocumentUpload.mockReset(); + hoisted.headDocumentBlob.mockReset(); + hoisted.copyTempDocumentBlobToDocument.mockReset(); + hoisted.putDocumentBlob.mockReset(); + + hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' }); + hoisted.getTempDocumentFinalizeReceipt.mockResolvedValue(null); + hoisted.headTempDocumentBlob.mockResolvedValue({ + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + contentLength: 12, + lastModified: Date.now(), + }); + hoisted.getTempDocumentBlob.mockResolvedValue(Buffer.from('docx-bytes')); + hoisted.convertDocxBufferToPdfBuffer.mockResolvedValue(Buffer.from('pdf-bytes')); + hoisted.headDocumentBlob + .mockRejectedValueOnce({ code: 'NoSuchKey' }) + .mockResolvedValue({ + contentLength: Buffer.byteLength('pdf-bytes'), + contentType: 'application/pdf', + eTag: 'etag-1', + }); + hoisted.registerUploadedDocument.mockImplementation(async (input: { documentId: string; name: string; type: string; size: number; lastModified: number }) => ({ + id: input.documentId, + name: input.name, + type: input.type, + size: input.size, + lastModified: input.lastModified, + scope: 'user', + })); + hoisted.putTempDocumentFinalizeReceipt.mockResolvedValue(undefined); + hoisted.deleteTempDocumentUpload.mockResolvedValue(undefined); + }); + + test('converts raw DOCX during finalize and registers a PDF', async () => { + const { POST } = await import('../../src/app/api/documents/blob/upload/finalize/route'); + const request = new NextRequest('http://localhost/api/documents/blob/upload/finalize', { + method: 'POST', + body: JSON.stringify({ + uploads: [{ + token: '123e4567-e89b-12d3-a456-426614174000', + name: 'Report.docx', + type: 'docx', + lastModified: 1700000000000, + }], + }), + headers: { + 'Content-Type': 'application/json', + }, + }); + + const response = await POST(request); + const expectedId = createHash('sha256').update(Buffer.from('pdf-bytes')).digest('hex'); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + stored: [{ + id: expectedId, + name: 'Report.pdf', + type: 'pdf', + }], + }); + expect(hoisted.convertDocxBufferToPdfBuffer).toHaveBeenCalledTimes(1); + expect(hoisted.putDocumentBlob).toHaveBeenCalledWith( + expectedId, + Buffer.from('pdf-bytes'), + 'application/pdf', + null, + { ifNoneMatch: true }, + ); + expect(hoisted.copyTempDocumentBlobToDocument).not.toHaveBeenCalled(); + expect(hoisted.registerUploadedDocument).toHaveBeenCalledWith(expect.objectContaining({ + documentId: expectedId, + name: 'Report.pdf', + type: 'pdf', + })); + }); +}); diff --git a/tests/unit/pdf-op-key.vitest.spec.ts b/tests/unit/pdf-op-key.vitest.spec.ts new file mode 100644 index 0000000..fbe746e --- /dev/null +++ b/tests/unit/pdf-op-key.vitest.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from 'vitest'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; + +describe('pdf worker op key', () => { + test('includes parser version in the reuse boundary', () => { + expect(buildPdfOpKey({ + documentId: 'doc-1', + namespace: null, + documentObjectKey: 'documents/doc-1.pdf', + })).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`); + }); +}); diff --git a/tests/unit/pdf-parse-state.vitest.spec.ts b/tests/unit/pdf-parse-state.vitest.spec.ts new file mode 100644 index 0000000..aa3384b --- /dev/null +++ b/tests/unit/pdf-parse-state.vitest.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from 'vitest'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; +import { + normalizeDocumentParseStateForCurrentParserVersion, + parseDocumentParseState, + stringifyDocumentParseState, +} from '../../src/lib/server/documents/parse-state'; + +describe('document parse state parser-version handling', () => { + test('preserves parserVersion through stringify/parse', () => { + const state = parseDocumentParseState(stringifyDocumentParseState({ + status: 'ready', + progress: null, + updatedAt: 1234, + parserVersion: PDF_PARSER_VERSION, + })); + + expect(state).toEqual({ + status: 'ready', + progress: null, + updatedAt: 1234, + parserVersion: PDF_PARSER_VERSION, + }); + }); + + test('invalidates ready and inflight states from older parser versions', () => { + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'ready', + progress: null, + updatedAt: 1111, + parserVersion: 'old-parser', + opId: 'op-old', + }, 2222)).toEqual({ + status: 'pending', + progress: null, + updatedAt: 2222, + }); + + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'running', + progress: { + totalPages: 10, + pagesParsed: 3, + currentPage: 4, + phase: 'infer', + }, + updatedAt: 1111, + parserVersion: 'old-parser', + opId: 'op-old', + }, 3333)).toEqual({ + status: 'pending', + progress: null, + updatedAt: 3333, + }); + }); + + test('keeps failed states and current-version ready states intact', () => { + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'failed', + progress: null, + updatedAt: 1111, + error: 'no text layer', + parserVersion: 'old-parser', + })).toEqual({ + status: 'failed', + progress: null, + updatedAt: 1111, + error: 'no text layer', + parserVersion: 'old-parser', + }); + + expect(normalizeDocumentParseStateForCurrentParserVersion({ + status: 'ready', + progress: null, + updatedAt: 1111, + parserVersion: PDF_PARSER_VERSION, + })).toEqual({ + status: 'ready', + progress: null, + updatedAt: 1111, + parserVersion: PDF_PARSER_VERSION, + }); + }); +}); diff --git a/tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts b/tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts new file mode 100644 index 0000000..d39d9f5 --- /dev/null +++ b/tests/unit/pdf-parsed-events-route-legacy-backfill.vitest.spec.ts @@ -0,0 +1,156 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const hoisted = vi.hoisted(() => ({ + db: null as { + select: ReturnType; + update: ReturnType; + } | null, + row: { + id: 'doc-1', + userId: 'user-1', + parseState: null as string | null, + }, + requireAuthContext: vi.fn(), + backfillPendingPdfParseOperation: vi.fn(), + healStaleDocumentParseState: vi.fn(async ({ state }) => state), +})); + +vi.mock('@/db', () => ({ + get db() { + return hoisted.db; + }, +})); + +vi.mock('@/lib/server/auth/auth', () => ({ + requireAuthContext: hoisted.requireAuthContext, +})); + +vi.mock('@/lib/server/documents/parse-state-backfill', () => ({ + backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation, +})); + +vi.mock('@/lib/server/documents/parse-state-healing', () => ({ + healStaleDocumentParseState: hoisted.healStaleDocumentParseState, +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + isValidDocumentId: vi.fn(() => true), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + isS3Configured: vi.fn(() => true), +})); + +vi.mock('@/lib/server/testing/test-namespace', () => ({ + getOpenReaderTestNamespace: vi.fn(() => null), +})); + +vi.mock('@/lib/server/logger', () => ({ + createRequestLogger: vi.fn(() => ({ + logger: { + warn: vi.fn(), + info: vi.fn(), + error: vi.fn(), + }, + requestId: 'req-test', + })), + hashForLog: vi.fn(() => 'user-hash'), +})); + +describe('GET /api/documents/[id]/parsed/events legacy backfill', () => { + const originalFetch = global.fetch; + + beforeEach(() => { + process.env.COMPUTE_WORKER_URL = 'http://localhost:4010'; + process.env.COMPUTE_WORKER_TOKEN = 'worker-test-token'; + + hoisted.row = { + id: 'doc-1', + userId: 'user-1', + parseState: null, + }; + hoisted.db = { + select: vi.fn(() => ({ + from: vi.fn(() => ({ + where: vi.fn(async () => ([{ ...hoisted.row }])), + })), + })), + update: vi.fn(() => ({ + set: vi.fn((values: { parseState?: string | null }) => ({ + where: vi.fn(async () => { + if (typeof values.parseState !== 'undefined') { + hoisted.row.parseState = values.parseState; + } + return []; + }), + })), + })), + }; + hoisted.requireAuthContext.mockReset(); + hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' }); + hoisted.backfillPendingPdfParseOperation.mockReset(); + hoisted.healStaleDocumentParseState.mockClear(); + + global.fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_, reject) => { + const signal = init?.signal; + if (signal) { + signal.addEventListener('abort', () => { + reject(new DOMException('Aborted', 'AbortError')); + }, { once: true }); + } + })) as typeof fetch; + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + test('creates a worker op for legacy pending PDFs without opId', async () => { + hoisted.backfillPendingPdfParseOperation.mockResolvedValue({ + opId: 'op-legacy-1', + opKey: 'pdf_layout|v1|doc-1||doc-1|', + jobId: 'job-legacy-1', + kind: 'pdf_layout', + status: 'queued', + queuedAt: Date.now(), + progress: null, + result: undefined, + error: undefined, + updatedAt: Date.now(), + }); + + const { GET } = await import('../../src/app/api/documents/[id]/parsed/events/route'); + const controller = new AbortController(); + const request = new NextRequest('http://localhost/api/documents/doc-1/parsed/events', { + signal: controller.signal, + }); + const response = await GET(request, { + params: Promise.resolve({ id: 'doc-1' }), + }); + + expect(response.status).toBe(200); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + + const first = await reader!.read(); + const chunk = new TextDecoder().decode(first.value); + expect(chunk).toContain('event: snapshot'); + expect(chunk).toContain('"parseStatus":"pending"'); + expect(chunk).toContain('"opId":"op-legacy-1"'); + + controller.abort(); + await reader!.cancel().catch(() => {}); + + expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({ + documentId: 'doc-1', + userId: 'user-1', + namespace: null, + state: expect.objectContaining({ status: 'pending' }), + })); + const parseState = String(hoisted.row.parseState ?? ''); + expect(parseState).toContain('"status":"pending"'); + expect(parseState).toContain('"opId":"op-legacy-1"'); + expect(parseState).toContain('"jobId":"job-legacy-1"'); + }); +}); diff --git a/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts b/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts index 15b4bf1..75c5cd2 100644 --- a/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts +++ b/tests/unit/pdf-parsed-route-legacy-backfill.vitest.spec.ts @@ -14,8 +14,9 @@ const hoisted = vi.hoisted(() => ({ }, requireAuthContext: vi.fn(), fetchWorkerOperationState: vi.fn(), - backfillPendingPdfParseOperation: vi.fn(), healStaleDocumentParseState: vi.fn(async ({ state }) => state), + startPdfParseOperation: vi.fn(), + enqueueParsePdfJob: vi.fn(), })); vi.mock('@/db', () => ({ @@ -32,14 +33,18 @@ vi.mock('@/lib/server/compute/worker-op-state', () => ({ fetchWorkerOperationState: hoisted.fetchWorkerOperationState, })); -vi.mock('@/lib/server/documents/parse-state-backfill', () => ({ - backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation, -})); - vi.mock('@/lib/server/documents/parse-state-healing', () => ({ healStaleDocumentParseState: hoisted.healStaleDocumentParseState, })); +vi.mock('@/lib/server/documents/pdf-parse-operation', () => ({ + startPdfParseOperation: hoisted.startPdfParseOperation, +})); + +vi.mock('@/lib/server/jobs/user-pdf-layout-job', () => ({ + enqueueParsePdfJob: hoisted.enqueueParsePdfJob, +})); + vi.mock('@/lib/server/documents/blobstore', () => ({ documentKey: vi.fn(), getParsedDocumentBlob: vi.fn(), @@ -69,7 +74,7 @@ vi.mock('@/lib/server/logger', () => ({ hashForLog: vi.fn(() => 'user-hash'), })); -describe('GET /api/documents/[id]/parsed legacy backfill', () => { +describe('GET /api/documents/[id]/parsed pure data fetch', () => { beforeEach(async () => { process.env.BASE_URL = 'http://localhost:3003'; process.env.AUTH_SECRET = 'test-secret'; @@ -104,44 +109,25 @@ describe('GET /api/documents/[id]/parsed legacy backfill', () => { hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' }); hoisted.fetchWorkerOperationState.mockReset(); hoisted.fetchWorkerOperationState.mockResolvedValue(null); - hoisted.backfillPendingPdfParseOperation.mockReset(); hoisted.healStaleDocumentParseState.mockClear(); + hoisted.startPdfParseOperation.mockReset(); + hoisted.enqueueParsePdfJob.mockReset(); }); - test('creates a worker op for legacy pending PDFs without opId', async () => { - hoisted.backfillPendingPdfParseOperation.mockResolvedValue({ - opId: 'op-legacy-1', - opKey: 'pdf_layout|v1|doc-1||doc-1|', - jobId: 'job-legacy-1', - kind: 'pdf_layout', - status: 'queued', - queuedAt: Date.now(), - progress: null, - result: undefined, - error: undefined, - updatedAt: Date.now(), - }); - + test('returns non-ready status without creating a worker op for legacy pending PDFs without opId', async () => { const { GET } = await import('../../src/app/api/documents/[id]/parsed/route'); const request = new NextRequest('http://localhost/api/documents/doc-1/parsed'); const response = await GET(request, { params: Promise.resolve({ id: 'doc-1' }), }); - expect(response.status).toBe(202); + expect(response.status).toBe(409); await expect(response.json()).resolves.toMatchObject({ parseStatus: 'pending', - opId: 'op-legacy-1', + opId: null, }); - expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({ - documentId: 'doc-1', - userId: 'user-1', - namespace: null, - state: expect.objectContaining({ status: 'pending' }), - })); - const parseState = String(hoisted.row.parseState ?? ''); - expect(parseState).toContain('"status":"pending"'); - expect(parseState).toContain('"opId":"op-legacy-1"'); - expect(parseState).toContain('"jobId":"job-legacy-1"'); + expect(hoisted.startPdfParseOperation).not.toHaveBeenCalled(); + expect(hoisted.enqueueParsePdfJob).not.toHaveBeenCalled(); + expect(hoisted.row.parseState).toBeNull(); }); }); diff --git a/tests/unit/worker-parse-state.vitest.spec.ts b/tests/unit/worker-parse-state.vitest.spec.ts index dfdd37e..91563b3 100644 --- a/tests/unit/worker-parse-state.vitest.spec.ts +++ b/tests/unit/worker-parse-state.vitest.spec.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from 'vitest'; +import { PDF_PARSER_VERSION } from '@openreader/compute-core'; import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; import { documentParseStateFromWorkerState, @@ -11,7 +12,7 @@ function makeWorkerState( ): WorkerOperationState { return { opId: 'op-123', - opKey: 'pdf_layout|v1|doc-1', + opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`, kind: 'pdf_layout', jobId: 'job-123', status: 'queued', @@ -41,6 +42,7 @@ describe('worker parse state mapping', () => { status: 'pending', progress: null, updatedAt: 1234, + parserVersion: PDF_PARSER_VERSION, opId: 'op-123', jobId: 'job-123', }); @@ -66,6 +68,7 @@ describe('worker parse state mapping', () => { phase: 'infer', }, updatedAt: 5678, + parserVersion: PDF_PARSER_VERSION, opId: 'op-123', jobId: 'job-123', }); @@ -84,6 +87,7 @@ describe('worker parse state mapping', () => { status: 'failed', progress: null, updatedAt: 9999, + parserVersion: PDF_PARSER_VERSION, opId: 'op-123', jobId: 'job-123', error: 'layout model crashed', diff --git a/tests/upload.spec.ts b/tests/upload.spec.ts index 2008f72..dad3ae0 100644 --- a/tests/upload.spec.ts +++ b/tests/upload.spec.ts @@ -30,6 +30,29 @@ test.describe('Document Upload Tests', () => { await expectDocumentListed(page, 'sample.txt'); }); + test('reuses the same canonical id for identical uploads', async ({ page }) => { + await uploadFile(page, 'sample.pdf'); + await uploadFile(page, 'sample.pdf'); + + const result = await page.evaluate(async () => { + const res = await fetch('/api/documents', { cache: 'no-store' }); + if (!res.ok) { + return { ok: false as const, reason: `status:${res.status}` }; + } + const data = await res.json() as { documents?: Array<{ id: string; name: string }> }; + const matching = (data.documents || []).filter((doc) => doc.name === 'sample.pdf'); + const uniqueIds = Array.from(new Set(matching.map((doc) => doc.id))); + return { ok: true as const, matchingCount: matching.length, uniqueIds }; + }); + + expect(result.ok).toBeTruthy(); + if (!result.ok) { + throw new Error(`Failed to inspect uploaded documents: ${result.reason}`); + } + expect(result.matchingCount).toBe(1); + expect(result.uniqueIds).toHaveLength(1); + }); + test('hashes text/HTML docs using UTF-8 encoded stored string', async ({ page }) => { await uploadFile(page, 'sample.txt'); await expectDocumentListed(page, 'sample.txt'); @@ -77,13 +100,7 @@ test.describe('Document Upload Tests', () => { test('uploads and converts a DOCX document', async ({ page }) => { await uploadFile(page, 'sample.docx'); - // Should see the converting message (best-effort; conversion may complete extremely fast) - try { - await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 }); - } catch { - // ignore - } - // After conversion, should see the PDF with the same name + // DOCX uploads are normalized into stored PDFs with the same basename. await expectDocumentListed(page, 'sample.pdf'); });