Merge pull request #99 from richardr1126/pdf/parse-fixes

## Summary by CodeRabbit

* **New Features**
  * Expose PDF parser version system-wide; parser-version-aware parsing, reuse, and caching. Added tokenized temp upload flow, presign/finalize endpoints, and server-side DOCX→PDF conversion.
  * Playback readiness propagated to TTS players/pages; unified upload/delete APIs and simplified uploader/sidebar UX.

* **Bug Fixes**
  * Normalize stored parse state to current parser version to avoid stale readiness and improve reuse.
  * Prevent premature playback starts and redundant preparation.

* **Tests**
  * Added unit and E2E tests for parser-version handling, op-key inclusion, parse-state behavior, and canonical upload reuse.
This commit is contained in:
Richard R 2026-06-04 18:11:31 -06:00 committed by GitHub
commit e2c0240add
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
46 changed files with 1786 additions and 1049 deletions

View file

@ -16,6 +16,7 @@ export {
} from './config/timeout';
export { renderPage } from './pdf/render';
export { mergeTextWithRegions } from './pdf/merge';
export { PDF_PARSER_VERSION } from './pdf/parser-version';
export { stitchCrossPageBlocks } from './pdf/stitch';
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';

View file

@ -4,6 +4,7 @@ import { ensureModel } from './model';
import { runLayoutModel } from './runLayoutModel';
import { mergeTextWithRegions } from './merge';
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
import { PDF_PARSER_VERSION } from './parser-version';
import { stitchCrossPageBlocks } from './stitch';
import { renderPage } from './render';
import { normalizeTextItemsForLayout } from './normalize-text';
@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
const doc: ParsedPdfDocument = {
schemaVersion: 1,
documentId: input.documentId,
parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69',
parserVersion: PDF_PARSER_VERSION,
parsedAt: Date.now(),
pages,
};

View file

@ -0,0 +1 @@
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';

View file

@ -1,12 +1,15 @@
import fs from 'node:fs';
import { createRequire } from 'node:module';
import path from 'node:path';
let cachedStandardFontDataUrl: string | null = null;
const require = createRequire(import.meta.url);
export function resolvePdfjsStandardFontDataUrl(): string {
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json'));
const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts');
if (!fs.existsSync(standardFontDir)) {
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);

View file

@ -31,6 +31,7 @@ export default function EPUBPage() {
const {
setCurrentDocument,
currDocName,
isPlaybackReady,
clearCurrDoc,
createFullAudioBook: createEPUBAudioBook,
regenerateChapter: regenerateEPUBChapter,
@ -242,7 +243,7 @@ export default function EPUBPage() {
</div>
</div>
) : (
<TTSPlayer />
<TTSPlayer isPlaybackReady={isPlaybackReady} />
)}
<DocumentSettings
epub

View file

@ -51,6 +51,7 @@ export interface EpubDocumentState {
currDocPages: number | undefined;
currDocPage: number | string;
currDocText: string | undefined;
isPlaybackReady: boolean;
setCurrentDocument: (id: string) => Promise<void>;
clearCurrDoc: () => void;
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
const [currDocName, setCurrDocName] = useState<string>();
const [currDocText, setCurrDocText] = useState<string>();
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
// Mirror state into a ref so resolveEpubLocator (registered once with
// TTSContext via a stable callback) can always read the latest page text
// without forcing re-registration on every page turn.
@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
setCurrDocData(undefined);
setCurrDocName(undefined);
setCurrDocText(undefined);
setIsPlaybackReady(false);
setCurrDocPages(undefined);
isEPUBSetOnce.current = false;
shouldPauseRef.current = true;
@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
*/
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
try {
setIsPlaybackReady(false);
const meta = await getDocumentMetadata(id);
if (!meta) {
clearCurrDoc();
@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
*/
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
try {
setIsPlaybackReady(false);
const location = rendition?.location;
if (!location) return '';
const { start, end } = location;
@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
nextText: continuationPreview
});
setCurrDocText(textContent);
setIsPlaybackReady(true);
return textContent;
} catch (error) {
@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
currDocPages,
currDocPage,
currDocText,
isPlaybackReady,
clearCurrDoc,
extractPageText,
walkUpcomingRenderedLocations,
@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
currDocPages,
currDocPage,
currDocText,
isPlaybackReady,
clearCurrDoc,
extractPageText,
walkUpcomingRenderedLocations,

View file

@ -31,6 +31,7 @@ export default function HTMLPage() {
setCurrentDocument,
currDocData,
currDocName,
isPlaybackReady,
blocks,
isTxt,
clearCurrDoc,
@ -233,7 +234,7 @@ export default function HTMLPage() {
</div>
</div>
) : (
<TTSPlayer />
<TTSPlayer isPlaybackReady={isPlaybackReady} />
)}
<DocumentSettings
html

View file

@ -22,6 +22,7 @@ export interface HtmlDocumentState {
currDocData: string | undefined;
currDocName: string | undefined;
currDocText: string | undefined;
isPlaybackReady: boolean;
blocks: HtmlBlock[];
isTxt: boolean;
setCurrentDocument: (id: string) => Promise<void>;
@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState {
const [currDocData, setCurrDocData] = useState<string>();
const [currDocName, setCurrDocName] = useState<string>();
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
const blocks = useMemo(
@ -91,22 +93,43 @@ export function useHtmlDocument(): HtmlDocumentState {
// sentence splitting + sequential advancement from there.
const lastFedDocRef = useRef<string | null>(null);
useEffect(() => {
if (!currDocText) return;
const key = `${currDocData ?? ''}::${currDocText.length}`;
if (lastFedDocRef.current === key) return;
if (currDocData === undefined) {
lastFedDocRef.current = null;
setTTSText('');
setIsPlaybackReady(false);
return;
}
if (!currDocText) {
lastFedDocRef.current = null;
setTTSText('');
setIsPlaybackReady(true);
return;
}
const key = `${currDocName ?? ''}::${currDocData ?? ''}::${currDocText.length}`;
if (lastFedDocRef.current === key) {
setIsPlaybackReady(true);
return;
}
setIsPlaybackReady(false);
lastFedDocRef.current = key;
setTTSText(currDocText);
}, [currDocText, currDocData, setTTSText]);
setIsPlaybackReady(true);
}, [currDocName, currDocText, currDocData, setTTSText]);
const clearCurrDoc = useCallback(() => {
setCurrDocData(undefined);
setCurrDocName(undefined);
setIsPlaybackReady(false);
lastFedDocRef.current = null;
setTTSText('');
stop();
}, [stop]);
}, [setTTSText, stop]);
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
try {
setIsPlaybackReady(false);
lastFedDocRef.current = null;
setTTSText('');
const meta = await getDocumentMetadata(id);
if (!meta) {
console.error('Document not found on server');
@ -125,7 +148,7 @@ export function useHtmlDocument(): HtmlDocumentState {
console.error('Failed to get HTML document:', error);
clearCurrDoc();
}
}, [clearCurrDoc]);
}, [clearCurrDoc, setTTSText]);
const audiobookAdapter = useMemo(
() =>
@ -207,6 +230,7 @@ export function useHtmlDocument(): HtmlDocumentState {
currDocData,
currDocName,
currDocText,
isPlaybackReady,
blocks,
isTxt,
setCurrentDocument,
@ -218,6 +242,7 @@ export function useHtmlDocument(): HtmlDocumentState {
currDocData,
currDocName,
currDocText,
isPlaybackReady,
blocks,
isTxt,
setCurrentDocument,

View file

@ -52,6 +52,7 @@ export default function PDFViewerPage() {
clearCurrDoc,
currDocPage,
currDocPages,
isPlaybackReady,
parseStatus,
parseProgress,
documentSettings,
@ -487,7 +488,7 @@ export default function PDFViewerPage() {
</div>
</div>
) : isParseReady ? (
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} />
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} isPlaybackReady={isPlaybackReady} />
) : null}
<DocumentSettings
isOpen={activeSidebar === 'settings'}

View file

@ -66,6 +66,7 @@ export interface PdfDocumentState {
currDocPages: number | undefined;
currDocPage: number;
currDocText: string | undefined;
isPlaybackReady: boolean;
pdfDocument: PDFDocumentProxy | undefined;
parsedDocument: ParsedPdfDocument | null;
parseStatus: PdfParseStatus | null;
@ -142,6 +143,7 @@ export function usePdfDocument(): PdfDocumentState {
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
const [currDocName, setCurrDocName] = useState<string>();
const [currDocText, setCurrDocText] = useState<string>();
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
@ -167,52 +169,29 @@ export function usePdfDocument(): PdfDocumentState {
// Guards for setCurrentDocument to prevent stale loads from overwriting newer selections.
const docLoadSeqRef = useRef(0);
const docLoadAbortRef = useRef<AbortController | null>(null);
const parsePollAbortRef = useRef<AbortController | null>(null);
const parseStreamAbortRef = useRef<AbortController | null>(null);
const parseSseCloseRef = useRef<(() => void) | null>(null);
const lastPreparedPlaybackPageRef = useRef<number | null>(null);
const fetchParsedDocument = useCallback(async (
const loadParsedDocumentOnce = useCallback(async (
documentId: string,
initialStatus: PdfParseStatus | null,
signal: AbortSignal,
initialOpId?: string | null,
): Promise<void> => {
// Legacy PDFs may have null parseStatus; treat as pending so opening the
// document backfills parse output via the parsed endpoint polling path.
const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending';
setParseStatus(effectiveInitialStatus);
if (signal.aborted) return;
const parsed = await getParsedPdfDocument(documentId, { signal });
if (signal.aborted) return;
setParsedDocument(parsed);
setParseStatus('ready');
setParseProgress(null);
const delayMs = 1200;
const retryFailed = effectiveInitialStatus === 'failed';
let attempt = 0;
let effectiveOpId = initialOpId?.trim() || null;
while (!signal.aborted) {
if (signal.aborted) return;
const result = await getParsedPdfDocument(documentId, {
signal,
retryFailed: retryFailed && attempt === 0,
...(effectiveOpId ? { opId: effectiveOpId } : {}),
});
if (result.status === 'ready') {
setParsedDocument(result.parsed);
setParseStatus('ready');
setParseProgress(null);
setActiveParseOpId(null);
return;
}
if ('opId' in result && typeof result.opId === 'string' && result.opId.trim()) {
effectiveOpId = result.opId.trim();
setActiveParseOpId(effectiveOpId);
}
setParseStatus(result.status);
setParseProgress(result.parseProgress ?? null);
if (result.status === 'failed') {
setParsedDocument(null);
setActiveParseOpId(null);
return;
}
attempt += 1;
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
setActiveParseOpId(null);
}, []);
const resetParsedDocumentState = useCallback(() => {
setParsedDocument(null);
setCurrDocText(undefined);
setIsPlaybackReady(false);
lastPreparedPlaybackPageRef.current = null;
pageTextCacheRef.current.clear();
}, []);
const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise<void> => {
@ -226,42 +205,57 @@ export function usePdfDocument(): PdfDocumentState {
}
}, []);
const startParsedPolling = useCallback((documentId: string, initialOpId?: string | null) => {
parsePollAbortRef.current?.abort();
const startParsedEventStream = useCallback((documentId: string, initialOpId?: string | null) => {
parseStreamAbortRef.current?.abort();
parseSseCloseRef.current?.();
parseSseCloseRef.current = null;
setParseProgress(null);
setActiveParseOpId(initialOpId?.trim() || null);
const controller = new AbortController();
parsePollAbortRef.current = controller;
parseStreamAbortRef.current = controller;
let isResolvingTerminalState = false;
const closeSse = subscribeParsedPdfDocumentEvents(documentId, {
opId: initialOpId?.trim() || null,
}, {
onSnapshot: (snapshot) => {
if (controller.signal.aborted) return;
if (isResolvingTerminalState) return;
if (typeof snapshot.opId === 'string' && snapshot.opId.trim()) {
setActiveParseOpId(snapshot.opId.trim());
}
setParseStatus(snapshot.parseStatus);
setParseProgress(snapshot.parseProgress);
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
if (snapshot.parseStatus === 'failed') {
setParsedDocument(null);
setActiveParseOpId(null);
} else {
void fetchParsedDocument(
documentId,
'ready',
controller.signal,
typeof snapshot.opId === 'string' ? snapshot.opId : (initialOpId ?? null),
);
}
if (snapshot.parseStatus === 'ready') {
isResolvingTerminalState = true;
void (async () => {
try {
await loadParsedDocumentOnce(documentId, controller.signal);
} catch (error) {
if (error instanceof DOMException && error.name === 'AbortError') return;
console.error('Failed to load parsed PDF after ready status:', error);
resetParsedDocumentState();
} finally {
if (parseSseCloseRef.current === closeSse) {
closeSse();
parseSseCloseRef.current = null;
}
if (parseStreamAbortRef.current === controller) {
parseStreamAbortRef.current = null;
}
}
})();
return;
}
if (snapshot.parseStatus === 'failed') {
isResolvingTerminalState = true;
closeSse();
parseSseCloseRef.current = null;
if (parsePollAbortRef.current === controller) {
parsePollAbortRef.current = null;
if (parseStreamAbortRef.current === controller) {
parseStreamAbortRef.current = null;
}
resetParsedDocumentState();
setActiveParseOpId(null);
return;
}
},
@ -281,11 +275,11 @@ export function usePdfDocument(): PdfDocumentState {
if (parseSseCloseRef.current === closeSse) {
parseSseCloseRef.current = null;
}
if (parsePollAbortRef.current === controller) {
parsePollAbortRef.current = null;
if (parseStreamAbortRef.current === controller) {
parseStreamAbortRef.current = null;
}
}, { once: true });
}, [fetchParsedDocument, setActiveParseOpId]);
}, [loadParsedDocumentOnce, resetParsedDocumentState, setActiveParseOpId]);
useEffect(() => {
pdfDocumentRef.current = pdfDocument;
@ -297,6 +291,7 @@ export function usePdfDocument(): PdfDocumentState {
useEffect(() => {
setCurrDocPage(currDocPageNumber);
setIsPlaybackReady(false);
}, [currDocPageNumber]);
/**
@ -324,11 +319,13 @@ export function usePdfDocument(): PdfDocumentState {
if (!currentPdf) return;
const seq = ++loadSeqRef.current;
const pageNumber = currDocPageNumber;
setIsPlaybackReady(false);
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
if (parseStatus !== 'ready' || !parsedDocument) {
lastPreparedPlaybackPageRef.current = null;
setCurrDocText(undefined);
setTTSText('', { location: currDocPageNumber });
return;
@ -401,7 +398,8 @@ export function usePdfDocument(): PdfDocumentState {
return;
}
if (text !== currDocText || text === '') {
const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber;
if (shouldPreparePlayback) {
setCurrDocText(text);
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
setTTSText(text, {
@ -414,6 +412,8 @@ export function usePdfDocument(): PdfDocumentState {
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
});
}
lastPreparedPlaybackPageRef.current = currDocPageNumber;
setIsPlaybackReady(true);
} catch (error) {
if (error instanceof DOMException && error.name === 'AbortError') {
return;
@ -461,14 +461,16 @@ export function usePdfDocument(): PdfDocumentState {
// or fast refresh.
pdfDocGenerationRef.current += 1;
loadSeqRef.current += 1;
parsePollAbortRef.current?.abort();
parsePollAbortRef.current = null;
parseStreamAbortRef.current?.abort();
parseStreamAbortRef.current = null;
parseSseCloseRef.current?.();
parseSseCloseRef.current = null;
pageTextCacheRef.current.clear();
setPdfDocument(undefined);
setCurrDocPages(undefined);
setCurrDocText(undefined);
setIsPlaybackReady(false);
lastPreparedPlaybackPageRef.current = null;
setCurrDocId(id);
setCurrDocName(undefined);
setCurrDocData(undefined);
@ -489,7 +491,7 @@ export function usePdfDocument(): PdfDocumentState {
setParseStatus(initialParseStatus);
setParseProgress(null);
setActiveParseOpId(null);
startParsedPolling(id, null);
startParsedEventStream(id, null);
void fetchDocumentSettings(id, controller.signal);
}
@ -524,7 +526,7 @@ export function usePdfDocument(): PdfDocumentState {
setCurrDocText,
setPdfDocument,
fetchDocumentSettings,
startParsedPolling,
startParsedEventStream,
]);
const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise<void> => {
@ -546,14 +548,16 @@ export function usePdfDocument(): PdfDocumentState {
pageTextCacheRef.current.clear();
setParsedDocument(null);
setCurrDocText(undefined);
setIsPlaybackReady(false);
lastPreparedPlaybackPageRef.current = null;
setParseStatus(forced.status);
setParseProgress(null);
setActiveParseOpId(forced.opId ?? null);
startParsedPolling(currDocId, forced.opId ?? null);
startParsedEventStream(currDocId, forced.opId ?? null);
} catch (error) {
console.error('Failed to force PDF reparse:', error);
}
}, [currDocId, startParsedPolling]);
}, [currDocId, startParsedEventStream]);
/**
* Clears the current document state
@ -567,14 +571,15 @@ export function usePdfDocument(): PdfDocumentState {
docLoadSeqRef.current += 1;
docLoadAbortRef.current?.abort();
docLoadAbortRef.current = null;
parsePollAbortRef.current?.abort();
parsePollAbortRef.current = null;
parseStreamAbortRef.current?.abort();
parseStreamAbortRef.current = null;
parseSseCloseRef.current?.();
parseSseCloseRef.current = null;
setCurrDocId(undefined);
setCurrDocName(undefined);
setCurrDocData(undefined);
setCurrDocText(undefined);
setIsPlaybackReady(false);
setCurrDocPages(undefined);
setPdfDocument(undefined);
setParsedDocument(null);
@ -582,6 +587,7 @@ export function usePdfDocument(): PdfDocumentState {
setParseProgress(null);
setActiveParseOpId(null);
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
lastPreparedPlaybackPageRef.current = null;
pageTextCacheRef.current.clear();
stop();
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
@ -681,6 +687,7 @@ export function usePdfDocument(): PdfDocumentState {
currDocPages,
currDocPage,
currDocText,
isPlaybackReady,
parsedDocument,
parseStatus,
parseProgress,
@ -708,6 +715,7 @@ export function usePdfDocument(): PdfDocumentState {
currDocPages,
currDocPage,
currDocText,
isPlaybackReady,
parsedDocument,
parseStatus,
parseProgress,

View file

@ -5,10 +5,15 @@ import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker';
import { isAbortLikeError } from '@/lib/server/compute/abort-like-error';
import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/server/compute/worker-parse-state';
import {
isWorkerOperationStateStale,
mergeNonReadyParseSnapshot,
snapshotFromWorkerState,
} from '@/lib/server/compute/worker-parse-state';
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
import {
normalizeDocumentParseStateForCurrentParserVersion,
normalizeParseStatus,
parseDocumentParseState,
stringifyDocumentParseState,
@ -67,7 +72,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
const state = await healStaleDocumentParseState({
documentId: row.id,
userId: row.userId,
state: parseDocumentParseState(row.parseState),
state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)),
});
const parseStatus = normalizeParseStatus(state.status);
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
@ -82,13 +87,20 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
&& workerState.opId === opId
&& !isWorkerOperationStateStale(workerState, opStaleMs)
) {
const merged = mergeNonReadyParseSnapshot({
parseStatus,
parseProgress: state.progress ?? null,
workerState,
});
const workerSnapshot = snapshotFromWorkerState(workerState);
const fromWorker = workerSnapshot.parseStatus === 'pending' || workerSnapshot.parseStatus === 'running';
return {
snapshot: {
...snapshotFromWorkerState(workerState),
...merged,
opId: workerState.opId,
},
opId: workerState.opId,
fromWorker: true,
fromWorker,
};
}
}
@ -199,7 +211,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
let initialState = await toSnapshotState(row, requestedOpId);
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
const state = parseDocumentParseState(row.parseState);
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
const created = await backfillPendingPdfParseOperation({
documentId: id,
userId: row.userId,
@ -261,22 +273,19 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
let current = initialState.snapshot;
let signature = JSON.stringify(current);
let currentOpId = requestedOpId ?? initialState.opId;
let currentFromWorker = initialState.fromWorker;
let lastEventId: number | null = null;
let loggedMissingOpId = false;
const pinnedRequestedOp = requestedOpId ?? null;
const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot, fromWorker: boolean): boolean => {
const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot): boolean => {
const isTerminal = snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed';
if (!isTerminal) return false;
// If caller pinned an opId, keep streaming until worker confirms that
// op state. DB fallback can report stale terminal status for other rows.
if (pinnedRequestedOp && snapshot.opId === pinnedRequestedOp && !fromWorker) return false;
if (pinnedRequestedOp && snapshot.opId !== pinnedRequestedOp) return false;
return true;
};
writeSnapshot(current);
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
if (shouldCloseForTerminalSnapshot(current)) {
closeStream();
return;
}
@ -303,7 +312,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
}
current = next.snapshot;
signature = next.signature;
currentFromWorker = next.fromWorker;
if (!requestedOpId && next.opId !== currentOpId) {
currentOpId = next.opId;
lastEventId = null;
@ -312,7 +320,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
workerAbort = null;
}
}
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
if (shouldCloseForTerminalSnapshot(current)) {
closeStream();
}
}).catch((error) => {
@ -349,7 +357,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
}
current = next.snapshot;
signature = next.signature;
currentFromWorker = next.fromWorker;
currentOpId = requestedOpId ?? next.opId;
if (!currentOpId) {
if (!loggedMissingOpId) {
@ -357,7 +364,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
logger.warn({
event: 'documents.parsed.events.missing_opid_non_terminal',
degraded: true,
step: 'poll_without_worker_op',
step: 'missing_opid_fallback',
documentId: id,
storageUserIdHash,
parseStatus: current.parseStatus,
@ -367,7 +374,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
} else if (loggedMissingOpId) {
loggedMissingOpId = false;
}
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
if (shouldCloseForTerminalSnapshot(current)) {
closeStream();
return;
}
@ -471,19 +478,23 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
);
if (!workerSnapshot || workerSnapshot.opId !== currentOpId) continue;
const mergedSnapshot = mergeNonReadyParseSnapshot({
parseStatus: current.parseStatus,
parseProgress: current.parseProgress,
workerState: workerSnapshot,
});
const nextSnapshot: ParsedSnapshot = {
...snapshotFromWorkerState(workerSnapshot),
...mergedSnapshot,
opId: workerSnapshot.opId,
};
const nextSignature = JSON.stringify(nextSnapshot);
if (nextSignature !== signature) {
current = nextSnapshot;
signature = nextSignature;
currentFromWorker = true;
writeSnapshot(current);
}
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
if (shouldCloseForTerminalSnapshot(current)) {
closeStream();
return;
}
@ -522,12 +533,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
}
current = next.snapshot;
signature = next.signature;
currentFromWorker = next.fromWorker;
if (!requestedOpId && next.opId !== currentOpId) {
currentOpId = next.opId;
lastEventId = null;
}
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
if (shouldCloseForTerminalSnapshot(current)) {
closeStream();
return;
}

View file

@ -4,39 +4,36 @@ import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
import {
documentParseStateFromWorkerState,
isWorkerOperationStateStale,
snapshotFromWorkerState,
} from '@/lib/server/compute/worker-parse-state';
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
import {
documentKey,
getParsedDocumentBlob,
getParsedDocumentBlobByKey,
isMissingBlobError,
isValidDocumentId,
putParsedDocumentBlob,
} from '@/lib/server/documents/blobstore';
import {
normalizeDocumentParseStateForCurrentParserVersion,
normalizeParseStatus,
parseDocumentParseState,
stringifyDocumentParseState,
} from '@/lib/server/documents/parse-state';
import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill';
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
import { checkJobRate, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { isS3Configured } from '@/lib/server/storage/s3';
import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger';
import { createRequestLogger, hashForLog } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response';
import { logDegraded } from '@/lib/server/errors/logging';
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
import { getComputeOpStaleMs } from '@openreader/compute-core';
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
import type { PdfLayoutJobResult } from '@openreader/compute-core/api-contracts';
export const dynamic = 'force-dynamic';
@ -100,102 +97,12 @@ async function writeParseRowState(input: {
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
}
async function finalizeFromWorkerState(input: {
workerState: WorkerOperationState<PdfLayoutJobResult>;
row: ParseRow;
namespace: string | null;
logger: ServerLogger;
}): Promise<NextResponse> {
const snapshot = snapshotFromWorkerState(input.workerState);
const workerParseState = documentParseStateFromWorkerState(input.workerState);
if (snapshot.parseStatus === 'pending' || snapshot.parseStatus === 'running') {
await writeParseRowState({
documentId: input.row.id,
userId: input.row.userId,
parseState: stringifyDocumentParseState(workerParseState),
});
return NextResponse.json({
parseStatus: snapshot.parseStatus,
parseProgress: snapshot.parseProgress,
opId: input.workerState.opId,
}, { status: 202 });
}
if (snapshot.parseStatus === 'failed') {
await writeParseRowState({
documentId: input.row.id,
userId: input.row.userId,
parseState: stringifyDocumentParseState(workerParseState),
});
return NextResponse.json({
parseStatus: 'failed',
parseProgress: null,
opId: input.workerState.opId,
error: input.workerState.error?.message ?? 'Worker parse failed',
}, { status: 202 });
}
let parsedJsonKey: string | null = null;
if (input.workerState.result && 'parsedObjectKey' in input.workerState.result) {
parsedJsonKey = typeof input.workerState.result.parsedObjectKey === 'string'
? input.workerState.result.parsedObjectKey
: null;
}
if (!parsedJsonKey && input.workerState.result && 'parsed' in input.workerState.result && input.workerState.result.parsed) {
const parsedJson = Buffer.from(JSON.stringify(input.workerState.result.parsed));
parsedJsonKey = await putParsedDocumentBlob(input.row.id, parsedJson, input.namespace);
}
if (!parsedJsonKey) {
return errorResponse(new Error('Worker completed without parsed output'), {
apiErrorMessage: 'Worker completed without parsed output',
normalize: { code: 'DOCUMENTS_PARSED_WORKER_OUTPUT_MISSING', errorClass: 'upstream' },
});
}
await writeParseRowState({
documentId: input.row.id,
userId: input.row.userId,
parseState: stringifyDocumentParseState(workerParseState),
parsedJsonKey,
});
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
let parsedDoc: ParsedPdfDocument | null = null;
try {
parsedDoc = JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
} catch {
parsedDoc = null;
}
if (!hasAnyParsedBlocks(parsedDoc)) {
input.logger.warn({
event: 'documents.parsed.no_blocks_in_output',
documentId: input.row.id,
userIdHash: hashForLog(input.row.userId),
parsedJsonKey,
}, 'Worker output parsed successfully but contained no blocks');
}
return new NextResponse(new Uint8Array(json), {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
},
});
}
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
const { logger } = createRequestLogger({
route: '/api/documents/[id]/parsed',
request: req,
});
try {
const opStaleMs = getComputeOpStaleMs();
if (!isS3Configured()) return s3NotConfiguredResponse();
const authCtxOrRes = await requireAuthContext(req);
@ -204,8 +111,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
const params = await ctx.params;
const id = (params.id || '').trim().toLowerCase();
const retryFailed = req.nextUrl.searchParams.get('retry') === '1';
const requestedOpId = normalizeOpId(req.nextUrl.searchParams.get('opId'));
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
}
@ -220,112 +125,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
if (requestedOpId) {
const workerState = await fetchWorkerOperationState<PdfLayoutJobResult>(requestedOpId);
if (workerState && workerState.opId === requestedOpId) {
return finalizeFromWorkerState({
workerState,
row,
namespace: testNamespace,
logger,
});
}
logDegraded(logger, {
event: 'documents.parsed.requested_op_unavailable',
msg: 'Requested worker operation id was unavailable',
step: 'requested_op_lookup',
context: {
documentId: id,
userIdHash: hashForLog(row.userId),
opId: requestedOpId,
error: {
name: 'RequestedOperationUnavailable',
message: `Requested worker operation ${requestedOpId} was unavailable`,
},
},
});
}
let state = parseDocumentParseState(row.parseState);
state = await healStaleDocumentParseState({
documentId: id,
userId: row.userId,
state,
});
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
const effectiveStatus = normalizeParseStatus(state.status);
const effectiveProgress = state.progress ?? null;
const effectiveOpId = normalizeOpId(state.opId);
if (effectiveOpId && effectiveStatus !== 'ready') {
const workerState = await fetchWorkerOperationState<PdfLayoutJobResult>(effectiveOpId);
if (
workerState
&& workerState.opId === effectiveOpId
&& !isWorkerOperationStateStale(workerState, opStaleMs)
) {
return finalizeFromWorkerState({
workerState,
row,
namespace: testNamespace,
logger,
});
}
}
if (!effectiveOpId && (effectiveStatus === 'pending' || effectiveStatus === 'running')) {
const created = await backfillPendingPdfParseOperation({
documentId: id,
userId: row.userId,
namespace: testNamespace,
state,
});
if (created) {
const snapshot = snapshotFromWorkerState(created);
await writeParseRowState({
documentId: row.id,
userId: row.userId,
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
});
return NextResponse.json({
parseStatus: snapshot.parseStatus,
parseProgress: snapshot.parseProgress,
opId: created.opId,
}, { status: 202 });
}
}
if (effectiveStatus === 'failed' && retryFailed) {
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig);
if (!rateDecision.allowed) {
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
}
const created = await createOrReusePdfWorkerOperation({
documentId: id,
namespace: testNamespace,
documentObjectKey: documentKey(id, testNamespace),
});
await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig);
const snapshot = snapshotFromWorkerState(created);
await writeParseRowState({
documentId: row.id,
userId: row.userId,
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
});
return NextResponse.json({
parseStatus: snapshot.parseStatus,
parseProgress: snapshot.parseProgress,
opId: created.opId,
}, { status: 202 });
}
if (effectiveStatus !== 'ready') {
return NextResponse.json({
parseStatus: effectiveStatus,
parseProgress: effectiveProgress,
opId: effectiveOpId,
}, { status: 202 });
}, { status: 409 });
}
try {
@ -409,7 +219,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
let state = parseDocumentParseState(row.parseState);
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
state = await healStaleDocumentParseState({
documentId: id,
userId: row.userId,
@ -441,25 +251,33 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
}
const created = await createOrReusePdfWorkerOperation({
const forceToken = randomUUID();
const startedParse = await startPdfParseOperation({
documentId: id,
userId: authCtxOrRes.userId,
namespace: testNamespace,
documentObjectKey: documentKey(id, testNamespace),
forceToken: randomUUID(),
forceToken,
});
await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig);
const snapshot = snapshotFromWorkerState(created);
const snapshot = snapshotFromWorkerState(startedParse.workerState);
await writeParseRowState({
documentId: row.id,
userId: row.userId,
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
parseState: stringifyDocumentParseState(startedParse.parseState),
});
enqueueParsePdfJob({
documentId: id,
userId: row.userId,
namespace: testNamespace,
forceToken,
initialOpId: startedParse.workerState.opId,
initialJobId: startedParse.workerState.jobId,
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
});
return NextResponse.json({
parseStatus: snapshot.parseStatus,
parseProgress: snapshot.parseProgress,
opId: created.opId,
opId: startedParse.workerState.opId,
}, { status: 202 });
} catch (error) {
return errorResponse(error, {

View file

@ -1,6 +1,6 @@
import { NextRequest, NextResponse } from 'next/server';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore';
import { isValidTempUploadToken, putTempDocumentBlob } from '@/lib/server/documents/blobstore';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { isS3Configured } from '@/lib/server/storage/s3';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
@ -26,11 +26,12 @@ export async function PUT(req: NextRequest) {
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
const token = (url.searchParams.get('token') || '').trim().toLowerCase();
if (!isValidTempUploadToken(token)) {
return NextResponse.json({ error: 'Invalid upload token' }, { status: 400 });
}
const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream';
@ -86,7 +87,7 @@ export async function PUT(req: NextRequest) {
const namespace = getOpenReaderTestNamespace(req.headers);
try {
await putDocumentBlob(id, body, contentType, namespace);
await putTempDocumentBlob(token, ctxOrRes.userId, body, contentType, namespace);
} catch (error) {
if (!isPreconditionFailed(error)) {
throw error;
@ -97,12 +98,12 @@ export async function PUT(req: NextRequest) {
event: 'documents.blob.upload.fallback.proxy_used',
degraded: true,
fallbackPath: 'upload_proxy',
documentId: id,
uploadToken: token,
contentType,
bytes: body.byteLength,
}, 'Document upload fallback proxy used');
return NextResponse.json({ success: true, id });
return NextResponse.json({ success: true, token });
} catch (error) {
serverLogger.error({
event: 'documents.blob.upload.fallback.failed',

View file

@ -0,0 +1,257 @@
import path from 'path';
import { createHash } from 'node:crypto';
import { NextRequest, NextResponse } from 'next/server';
import { requireAuthContext } from '@/lib/server/auth/auth';
import {
TEMP_DOCUMENT_UPLOAD_TTL_MS,
copyTempDocumentBlobToDocument,
deleteTempDocumentUpload,
getTempDocumentBlob,
getTempDocumentFinalizeReceipt,
headDocumentBlob,
headTempDocumentBlob,
isMissingBlobError,
isPreconditionFailed,
isValidTempUploadToken,
putDocumentBlob,
putTempDocumentFinalizeReceipt,
} from '@/lib/server/documents/blobstore';
import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert';
import { registerUploadedDocument } from '@/lib/server/documents/register-upload';
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
import { errorResponse } from '@/lib/server/errors/next-response';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
import type { BaseDocument, DocumentType } from '@/types/documents';
export const dynamic = 'force-dynamic';
type FinalizeUpload = {
token: string;
name: string;
type: DocumentType;
lastModified: number;
};
type FinalizeReceipt = {
stored: BaseDocument;
};
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType {
if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') {
return rawType;
}
return toDocumentTypeFromName(safeName);
}
function normalizeLastModified(value: unknown): number {
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
}
function parseFinalizePayload(body: unknown): FinalizeUpload[] {
if (!body || typeof body !== 'object') return [];
const rawUploads = (body as { uploads?: unknown }).uploads;
if (!Array.isArray(rawUploads)) return [];
const uploads: FinalizeUpload[] = [];
for (const rawUpload of rawUploads) {
if (!rawUpload || typeof rawUpload !== 'object') continue;
const rec = rawUpload as Record<string, unknown>;
const token = typeof rec.token === 'string' ? rec.token.trim().toLowerCase() : '';
if (!isValidTempUploadToken(token)) continue;
const fallbackName = `upload-${token}.txt`;
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
uploads.push({
token,
name,
type: normalizeDocumentType(rec.type, name),
lastModified: normalizeLastModified(rec.lastModified),
});
}
return uploads;
}
async function loadTempUpload(input: {
token: string;
userId: string;
namespace: string | null;
}): Promise<{ contentType: string; size: number; lastModified: number; body: Buffer }> {
const RETRIES = 3;
const RETRY_DELAY_MS = 500;
let lastError: unknown = null;
for (let attempt = 0; attempt < RETRIES; attempt += 1) {
try {
const head = await headTempDocumentBlob(input.token, input.userId, input.namespace);
const body = await getTempDocumentBlob(input.token, input.userId, input.namespace);
return {
contentType: head.contentType ?? 'application/octet-stream',
size: head.contentLength > 0 ? head.contentLength : body.byteLength,
lastModified: head.lastModified ?? Date.now(),
body,
};
} catch (error) {
lastError = error;
if (isMissingBlobError(error) && attempt < RETRIES - 1) {
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));
continue;
}
throw error;
}
}
throw lastError instanceof Error ? lastError : new Error('Temporary upload is unavailable');
}
async function finalizeOne(input: {
upload: FinalizeUpload;
userId: string;
namespace: string | null;
}): Promise<BaseDocument> {
const existingReceipt = await getTempDocumentFinalizeReceipt<FinalizeReceipt>(
input.upload.token,
input.userId,
input.namespace,
);
if (existingReceipt?.stored) {
return existingReceipt.stored;
}
const temp = await loadTempUpload({
token: input.upload.token,
userId: input.userId,
namespace: input.namespace,
});
if (Date.now() - temp.lastModified > TEMP_DOCUMENT_UPLOAD_TTL_MS) {
await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch(() => undefined);
throw new Error('Temporary upload expired before finalize');
}
const isDocxUpload = input.upload.type === 'docx';
const finalizedType: DocumentType = isDocxUpload ? 'pdf' : input.upload.type;
const finalizedBody = isDocxUpload
? await convertDocxBufferToPdfBuffer(temp.body)
: temp.body;
const finalizedContentType = finalizedType === 'pdf'
? 'application/pdf'
: temp.contentType;
const finalizedName = finalizedType === 'pdf' && input.upload.type === 'docx'
? safeDocumentName(`${path.parse(input.upload.name).name}.pdf`, 'upload.pdf')
: input.upload.name;
const documentId = createHash('sha256').update(finalizedBody).digest('hex');
try {
await headDocumentBlob(documentId, input.namespace);
} catch (error) {
if (!isMissingBlobError(error)) throw error;
if (!isDocxUpload) {
try {
await copyTempDocumentBlobToDocument(
input.upload.token,
input.userId,
documentId,
input.namespace,
finalizedContentType,
{ ifNoneMatch: true },
);
} catch (copyError) {
if (!isPreconditionFailed(copyError)) throw copyError;
}
} else {
try {
await putDocumentBlob(
documentId,
finalizedBody,
finalizedContentType,
input.namespace,
{ ifNoneMatch: true },
);
} catch (putError) {
if (!isPreconditionFailed(putError)) throw putError;
}
}
}
const canonicalHead = await headDocumentBlob(documentId, input.namespace);
const stored = await registerUploadedDocument({
documentId,
userId: input.userId,
namespace: input.namespace,
name: finalizedName,
type: finalizedType,
size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength,
lastModified: input.upload.lastModified,
});
await putTempDocumentFinalizeReceipt(
input.upload.token,
input.userId,
input.namespace,
Buffer.from(JSON.stringify({ stored }), 'utf8'),
);
await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch((error) => {
serverLogger.warn({
event: 'documents.blob.upload.finalize.temp_delete_failed',
degraded: true,
fallbackPath: 'leave_temp_upload',
documentId,
token: input.upload.token,
error: errorToLog(error),
}, 'Failed to delete temp upload after finalize');
});
return stored;
}
export async function POST(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
const userId = ctxOrRes.userId;
const namespace = getOpenReaderTestNamespace(req.headers);
const uploads = parseFinalizePayload(await req.json().catch(() => null));
if (uploads.length === 0) {
return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 });
}
const stored = await Promise.all(
uploads.map((upload) => finalizeOne({
upload,
userId,
namespace,
})),
);
return NextResponse.json({ stored });
} catch (error) {
if (error instanceof Error && error.message === 'Temporary upload expired before finalize') {
return NextResponse.json({ error: error.message }, { status: 410 });
}
if (isMissingBlobError(error)) {
return NextResponse.json({ error: 'Temporary upload missing. Upload bytes again and retry finalize.' }, { status: 409 });
}
serverLogger.error({
event: 'documents.blob.upload.finalize.failed',
error: errorToLog(error),
}, 'Failed to finalize uploaded documents');
return errorResponse(error, {
apiErrorMessage: 'Failed to finalize uploaded documents',
normalize: { code: 'DOCUMENTS_BLOB_UPLOAD_FINALIZE_FAILED', errorClass: 'storage' },
});
}
}

View file

@ -1,6 +1,11 @@
import { randomUUID } from 'node:crypto';
import { NextRequest, NextResponse } from 'next/server';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore';
import {
TEMP_DOCUMENT_UPLOAD_TTL_MS,
deleteExpiredTempDocumentUploads,
presignTempPut,
} from '@/lib/server/documents/blobstore';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
@ -10,7 +15,6 @@ import { errorResponse } from '@/lib/server/errors/next-response';
export const dynamic = 'force-dynamic';
type PresignUpload = {
id: string;
contentType: string;
size: number;
};
@ -24,14 +28,12 @@ function parseUploads(body: unknown): PresignUpload[] {
for (const raw of rawUploads) {
if (!raw || typeof raw !== 'object') continue;
const rec = raw as Record<string, unknown>;
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
if (!isValidDocumentId(id)) continue;
const contentType =
typeof rec.contentType === 'string' && rec.contentType.trim()
? rec.contentType.trim()
: 'application/octet-stream';
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
uploads.push({ id, contentType, size });
uploads.push({ contentType, size });
}
return uploads;
}
@ -47,6 +49,8 @@ export async function POST(req: NextRequest) {
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
const userId = ctxOrRes.userId;
const body = await req.json().catch(() => null);
const uploads = parseUploads(body);
@ -68,13 +72,16 @@ export async function POST(req: NextRequest) {
}
const namespace = getOpenReaderTestNamespace(req.headers);
await deleteExpiredTempDocumentUploads(userId, namespace, Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS)
.catch(() => undefined);
const signed = await Promise.all(
uploads.map(async (upload) => {
const res = await presignPut(upload.id, upload.contentType, namespace, {
const token = randomUUID();
const res = await presignTempPut(token, userId, upload.contentType, namespace, {
contentLength: upload.size,
});
return {
id: upload.id,
token,
url: res.url,
headers: res.headers,
};

View file

@ -1,209 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { writeFile, mkdir, readFile, readdir, rm, stat } from 'fs/promises';
import { spawn } from 'child_process';
import path from 'path';
import { existsSync } from 'fs';
import { randomUUID, createHash } from 'crypto';
import { pathToFileURL } from 'url';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { safeDocumentName } from '@/lib/server/documents/utils';
import { enqueueDocumentPreview } from '@/lib/server/documents/previews';
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response';
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
async function ensureTempDir() {
if (!existsSync(DOCSTORE_DIR)) {
await mkdir(DOCSTORE_DIR, { recursive: true });
}
if (!existsSync(TEMP_DIR)) {
await mkdir(TEMP_DIR, { recursive: true });
}
}
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
return new Promise((resolve, reject) => {
const args: string[] = [];
if (profileDir) {
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
}
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
const proc = spawn('soffice', args);
proc.on('error', (error) => reject(error));
proc.on('close', (code) => {
if (code === 0) resolve();
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
});
});
}
async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100): Promise<string> {
const end = Date.now() + timeoutMs;
while (Date.now() < end) {
const files = await readdir(dir);
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
if (pdf) {
const pdfPath = path.join(dir, pdf);
try {
const first = await stat(pdfPath);
await new Promise((res) => setTimeout(res, intervalMs));
const second = await stat(pdfPath);
if (second.size > 0 && second.size === first.size) {
return pdfPath;
}
} catch {
// ignore transient errors
}
}
await new Promise((res) => setTimeout(res, intervalMs));
}
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
}
export async function POST(req: NextRequest) {
try {
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
const testNamespace = getOpenReaderTestNamespace(req.headers);
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
const storageUserId = ctxOrRes.userId;
await ensureTempDir();
const formData = await req.formData();
const file = formData.get('file');
if (!(file instanceof File)) {
return NextResponse.json({ error: 'No file provided' }, { status: 400 });
}
if (!file.name.toLowerCase().endsWith('.docx')) {
return NextResponse.json({ error: 'File must be a .docx document' }, { status: 400 });
}
const docxBytes = Buffer.from(await file.arrayBuffer());
// Keep stable IDs tied to source bytes.
const id = createHash('sha256').update(docxBytes).digest('hex');
const tempId = randomUUID();
const jobDir = path.join(TEMP_DIR, tempId);
await mkdir(jobDir, { recursive: true });
const profileDir = path.join(jobDir, 'lo-profile');
await mkdir(profileDir, { recursive: true });
const inputPath = path.join(jobDir, 'input.docx');
await writeFile(inputPath, docxBytes);
try {
await convertDocxToPdf(inputPath, jobDir, profileDir);
const pdfPath = await waitForPdfReady(jobDir);
const pdfContent = await readFile(pdfPath);
try {
await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace);
} catch (error) {
// Idempotent behavior: if blob already exists for this sha, continue.
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined;
const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed';
if (!isPreconditionFailed) {
throw error;
}
}
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
await db
.insert(documents)
.values({
id,
userId: storageUserId,
name: derivedName,
type: 'pdf',
size: pdfContent.length,
lastModified,
filePath: id,
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
parsedJsonKey: null,
})
.onConflictDoUpdate({
target: [documents.id, documents.userId],
set: {
name: derivedName,
type: 'pdf',
size: pdfContent.length,
lastModified,
filePath: id,
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
parsedJsonKey: null,
},
});
await enqueueDocumentPreview(
{
id,
type: 'pdf',
lastModified,
},
testNamespace,
).catch((error) => {
serverLogger.warn({
event: 'documents.docx.preview_enqueue.failed',
degraded: true,
fallbackPath: 'skip_preview_enqueue',
documentId: id,
error: errorToLog(error),
}, 'Failed to enqueue preview for converted DOCX');
});
// Record upload-driven parse load (see register route for rationale).
const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig);
enqueueParsePdfJob({
documentId: id,
userId: storageUserId,
namespace: testNamespace,
});
return NextResponse.json({
stored: {
id,
name: derivedName,
type: 'pdf',
size: pdfContent.length,
lastModified,
},
});
} finally {
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
}
} catch (error) {
serverLogger.error({
event: 'documents.docx.convert_upload.failed',
error: errorToLog(error),
}, 'Failed converting/uploading DOCX');
return errorResponse(error, {
apiErrorMessage: 'Failed to convert document',
normalize: { code: 'DOCUMENTS_DOCX_CONVERT_UPLOAD_FAILED', errorClass: 'upstream' },
});
}
}

View file

@ -1,25 +1,20 @@
import { randomUUID } from 'node:crypto';
import { NextRequest, NextResponse } from 'next/server';
import { and, count, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
import { toDocumentTypeFromName } from '@/lib/server/documents/utils';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response';
import {
cleanupDocumentPreviewArtifacts,
deleteDocumentPreviewRows,
enqueueDocumentPreview,
} from '@/lib/server/documents/previews';
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
import {
normalizeDocumentParseStateForCurrentParserVersion,
normalizeParseStatus,
parseDocumentParseState,
stringifyDocumentParseState,
} from '@/lib/server/documents/parse-state';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
@ -27,14 +22,6 @@ import type { BaseDocument, DocumentType } from '@/types/documents';
export const dynamic = 'force-dynamic';
type RegisterDocument = {
id: string;
name: string;
type: DocumentType;
size: number;
lastModified: number;
};
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
@ -49,175 +36,6 @@ function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType
return toDocumentTypeFromName(safeName);
}
function normalizeLastModified(value: unknown): number {
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
}
function parseDocumentPayload(body: unknown): RegisterDocument[] {
if (!body || typeof body !== 'object') return [];
const rawDocs = (body as { documents?: unknown }).documents;
if (!Array.isArray(rawDocs)) return [];
const docs: RegisterDocument[] = [];
for (const rawDoc of rawDocs) {
if (!rawDoc || typeof rawDoc !== 'object') continue;
const rec = rawDoc as Record<string, unknown>;
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
if (!isValidDocumentId(id)) continue;
const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`;
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
const type = normalizeDocumentType(rec.type, name);
const lastModified = normalizeLastModified(rec.lastModified);
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
docs.push({ id, name, type, size, lastModified });
}
return docs;
}
export async function POST(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
const testNamespace = getOpenReaderTestNamespace(req.headers);
const storageUserId = ctxOrRes.userId;
const body = await req.json().catch(() => null);
const documentsData = parseDocumentPayload(body);
if (documentsData.length === 0) {
return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 });
}
const stored: BaseDocument[] = [];
// Resolve the parse rate-limit config once (only when a PDF is present).
const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf')
? getPdfLayoutRateConfig(await getResolvedRuntimeConfig())
: null;
for (const doc of documentsData) {
let headSize = doc.size;
// Retry HEAD check to handle S3 read-after-write propagation delays.
// The client uploads bytes directly to S3 via presigned URL, then
// immediately calls this endpoint. On serverless platforms the HEAD
// request may reach S3 before the PUT is visible.
const HEAD_RETRIES = 3;
const HEAD_RETRY_DELAY_MS = 500;
let headError: unknown = null;
for (let attempt = 0; attempt < HEAD_RETRIES; attempt++) {
headError = null;
try {
const head = await headDocumentBlob(doc.id, testNamespace);
if (head.contentLength > 0) headSize = head.contentLength;
break;
} catch (error) {
if (isMissingBlobError(error)) {
headError = error;
if (attempt < HEAD_RETRIES - 1) {
await new Promise((r) => setTimeout(r, HEAD_RETRY_DELAY_MS));
continue;
}
} else {
throw error;
}
}
}
if (headError && isMissingBlobError(headError)) {
return NextResponse.json(
{
error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`,
},
{ status: 409 },
);
}
await db
.insert(documents)
.values({
id: doc.id,
userId: storageUserId,
name: doc.name,
type: doc.type,
size: headSize,
lastModified: doc.lastModified,
filePath: doc.id,
parseState: doc.type === 'pdf'
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
: null,
parsedJsonKey: null,
})
.onConflictDoUpdate({
target: [documents.id, documents.userId],
set: {
name: doc.name,
type: doc.type,
size: headSize,
lastModified: doc.lastModified,
filePath: doc.id,
parseState: doc.type === 'pdf'
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
: null,
parsedJsonKey: null,
},
});
stored.push({
id: doc.id,
name: doc.name,
type: doc.type,
size: headSize,
lastModified: doc.lastModified,
scope: 'user',
});
await enqueueDocumentPreview(
{
id: doc.id,
type: doc.type,
lastModified: doc.lastModified,
},
testNamespace,
).catch((error) => {
serverLogger.warn({
event: 'documents.preview.enqueue.failed',
degraded: true,
fallbackPath: 'skip_preview_enqueue',
documentId: doc.id,
error: errorToLog(error),
}, 'Failed to enqueue document preview');
});
if (doc.type === 'pdf') {
// Account for upload-driven parse load in the same ledger the explicit
// re-parse limiter reads. We record (not reject) here so a legitimate
// bulk upload always parses; the recorded load still throttles
// subsequent loopable re-parse spam via /parsed.
await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] });
enqueueParsePdfJob({
documentId: doc.id,
userId: storageUserId,
namespace: testNamespace,
});
}
}
return NextResponse.json({ success: true, stored });
} catch (error) {
serverLogger.error({
event: 'documents.register.failed',
error: errorToLog(error),
}, 'Failed to register documents');
return errorResponse(error, {
apiErrorMessage: 'Failed to register documents',
normalize: { code: 'DOCUMENTS_REGISTER_FAILED', errorClass: 'db' },
});
}
}
export async function GET(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
@ -260,13 +78,16 @@ export async function GET(req: NextRequest) {
const results: BaseDocument[] = rows.map((doc) => {
const type = normalizeDocumentType(doc.type, doc.name);
const parseState = type === 'pdf'
? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState))
: null;
return {
id: doc.id,
name: doc.name,
size: Number(doc.size),
lastModified: Number(doc.lastModified),
type,
parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null,
parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null,
parsedJsonKey: doc.parsedJsonKey,
scope: 'user',
};

View file

@ -122,16 +122,14 @@ interface DocumentListInnerProps {
function SidebarUploadLoader({
totalFiles,
completedFiles,
phase,
currentFileName,
}: {
totalFiles: number;
completedFiles: number;
phase: 'uploading' | 'converting';
phase: 'uploading';
currentFileName: string | null;
}) {
const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0;
const label = phase === 'converting' ? 'Converting' : 'Uploading';
const radius = 7;
const stroke = 2;
const size = 18;
@ -143,12 +141,12 @@ function SidebarUploadLoader({
<div className="rounded-md border border-line bg-surface-sunken px-2 py-1.5">
<div className="flex items-center justify-between gap-2">
<div className="min-w-0 flex items-center gap-1.5 text-[11px] leading-tight">
<span className="font-medium text-foreground">{label}</span>
<span className="font-medium text-foreground">Uploading</span>
<span className="shrink-0 tabular-nums text-soft">{completedFiles}/{totalFiles}</span>
</div>
<div className="shrink-0 flex items-center gap-1 text-accent" aria-label={`Upload progress ${progress}%`}>
<span className="text-[10px] tabular-nums text-soft">{progress}%</span>
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`} className={phase === 'converting' ? 'animate-spin' : ''}>
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`}>
<circle
cx={size / 2}
cy={size / 2}
@ -228,13 +226,11 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
const {
pdfDocs,
removePDFDocument,
isPDFLoading,
epubDocs,
removeEPUBDocument,
isEPUBLoading,
htmlDocs,
removeHTMLDocument,
deleteDocument,
isHTMLLoading,
} = useDocuments();
@ -436,9 +432,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
const handleDelete = useCallback(async () => {
if (!documentToDelete) return;
try {
if (documentToDelete.type === 'pdf') await removePDFDocument(documentToDelete.id);
else if (documentToDelete.type === 'epub') await removeEPUBDocument(documentToDelete.id);
else if (documentToDelete.type === 'html') await removeHTMLDocument(documentToDelete.id);
await deleteDocument(documentToDelete.id);
setFolders((prev) =>
prev.map((f) => ({
...f,
@ -451,7 +445,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
} catch (err) {
console.error('Failed to remove document:', err);
}
}, [documentToDelete, removePDFDocument, removeEPUBDocument, removeHTMLDocument]);
}, [deleteDocument, documentToDelete]);
const handleDeleteDoc = useCallback((doc: DocumentListDocument) => {
setDocumentToDelete({ id: doc.id, name: doc.name, type: doc.type });
@ -577,10 +571,8 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
if (batches.length === 0) return null;
const totalFiles = batches.reduce((sum, batch) => sum + batch.totalFiles, 0);
const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0);
const convertingBatch = batches.find((batch) => batch.phase === 'converting');
const phase: 'uploading' | 'converting' = convertingBatch ? 'converting' : 'uploading';
const currentFileName = convertingBatch?.currentFileName ?? batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
return { totalFiles, completedFiles, phase, currentFileName };
const currentFileName = batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
return { totalFiles, completedFiles, phase: 'uploading' as const, currentFileName };
}, [activeUploadBatches]);
const fallbackViewMode: ViewMode = viewMode;

View file

@ -4,7 +4,6 @@ import { useState, useCallback, useId, type ReactNode } from 'react';
import { useDropzone } from 'react-dropzone';
import { UploadIcon } from '@/components/icons/Icons';
import { useDocuments } from '@/contexts/DocumentContext';
import { uploadDocxAsPdf } from '@/lib/client/api/documents';
import { useFeatureFlag } from '@/contexts/RuntimeConfigContext';
import { dropzoneSurfaceClass } from '@/components/ui';
@ -20,7 +19,7 @@ export interface UploadBatchState {
isActive: boolean;
totalFiles: number;
completedFiles: number;
phase: 'uploading' | 'converting';
phase: 'uploading';
currentFileName: string | null;
}
@ -33,13 +32,9 @@ export function DocumentUploader({
const uploaderId = useId();
const enableDocx = useFeatureFlag('enableDocxConversion');
const {
addPDFDocument: addPDF,
addEPUBDocument: addEPUB,
addHTMLDocument: addHTML,
refreshDocuments,
uploadDocuments,
} = useDocuments();
const [isUploading, setIsUploading] = useState(false);
const [isConverting, setIsConverting] = useState(false);
const [error, setError] = useState<string | null>(null);
const emitBatchState = useCallback((state: Omit<UploadBatchState, 'uploaderId'>) => {
@ -63,72 +58,13 @@ export function DocumentUploader({
});
try {
for (const file of acceptedFiles) {
if (file.type === 'application/pdf') {
emitBatchState({
isActive: true,
totalFiles,
completedFiles,
phase: 'uploading',
currentFileName: file.name,
});
await addPDF(file);
completedFiles += 1;
} else if (file.type === 'application/epub+zip') {
emitBatchState({
isActive: true,
totalFiles,
completedFiles,
phase: 'uploading',
currentFileName: file.name,
});
await addEPUB(file);
completedFiles += 1;
} else if (file.type === 'text/plain' || file.type === 'text/markdown' || file.name.endsWith('.md')) {
emitBatchState({
isActive: true,
totalFiles,
completedFiles,
phase: 'uploading',
currentFileName: file.name,
});
await addHTML(file);
completedFiles += 1;
} else if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
// Preserve prior UX: show "Converting DOCX..." state rather than generic uploading.
setIsUploading(false);
setIsConverting(true);
emitBatchState({
isActive: true,
totalFiles,
completedFiles,
phase: 'converting',
currentFileName: file.name,
});
// Convert+upload directly on the server. Use sha(docx) as stable ID to avoid duplicates.
await uploadDocxAsPdf(file);
await refreshDocuments();
setIsConverting(false);
setIsUploading(true);
completedFiles += 1;
} else {
continue;
}
emitBatchState({
isActive: true,
totalFiles,
completedFiles,
phase: 'uploading',
currentFileName: null,
});
}
await uploadDocuments(acceptedFiles);
completedFiles = acceptedFiles.length;
} catch (err) {
setError('Failed to upload file. Please try again.');
console.error('Upload error:', err);
} finally {
setIsUploading(false);
setIsConverting(false);
emitBatchState({
isActive: false,
totalFiles,
@ -137,7 +73,7 @@ export function DocumentUploader({
currentFileName: null,
});
}
}, [addHTML, addPDF, addEPUB, refreshDocuments, enableDocx, emitBatchState]);
}, [uploadDocuments, emitBatchState]);
const { getRootProps, getInputProps, isDragActive } = useDropzone({
onDrop,
@ -151,12 +87,12 @@ export function DocumentUploader({
} : {})
},
multiple: true,
disabled: isUploading || isConverting,
disabled: isUploading,
noClick: variant === 'overlay',
noKeyboard: variant === 'overlay'
});
const isDisabled = isUploading || isConverting;
const isDisabled = isUploading;
if (variant === 'overlay') {
const rootProps = getRootProps();
@ -209,8 +145,6 @@ export function DocumentUploader({
<UploadIcon className="w-3.5 h-3.5 text-soft group-hover:text-accent shrink-0 transition-colors duration-base" />
{isUploading ? (
<p className="text-[12px] font-medium truncate flex-1">Uploading</p>
) : isConverting ? (
<p className="text-[12px] font-medium truncate flex-1">Converting DOCX</p>
) : (
<div className="flex items-center gap-2 min-w-0 flex-1">
<p className="text-[12px] truncate flex-1">
@ -225,8 +159,6 @@ export function DocumentUploader({
<UploadIcon className="w-7 h-7 sm:w-10 sm:h-10 mb-2 text-soft" />
{isUploading ? (
<p className="text-sm sm:text-lg font-semibold text-foreground">Uploading file...</p>
) : isConverting ? (
<p className="text-sm sm:text-lg font-semibold text-foreground">Converting DOCX to PDF...</p>
) : (
<>
<p className="mb-2 text-sm sm:text-lg font-semibold text-foreground">

View file

@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl';
import { Navigator } from '@/components/player/Navigator';
import { IconButton } from '@/components/ui';
export default function TTSPlayer({ currentPage, numPages }: {
export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: {
currentPage?: number;
numPages?: number | undefined;
isPlaybackReady?: boolean;
}) {
const {
isPlaying,
@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: {
<IconButton
onClick={skipBackward}
aria-label="Skip backward"
disabled={isProcessing}
disabled={isProcessing || !isPlaybackReady}
className="relative"
>
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: {
<IconButton
onClick={togglePlay}
aria-label={isPlaying ? 'Pause' : 'Play'}
disabled={isProcessing && !isPlaying}
disabled={!isPlaying && (!isPlaybackReady || isProcessing)}
className="relative"
>
{isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />}
{!isPlaying && !isPlaybackReady
? <LoadingSpinner />
: (isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />)}
</IconButton>
<IconButton
onClick={skipForward}
aria-label="Skip forward"
disabled={isProcessing}
disabled={isProcessing || !isPlaybackReady}
className="relative"
>
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}

View file

@ -3,32 +3,53 @@
import { createContext, useCallback, useContext, useEffect, useMemo, ReactNode } from 'react';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import type { BaseDocument } from '@/types/documents';
import { listDocuments, uploadDocuments, deleteDocuments } from '@/lib/client/api/documents';
import { putCachedEpub, putCachedHtml, putCachedPdf, evictCachedEpub, evictCachedHtml, evictCachedPdf } from '@/lib/client/cache/documents';
import {
deleteDocuments as deleteServerDocuments,
listDocuments,
uploadDocuments as uploadServerDocuments,
} from '@/lib/client/api/documents';
import { cacheStoredDocumentFromBytes, evictCachedDocument } from '@/lib/client/cache/documents';
import { useAuthSession } from '@/hooks/useAuthSession';
interface DocumentContextType {
pdfDocs: Array<BaseDocument & { type: 'pdf' }>;
addPDFDocument: (file: File) => Promise<string>;
removePDFDocument: (id: string) => Promise<void>;
isPDFLoading: boolean;
epubDocs: Array<BaseDocument & { type: 'epub' }>;
addEPUBDocument: (file: File) => Promise<string>;
removeEPUBDocument: (id: string) => Promise<void>;
isEPUBLoading: boolean;
htmlDocs: Array<BaseDocument & { type: 'html' }>;
addHTMLDocument: (file: File) => Promise<string>;
removeHTMLDocument: (id: string) => Promise<void>;
isHTMLLoading: boolean;
uploadDocuments: (files: File[]) => Promise<BaseDocument[]>;
deleteDocument: (id: string) => Promise<void>;
refreshDocuments: () => Promise<void>;
}
const DocumentContext = createContext<DocumentContextType | undefined>(undefined);
const DOCUMENTS_QUERY_KEY = 'documents';
type SupportedDocument = BaseDocument & { type: 'pdf' | 'epub' | 'html' };
function mergeStoredDocuments(
previous: SupportedDocument[] | undefined,
uploaded: BaseDocument[],
): SupportedDocument[] {
const next = [...(previous ?? [])];
for (let index = uploaded.length - 1; index >= 0; index -= 1) {
const stored = uploaded[index];
if (stored.type !== 'pdf' && stored.type !== 'epub' && stored.type !== 'html') {
continue;
}
const supportedStored = stored as SupportedDocument;
const withoutExisting = next.filter((document) => document.id !== supportedStored.id);
next.splice(0, next.length, supportedStored, ...withoutExisting);
}
return next;
}
export function DocumentProvider({ children }: { children: ReactNode }) {
const queryClient = useQueryClient();
const { data: sessionData, isPending: isSessionPending } = useAuthSession();
@ -81,67 +102,63 @@ export function DocumentProvider({ children }: { children: ReactNode }) {
return { pdfDocs, epubDocs, htmlDocs };
}, [docs]);
const cacheUploaded = useCallback(async (stored: BaseDocument, file: File) => {
try {
if (stored.type === 'pdf') {
await putCachedPdf(stored, await file.arrayBuffer());
} else if (stored.type === 'epub') {
await putCachedEpub(stored, await file.arrayBuffer());
} else if (stored.type === 'html') {
const buf = await file.arrayBuffer();
const decoded = new TextDecoder().decode(new Uint8Array(buf));
await putCachedHtml(stored, decoded);
}
} catch (err) {
// Cache failures should not block uploads.
console.warn('Failed to cache uploaded document:', stored.id, err);
}
}, []);
const uploadDocuments = useCallback(async (files: File[]): Promise<BaseDocument[]> => {
if (files.length === 0) return [];
const addDocument = useCallback(async (file: File): Promise<string> => {
const [stored] = await uploadDocuments([file]);
if (!stored) throw new Error('Upload succeeded but returned no document');
await cacheUploaded(stored, file);
const isSupported = stored.type === 'pdf' || stored.type === 'epub' || stored.type === 'html';
if (!isSupported) return stored.id;
const supportedStored = stored as BaseDocument & { type: 'pdf' | 'epub' | 'html' };
queryClient.setQueryData<Array<BaseDocument & { type: 'pdf' | 'epub' | 'html' }>>(documentsQueryKey, (prev = []) => {
const next = prev.filter((d) => d.id !== supportedStored.id);
return [supportedStored, ...next];
});
return stored.id;
}, [cacheUploaded, queryClient, documentsQueryKey]);
const addPDFDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
const addEPUBDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
const addHTMLDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
const removeById = useCallback(async (id: string) => {
await deleteDocuments({ ids: [id] });
await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]);
queryClient.setQueryData<Array<BaseDocument & { type: 'pdf' | 'epub' | 'html' }>>(documentsQueryKey, (prev = []) =>
prev.filter((d) => d.id !== id),
const stored = await uploadServerDocuments(files);
await Promise.allSettled(
stored.map(async (document, index) => {
const file = files[index];
if (!file) return;
const sourceType = file.name
? (
file.name.toLowerCase().endsWith('.pdf')
? 'pdf'
: file.name.toLowerCase().endsWith('.epub')
? 'epub'
: file.name.toLowerCase().endsWith('.docx')
? 'docx'
: 'html'
)
: (
file.type === 'application/pdf'
? 'pdf'
: file.type === 'application/epub+zip'
? 'epub'
: file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
? 'docx'
: 'html'
);
if (document.type !== sourceType) return;
await cacheStoredDocumentFromBytes(document, await file.arrayBuffer());
}),
);
}, [queryClient, documentsQueryKey]);
const removePDFDocument = useCallback(async (id: string) => removeById(id), [removeById]);
const removeEPUBDocument = useCallback(async (id: string) => removeById(id), [removeById]);
const removeHTMLDocument = useCallback(async (id: string) => removeById(id), [removeById]);
queryClient.setQueryData<SupportedDocument[]>(documentsQueryKey, (previous) =>
mergeStoredDocuments(previous, stored),
);
return stored;
}, [documentsQueryKey, queryClient]);
const deleteDocument = useCallback(async (id: string) => {
await deleteServerDocuments({ ids: [id] });
await evictCachedDocument(id);
queryClient.setQueryData<SupportedDocument[]>(documentsQueryKey, (previous = []) =>
previous.filter((document) => document.id !== id),
);
}, [documentsQueryKey, queryClient]);
return (
<DocumentContext.Provider value={{
pdfDocs: docsByType.pdfDocs,
addPDFDocument,
removePDFDocument,
isPDFLoading: isLoading,
epubDocs: docsByType.epubDocs,
addEPUBDocument,
removeEPUBDocument,
isEPUBLoading: isLoading,
htmlDocs: docsByType.htmlDocs,
addHTMLDocument,
removeHTMLDocument,
isHTMLLoading: isLoading,
uploadDocuments,
deleteDocument,
refreshDocuments,
}}>

View file

@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
const isPlayingRef = useRef(false);
const pauseEpochRef = useRef(0);
const pendingPlaybackStartRef = useRef(false);
const sentencesRef = useRef<string[]>([]);
const currentIndexRef = useRef(0);
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const recordManualPause = useCallback(() => {
// Cancel any queued auto-resume intent and mark an explicit user pause.
resumeAfterLocationChangeRef.current = false;
pendingPlaybackStartRef.current = false;
pauseEpochRef.current += 1;
}, []);
@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
* Used for external control of playback state
*/
const pause = useCallback(() => {
pendingPlaybackStartRef.current = false;
recordManualPause();
clearPendingEpubJump();
abortPendingTtsRequests(true);
@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const shouldPause = normalizedOptions.shouldPause ?? false;
const pauseEpochAtStart = pauseEpochRef.current;
const pendingAutoResume = resumeAfterLocationChangeRef.current;
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume);
if (shouldPause || pendingAutoResume) {
resumeAfterLocationChangeRef.current = false;
}
const pendingPlaybackStart = pendingPlaybackStartRef.current;
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart);
// Keep track of previous state and pause playback
invalidatePlaybackRun();
@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
try {
if (newSentences.length === 0) {
console.warn('No sentences found in text');
if (shouldPause || pendingAutoResume) {
resumeAfterLocationChangeRef.current = false;
}
if (pendingPlaybackStart) {
pendingPlaybackStartRef.current = false;
}
setIsProcessing(false);
return;
}
if (shouldPause || pendingAutoResume) {
resumeAfterLocationChangeRef.current = false;
}
if (pendingPlaybackStart) {
pendingPlaybackStartRef.current = false;
}
if (!isEPUB && typeof resolvedLocation === 'number') {
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
if (firstFingerprint) {
@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
// Ensure audio is unlocked while we're still in the click/tap handler.
unlockPlaybackOnUserGesture();
const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]);
if (!activeHowl && !hasPreparedSentence) {
pendingPlaybackStartRef.current = true;
setIsProcessing(true);
return;
}
// Resume current sentence if we already have a paused Howl.
if (activeHowl) {
applyPlaybackRateToHowl(activeHowl);
@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
setIsPlaying(true);
}, [
activeHowl,
currentIndex,
applyPlaybackRateToHowl,
isPlaying,
pauseActiveHowl,
playbackSegments,
recordManualPause,
clearPendingEpubJump,
abortPendingTtsRequests,
sentences,
unlockPlaybackOnUserGesture,
]);
@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
abortAudio();
clearWarmAudioCache();
playbackInFlightRef.current = false;
pendingPlaybackStartRef.current = false;
pendingJumpTargetRef.current = null;
clearPendingEpubJump();
bumpEpubPreloadGeneration();

View file

@ -1,10 +1,8 @@
import { sha256HexFromArrayBuffer } from '@/lib/client/sha256';
import type { BaseDocument, DocumentType } from '@/types/documents';
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
import type { DocumentSettings } from '@/types/document-settings';
export type UploadSource = {
id: string;
name: string;
type: DocumentType;
size: number;
@ -23,8 +21,12 @@ function toUploadBody(body: UploadSource['body']): BodyInit {
return body as unknown as BodyInit;
}
async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise<void> {
const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, {
async function uploadDocumentSourceViaProxy(
source: UploadSource,
token: string,
options?: UploadOptions,
): Promise<void> {
const res = await fetch(`/api/documents/blob/upload/fallback?token=${encodeURIComponent(token)}`, {
method: 'PUT',
headers: { 'Content-Type': source.contentType || 'application/octet-stream' },
body: toUploadBody(source.body),
@ -45,6 +47,14 @@ function documentTypeForName(name: string): DocumentType {
return 'html';
}
function documentTypeForMime(contentType: string): DocumentType | null {
const normalized = contentType.trim().toLowerCase();
if (normalized === 'application/pdf') return 'pdf';
if (normalized === 'application/epub+zip') return 'epub';
if (normalized === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') return 'docx';
return null;
}
export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string {
if (doc.type === 'pdf') return 'application/pdf';
if (doc.type === 'epub') return 'application/epub+zip';
@ -80,31 +90,20 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
export async function getParsedPdfDocument(
id: string,
options?: { signal?: AbortSignal; retryFailed?: boolean; opId?: string },
): Promise<
| { status: 'ready'; parsed: ParsedPdfDocument }
| { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null; opId?: string | null }
> {
const params = new URLSearchParams();
if (options?.retryFailed) params.set('retry', '1');
if (options?.opId) params.set('opId', options.opId);
const query = params.size > 0 ? `?${params.toString()}` : '';
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, {
options?: { signal?: AbortSignal },
): Promise<ParsedPdfDocument> {
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed`, {
signal: options?.signal,
cache: 'no-store',
});
if (res.status === 202) {
if (res.status === 409) {
const data = (await res.json().catch(() => null)) as {
parseStatus?: string;
parseProgress?: PdfParseProgress | null;
opId?: string | null;
} | null;
const parseStatus = data?.parseStatus;
if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') {
return { status: parseStatus, parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null };
}
return { status: 'pending', parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null };
throw new Error(data?.parseStatus ? `Parsed PDF is not ready (${data.parseStatus})` : 'Parsed PDF is not ready');
}
if (!res.ok) {
@ -112,8 +111,7 @@ export async function getParsedPdfDocument(
throw new Error(data?.error || 'Failed to load parsed PDF');
}
const parsed = (await res.json()) as ParsedPdfDocument;
return { status: 'ready', parsed };
return (await res.json()) as ParsedPdfDocument;
}
export function subscribeParsedPdfDocumentEvents(
@ -241,7 +239,6 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
uploads: sources.map((source) => ({
id: source.id,
contentType: source.contentType,
size: source.size,
})),
@ -255,14 +252,18 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
}
const presigned = (await presignRes.json()) as {
uploads?: Array<{ id: string; url: string; headers?: Record<string, string> }>;
uploads?: Array<{ token: string; url: string; headers?: Record<string, string> }>;
};
const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload]));
const uploads = presigned.uploads || [];
if (uploads.length !== sources.length) {
throw new Error('Upload preparation returned an unexpected number of temp uploads');
}
for (const source of sources) {
const upload = byId.get(source.id);
if (!upload?.url) {
throw new Error(`Missing presigned upload for document ${source.id}`);
for (let index = 0; index < sources.length; index += 1) {
const source = sources[index];
const upload = uploads[index];
if (!upload?.url || !upload.token) {
throw new Error(`Missing presigned upload for document ${source.name}`);
}
let putError: unknown = null;
@ -285,7 +286,7 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
}
try {
await uploadDocumentSourceViaProxy(source, options);
await uploadDocumentSourceViaProxy(source, upload.token, options);
} catch (proxyError) {
const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error';
const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error';
@ -293,27 +294,26 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
}
}
const registerRes = await fetch('/api/documents', {
const finalizeRes = await fetch('/api/documents/blob/upload/finalize', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
documents: sources.map((source) => ({
id: source.id,
uploads: sources.map((source, index) => ({
token: uploads[index]?.token,
name: source.name,
type: source.type,
size: source.size,
lastModified: source.lastModified,
})),
}),
signal: options?.signal,
});
if (!registerRes.ok) {
const data = (await registerRes.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to register uploaded documents');
if (!finalizeRes.ok) {
const data = (await finalizeRes.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to finalize uploaded documents');
}
const data = (await registerRes.json()) as { stored: BaseDocument[] };
const data = (await finalizeRes.json()) as { stored: BaseDocument[] };
return data.stored || [];
}
@ -322,14 +322,14 @@ export async function uploadDocuments(files: File[], options?: UploadOptions): P
const sources: UploadSource[] = [];
for (const file of files) {
const bytes = await file.arrayBuffer();
const id = await sha256HexFromArrayBuffer(bytes);
const type = documentTypeForName(file.name);
const name = file.name || `${id}.${type}`;
const contentType = file.type || mimeTypeForDoc({ name, type });
const name = file.name || '';
const type = name
? documentTypeForName(name)
: (documentTypeForMime(file.type) ?? 'html');
const resolvedName = name || `upload.${type}`;
const contentType = file.type || mimeTypeForDoc({ name: resolvedName, type });
sources.push({
id,
name,
name: resolvedName,
type,
size: file.size,
lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(),
@ -504,23 +504,3 @@ export async function getDocumentPreviewStatus(
throw new Error(`Failed to load preview status (status ${res.status})`);
}
export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise<BaseDocument> {
const form = new FormData();
form.append('file', file);
const res = await fetch('/api/documents/docx-to-pdf/upload', {
method: 'POST',
body: form,
signal: options?.signal,
});
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to convert DOCX');
}
const data = (await res.json()) as { stored: BaseDocument };
if (!data?.stored) throw new Error('DOCX conversion succeeded but returned no document');
return data.stored;
}

View file

@ -107,6 +107,10 @@ export async function evictCachedHtml(id: string): Promise<void> {
await removeHtmlDocument(id);
}
export async function evictCachedDocument(id: string): Promise<void> {
await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]);
}
export async function clearDocumentCache(): Promise<void> {
const { clearPdfDocuments, clearEpubDocuments, clearHtmlDocuments } = await import('@/lib/client/dexie');
await Promise.all([clearPdfDocuments(), clearEpubDocuments(), clearHtmlDocuments()]);

View file

@ -971,11 +971,9 @@ export async function syncDocumentsToServer(
for (const doc of pdfDocs) {
const bytes = new Uint8Array(doc.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: doc.id,
source: {
id,
name: doc.name,
type: 'pdf',
size: bytes.byteLength,
@ -992,11 +990,9 @@ export async function syncDocumentsToServer(
for (const doc of epubDocs) {
const bytes = new Uint8Array(doc.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: doc.id,
source: {
id,
name: doc.name,
type: 'epub',
size: bytes.byteLength,
@ -1013,11 +1009,9 @@ export async function syncDocumentsToServer(
for (const doc of htmlDocs) {
const encoded = textEncoder.encode(doc.data);
const id = await sha256HexFromBytes(encoded);
uploads.push({
oldId: doc.id,
source: {
id,
name: doc.name,
type: 'html',
size: encoded.byteLength,
@ -1036,11 +1030,13 @@ export async function syncDocumentsToServer(
onProgress(50, 'Uploading to server...');
}
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
for (const entry of uploads) {
if (entry.oldId === entry.source.id) continue;
await applyDocumentIdMapping(entry.oldId, entry.source.id);
for (let index = 0; index < uploads.length; index += 1) {
const entry = uploads[index];
const nextId = stored[index]?.id;
if (!nextId || entry.oldId === nextId) continue;
await applyDocumentIdMapping(entry.oldId, nextId);
}
if (onProgress) {
@ -1064,11 +1060,9 @@ export async function syncSelectedDocumentsToServer(
const data = await getPdfDocument(doc.id);
if (data) {
const bytes = new Uint8Array(data.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: data.id,
source: {
id,
name: data.name,
type: 'pdf',
size: bytes.byteLength,
@ -1082,11 +1076,9 @@ export async function syncSelectedDocumentsToServer(
const data = await getEpubDocument(doc.id);
if (data) {
const bytes = new Uint8Array(data.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: data.id,
source: {
id,
name: data.name,
type: 'epub',
size: bytes.byteLength,
@ -1100,11 +1092,9 @@ export async function syncSelectedDocumentsToServer(
const data = await getHtmlDocument(doc.id);
if (data) {
const bytes = textEncoder.encode(data.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: data.id,
source: {
id,
name: data.name,
type: 'html',
size: bytes.byteLength,
@ -1121,11 +1111,13 @@ export async function syncSelectedDocumentsToServer(
}
if (onProgress) onProgress(50, 'Uploading to server...');
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
for (const entry of uploads) {
if (entry.oldId === entry.source.id) continue;
await applyDocumentIdMapping(entry.oldId, entry.source.id);
for (let index = 0; index < uploads.length; index += 1) {
const entry = uploads[index];
const nextId = stored[index]?.id;
if (!nextId || entry.oldId === nextId) continue;
await applyDocumentIdMapping(entry.oldId, nextId);
}
if (onProgress) {

View file

@ -1,3 +1,4 @@
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState(
? parseProgress
: null,
updatedAt: nowMs,
parserVersion: PDF_PARSER_VERSION,
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),

View file

@ -1,7 +1,7 @@
import { createHash, randomUUID } from 'node:crypto';
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core';
import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
import type {
@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string {
return [
'pdf_layout',
'v1',
PDF_PARSER_VERSION,
input.documentId,
input.namespace ?? '',
input.documentObjectKey ?? '',

View file

@ -1,4 +1,5 @@
import {
CopyObjectCommand,
DeleteObjectCommand,
DeleteObjectsCommand,
GetObjectCommand,
@ -11,6 +12,9 @@ import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
const TEMP_UPLOAD_TOKEN_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const TEMP_UPLOAD_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/;
export const TEMP_DOCUMENT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;
function sanitizeNamespace(namespace: string | null): string | null {
if (!namespace) return null;
@ -69,6 +73,23 @@ export function isValidDocumentId(id: string): boolean {
return DOCUMENT_ID_REGEX.test(id);
}
export function isPreconditionFailed(error: unknown): boolean {
if (!error || typeof error !== 'object') return false;
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
}
export function isValidTempUploadToken(token: string): boolean {
return TEMP_UPLOAD_TOKEN_REGEX.test(token);
}
function sanitizeTempUploadUserId(userId: string): string {
if (!TEMP_UPLOAD_USER_ID_REGEX.test(userId)) {
throw new Error(`Invalid temp upload user id: ${userId}`);
}
return encodeURIComponent(userId);
}
export function documentKey(id: string, namespace: string | null): string {
if (!isValidDocumentId(id)) {
throw new Error(`Invalid document id: ${id}`);
@ -90,6 +111,27 @@ export function documentParsedKey(id: string, namespace: string | null): string
return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`;
}
export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string {
const cfg = getS3Config();
const ns = sanitizeNamespace(namespace);
const nsSegment = ns ? `ns/${ns}/` : '';
return `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}users/${sanitizeTempUploadUserId(userId)}/`;
}
export function tempDocumentUploadKey(token: string, userId: string, namespace: string | null): string {
if (!isValidTempUploadToken(token)) {
throw new Error(`Invalid temp upload token: ${token}`);
}
return `${tempDocumentUploadPrefix(userId, namespace)}${token}.bin`;
}
export function tempDocumentUploadReceiptKey(token: string, userId: string, namespace: string | null): string {
if (!isValidTempUploadToken(token)) {
throw new Error(`Invalid temp upload token: ${token}`);
}
return `${tempDocumentUploadPrefix(userId, namespace)}${token}.receipt.json`;
}
function legacyDocumentParsedKey(id: string, namespace: string | null): string {
if (!isValidDocumentId(id)) {
throw new Error(`Invalid document id: ${id}`);
@ -138,6 +180,40 @@ export async function presignPut(
};
}
export async function presignTempPut(
token: string,
userId: string,
contentType: string,
namespace: string | null,
options?: { contentLength?: number },
): Promise<{ url: string; headers: Record<string, string> }> {
const cfg = getS3Config();
const client = getS3Client();
const key = tempDocumentUploadKey(token, userId, namespace);
const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream';
const contentLength =
typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0
? Math.floor(options.contentLength)
: undefined;
const command = new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
ContentType: normalizedType,
ServerSideEncryption: 'AES256',
...(contentLength !== undefined ? { ContentLength: contentLength } : {}),
});
const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 });
return {
url,
headers: {
'Content-Type': normalizedType,
'x-amz-server-side-encryption': 'AES256',
},
};
}
export async function headDocumentBlob(
id: string,
namespace: string | null,
@ -153,6 +229,23 @@ export async function headDocumentBlob(
};
}
export async function headTempDocumentBlob(
token: string,
userId: string,
namespace: string | null,
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null; lastModified: number | null }> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = tempDocumentUploadKey(token, userId, namespace);
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
return {
contentLength: Number(res.ContentLength ?? 0),
contentType: res.ContentType ?? null,
eTag: res.ETag ?? null,
lastModified: res.LastModified?.getTime() ?? null,
};
}
export async function getDocumentRange(
id: string,
start: number,
@ -185,6 +278,23 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro
return bodyToBuffer(res.Body);
}
export async function getTempDocumentBlob(
token: string,
userId: string,
namespace: string | null,
): Promise<Buffer> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = tempDocumentUploadKey(token, userId, namespace);
const res = await client.send(
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
}),
);
return bodyToBuffer(res.Body);
}
export async function getDocumentBlobStream(id: string, namespace: string | null): Promise<DocumentBlobBody> {
const cfg = getS3Config();
const client = getS3ProxyClient();
@ -198,6 +308,29 @@ export async function getDocumentBlobStream(id: string, namespace: string | null
return res.Body as DocumentBlobBody;
}
export async function getTempDocumentFinalizeReceipt<T>(
token: string,
userId: string,
namespace: string | null,
): Promise<T | null> {
try {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = tempDocumentUploadReceiptKey(token, userId, namespace);
const res = await client.send(
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
}),
);
const body = await bodyToBuffer(res.Body);
return JSON.parse(body.toString('utf8')) as T;
} catch (error) {
if (isMissingBlobError(error)) return null;
throw error;
}
}
export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
const cfg = getS3Config();
const client = getS3ProxyClient();
@ -241,6 +374,26 @@ export async function putParsedDocumentBlob(id: string, body: Buffer, namespace:
return key;
}
export async function putTempDocumentFinalizeReceipt(
token: string,
userId: string,
namespace: string | null,
body: Buffer,
): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = tempDocumentUploadReceiptKey(token, userId, namespace);
await client.send(
new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
Body: body,
ContentType: 'application/json',
ServerSideEncryption: 'AES256',
}),
);
}
export async function presignGet(
id: string,
namespace: string | null,
@ -264,6 +417,7 @@ export async function putDocumentBlob(
body: Buffer,
contentType: string,
namespace: string | null,
options?: { ifNoneMatch?: boolean },
): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
@ -275,6 +429,51 @@ export async function putDocumentBlob(
Body: body,
ContentType: contentType,
ServerSideEncryption: 'AES256',
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
}),
);
}
export async function putTempDocumentBlob(
token: string,
userId: string,
body: Buffer,
contentType: string,
namespace: string | null,
): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = tempDocumentUploadKey(token, userId, namespace);
await client.send(
new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
Body: body,
ContentType: contentType,
ServerSideEncryption: 'AES256',
}),
);
}
export async function copyTempDocumentBlobToDocument(
token: string,
userId: string,
documentId: string,
namespace: string | null,
contentType: string,
options?: { ifNoneMatch?: boolean },
): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
await client.send(
new CopyObjectCommand({
Bucket: cfg.bucket,
Key: documentKey(documentId, namespace),
CopySource: `${cfg.bucket}/${tempDocumentUploadKey(token, userId, namespace)}`,
ContentType: contentType,
MetadataDirective: 'REPLACE',
ServerSideEncryption: 'AES256',
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
}),
);
}
@ -292,6 +491,18 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
}
export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadKey(token, userId, namespace) }));
}
export async function deleteTempDocumentFinalizeReceipt(token: string, userId: string, namespace: string | null): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadReceiptKey(token, userId, namespace) }));
}
export function isMissingBlobError(error: unknown): boolean {
if (!error || typeof error !== 'object') return false;
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
@ -339,3 +550,53 @@ export async function deleteDocumentPrefix(prefix: string): Promise<number> {
return deleted;
}
export async function deleteExpiredTempDocumentUploads(
userId: string,
namespace: string | null,
olderThanMs: number,
): Promise<number> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const prefix = tempDocumentUploadPrefix(userId, namespace);
let continuationToken: string | undefined;
const keys: string[] = [];
do {
const listRes = await client.send(
new ListObjectsV2Command({
Bucket: cfg.bucket,
Prefix: prefix,
ContinuationToken: continuationToken,
}),
);
for (const item of listRes.Contents ?? []) {
const key = item.Key;
const lastModified = item.LastModified?.getTime() ?? 0;
if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue;
keys.push(key);
}
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
} while (continuationToken);
if (keys.length === 0) return 0;
let deleted = 0;
for (let i = 0; i < keys.length; i += 1000) {
const batch = keys.slice(i, i + 1000);
const deleteRes = await client.send(
new DeleteObjectsCommand({
Bucket: cfg.bucket,
Delete: {
Objects: batch.map((Key) => ({ Key })),
Quiet: true,
},
}),
);
deleted += deleteRes.Deleted?.length ?? 0;
}
return deleted;
}

View file

@ -0,0 +1,94 @@
import { spawn } from 'child_process';
import { existsSync } from 'fs';
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises';
import path from 'path';
import { randomUUID } from 'node:crypto';
import { pathToFileURL } from 'url';
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
const DEFAULT_DOCX_CONVERSION_TIMEOUT_MS = 60_000;
async function ensureTempDir(): Promise<void> {
if (!existsSync(DOCSTORE_DIR)) {
await mkdir(DOCSTORE_DIR, { recursive: true });
}
if (!existsSync(TEMP_DIR)) {
await mkdir(TEMP_DIR, { recursive: true });
}
}
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
return new Promise((resolve, reject) => {
const args: string[] = [];
if (profileDir) {
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
}
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
const proc = spawn('soffice', args);
let settled = false;
const timeout = setTimeout(() => {
if (settled) return;
settled = true;
proc.kill();
reject(new Error(`LibreOffice conversion timed out after ${DEFAULT_DOCX_CONVERSION_TIMEOUT_MS}ms`));
}, DEFAULT_DOCX_CONVERSION_TIMEOUT_MS);
proc.on('error', (error) => {
clearTimeout(timeout);
if (settled) return;
settled = true;
reject(error);
});
proc.on('close', (code) => {
clearTimeout(timeout);
if (settled) return;
settled = true;
if (code === 0) resolve();
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
});
});
}
async function waitForPdfReady(dir: string, timeoutMs = 20_000, intervalMs = 100): Promise<string> {
const end = Date.now() + timeoutMs;
while (Date.now() < end) {
const files = await readdir(dir);
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
if (pdf) {
const pdfPath = path.join(dir, pdf);
try {
const first = await stat(pdfPath);
await new Promise((resolve) => setTimeout(resolve, intervalMs));
const second = await stat(pdfPath);
if (second.size > 0 && second.size === first.size) {
return pdfPath;
}
} catch {
// Ignore transient filesystem races while LibreOffice is still writing.
}
}
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
}
export async function convertDocxBufferToPdfBuffer(docxBytes: Buffer): Promise<Buffer> {
await ensureTempDir();
const tempId = randomUUID();
const jobDir = path.join(TEMP_DIR, tempId);
await mkdir(jobDir, { recursive: true });
const profileDir = path.join(jobDir, 'lo-profile');
await mkdir(profileDir, { recursive: true });
const inputPath = path.join(jobDir, 'input.docx');
try {
await writeFile(inputPath, docxBytes);
await convertDocxToPdf(inputPath, jobDir, profileDir);
const pdfPath = await waitForPdfReady(jobDir);
return await readFile(pdfPath);
} finally {
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
}
}

View file

@ -1,7 +1,5 @@
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
import { documentKey } from '@/lib/server/documents/blobstore';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter';
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
import { normalizeParseStatus, type DocumentParseState } from '@/lib/server/documents/parse-state';
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
@ -20,13 +18,20 @@ export async function backfillPendingPdfParseOperation(input: {
if (parseStatus === 'ready' || parseStatus === 'failed') return null;
if (normalizeOpId(input.state.opId)) return null;
const created = await createOrReusePdfWorkerOperation({
const startedParse = await startPdfParseOperation({
documentId: input.documentId,
userId: input.userId,
namespace: input.namespace,
documentObjectKey: documentKey(input.documentId, input.namespace),
});
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
await recordJobEvent(input.userId, 'pdf_layout', created.opId, rateConfig);
return created;
if (startedParse.parseState.status === 'pending' || startedParse.parseState.status === 'running') {
enqueueParsePdfJob({
documentId: input.documentId,
userId: input.userId,
namespace: input.namespace,
initialOpId: startedParse.workerState.opId,
initialJobId: startedParse.workerState.jobId,
initialStatus: startedParse.parseState.status,
});
}
return startedParse.workerState;
}

View file

@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: {
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
...(input.state.opId ? { opId: input.state.opId } : {}),
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}),
}));
await db

View file

@ -1,4 +1,5 @@
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
export interface DocumentParseState {
status: PdfParseStatus;
@ -7,6 +8,7 @@ export interface DocumentParseState {
error?: string | null;
opId?: string;
jobId?: string;
parserVersion?: string;
}
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars
return 'pending';
}
export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean {
return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION;
}
export function normalizeDocumentParseStateForCurrentParserVersion(
state: DocumentParseState,
nowMs = Date.now(),
): DocumentParseState {
const status = normalizeParseStatus(state.status);
if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) {
return state;
}
return {
status: 'pending',
progress: null,
updatedAt: nowMs,
};
}
export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string {
const parserVersion = parsed?.parserVersion?.trim();
return parserVersion || PDF_PARSER_VERSION;
}
function normalizeProgress(progress: unknown): PdfParseProgress | null {
if (!progress || typeof progress !== 'object') return null;
const rec = progress as Record<string, unknown>;
@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
const error = typeof parsed.error === 'string' ? parsed.error : null;
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined;
return {
status,
progress,
@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
...(error ? { error } : {}),
...(opId ? { opId } : {}),
...(jobId ? { jobId } : {}),
...(parserVersion ? { parserVersion } : {}),
};
} catch {
return { status: 'pending', progress: null };
@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string {
...(state.error ? { error: state.error } : {}),
...(state.opId ? { opId: state.opId } : {}),
...(state.jobId ? { jobId: state.jobId } : {}),
...(state.parserVersion ? { parserVersion: state.parserVersion } : {}),
});
}

View file

@ -0,0 +1,33 @@
import { eq } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import {
normalizeDocumentParseStateForCurrentParserVersion,
parseDocumentParseState,
} from '@/lib/server/documents/parse-state';
type ReadyParsedPdfResult = {
parsedJsonKey: string;
};
export async function findReusableParsedPdfResult(
documentId: string,
): Promise<ReadyParsedPdfResult | null> {
const rows = await db
.select({
parseState: documents.parseState,
parsedJsonKey: documents.parsedJsonKey,
})
.from(documents)
.where(eq(documents.id, documentId));
for (const row of rows) {
const parsedJsonKey = row.parsedJsonKey?.trim();
if (!parsedJsonKey) continue;
const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
if (parseState.status !== 'ready') continue;
return { parsedJsonKey };
}
return null;
}

View file

@ -0,0 +1,30 @@
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
import { documentParseStateFromWorkerState } from '@/lib/server/compute/worker-parse-state';
import { documentKey } from '@/lib/server/documents/blobstore';
import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
export async function startPdfParseOperation(input: {
documentId: string;
userId: string;
namespace: string | null;
forceToken?: string;
}): Promise<{
workerState: WorkerOperationState<PdfLayoutJobResult>;
parseState: DocumentParseState;
}> {
const workerState = await createOrReusePdfWorkerOperation({
documentId: input.documentId,
namespace: input.namespace,
documentObjectKey: documentKey(input.documentId, input.namespace),
...(input.forceToken ? { forceToken: input.forceToken } : {}),
});
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
await recordJobEvent(input.userId, 'pdf_layout', workerState.opId, rateConfig);
return {
workerState,
parseState: documentParseStateFromWorkerState(workerState),
};
}

View file

@ -0,0 +1,106 @@
import { db } from '@/db';
import { documents } from '@/db/schema';
import {
enqueueDocumentPreview,
} from '@/lib/server/documents/previews';
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
import type { BaseDocument, DocumentType } from '@/types/documents';
type RegisterUploadedDocumentInput = {
documentId: string;
userId: string;
namespace: string | null;
name: string;
type: DocumentType;
size: number;
lastModified: number;
};
export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> {
const reusableParsedPdf = input.type === 'pdf'
? await findReusableParsedPdfResult(input.documentId)
: null;
const startedParse = input.type === 'pdf' && !reusableParsedPdf
? await startPdfParseOperation({
documentId: input.documentId,
userId: input.userId,
namespace: input.namespace,
})
: null;
const parsedJsonKey = reusableParsedPdf?.parsedJsonKey ?? null;
const parseState = input.type === 'pdf'
? stringifyDocumentParseState(
reusableParsedPdf
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
: (startedParse?.parseState ?? { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }),
)
: null;
await db
.insert(documents)
.values({
id: input.documentId,
userId: input.userId,
name: input.name,
type: input.type,
size: input.size,
lastModified: input.lastModified,
filePath: input.documentId,
parseState,
parsedJsonKey,
})
.onConflictDoUpdate({
target: [documents.id, documents.userId],
set: {
name: input.name,
type: input.type,
size: input.size,
lastModified: input.lastModified,
filePath: input.documentId,
parseState,
parsedJsonKey,
},
});
await enqueueDocumentPreview(
{
id: input.documentId,
type: input.type,
lastModified: input.lastModified,
},
input.namespace,
).catch((error) => {
serverLogger.warn({
event: 'documents.preview.enqueue.failed',
degraded: true,
fallbackPath: 'skip_preview_enqueue',
documentId: input.documentId,
error: errorToLog(error),
}, 'Failed to enqueue document preview');
});
if (startedParse) {
enqueueParsePdfJob({
documentId: input.documentId,
userId: input.userId,
namespace: input.namespace,
initialOpId: startedParse.workerState.opId,
initialJobId: startedParse.workerState.jobId,
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
});
}
return {
id: input.documentId,
name: input.name,
type: input.type,
size: input.size,
lastModified: input.lastModified,
scope: 'user',
};
}

View file

@ -1,11 +1,14 @@
import { and, eq, inArray, isNull } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
import { serverLogger } from '@/lib/server/logger';
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
import {
normalizeDocumentParseStateForCurrentParserVersion,
parseDocumentParseState,
resolveParsedPdfParserVersion,
stringifyDocumentParseState,
type DocumentParseState,
} from '@/lib/server/documents/parse-state';
@ -13,15 +16,20 @@ import { getCompute } from '@/lib/server/compute';
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
import type {
ParsedPdfDocument,
PdfLayoutJobBase,
PdfLayoutJobResult,
PdfLayoutProgress,
WorkerOperationState,
} from '@openreader/compute-core/api-contracts';
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
userId: string;
forceToken?: string;
initialOpId?: string;
initialJobId?: string;
initialStatus?: 'pending' | 'running';
};
const running = new Set<string>();
@ -41,7 +49,12 @@ function keyFor(input: UserPdfLayoutJobRequest): string {
if (forceToken) {
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
}
return `shared:${input.documentId}:${input.namespace || ''}`;
return `shared:${input.documentId}`;
}
function normalizeOpId(value: string | undefined): string | undefined {
const normalized = value?.trim();
return normalized ? normalized : undefined;
}
function sleep(ms: number): Promise<void> {
@ -84,7 +97,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise<ParseRow[
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
if (!row.parsedJsonKey) return false;
return parseDocumentParseState(row.parseState).status === 'ready';
return normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)).status === 'ready';
}
function userIdsFromRows(rows: ParseRow[]): string[] {
@ -95,6 +108,20 @@ function parseStateMatchCondition(expected: string | null) {
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
}
async function loadParsedDocumentForResult(
parsed: ParsedPdfDocument | undefined,
parsedJsonKey: string,
): Promise<ParsedPdfDocument | null> {
if (parsed) return parsed;
try {
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
} catch {
return null;
}
}
async function updateParseStateForUsers(input: {
documentId: string;
userIds: string[];
@ -125,6 +152,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
while (Date.now() < deadline) {
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
if (reusableParsed) {
const readyState: DocumentParseState = {
status: 'ready',
progress: null,
updatedAt: Date.now(),
parserVersion: PDF_PARSER_VERSION,
};
await updateParseStateForUsers({
documentId: input.documentId,
userIds: [input.userId],
parseState: stringifyDocumentParseState(readyState),
parsedJsonKey: reusableParsed.parsedJsonKey,
});
return;
}
const rows = await loadScopedRows(input);
const ready = rows.find(isReadyRow);
if (ready) {
@ -132,6 +176,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
status: 'ready',
progress: null,
updatedAt: Date.now(),
parserVersion: PDF_PARSER_VERSION,
};
await updateParseStateForUsers({
documentId: input.documentId,
@ -142,7 +187,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
return;
}
const statuses = rows.map((row) => parseDocumentParseState(row.parseState));
const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)));
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
const failedState = statuses.find((state) => state.status === 'failed');
if (failedState && !hasInFlight) {
@ -151,6 +196,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
progress: null,
updatedAt: Date.now(),
...(failedState.error ? { error: failedState.error } : {}),
...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}),
};
await updateParseStateForUsers({
documentId: input.documentId,
@ -174,6 +220,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
}
running.add(key);
let activeOpId = normalizeOpId(input.initialOpId);
let activeJobId = normalizeOpId(input.initialJobId);
try {
const scopedRows = await loadScopedRows(input);
const scopedUserIds = userIdsFromRows(scopedRows);
@ -181,12 +229,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
// Non-force jobs can short-circuit by reusing existing ready output from
// any row in the same document scope.
if (!input.forceToken?.trim()) {
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
if (reusableParsed) {
const readyState: DocumentParseState = {
status: 'ready',
progress: null,
updatedAt: Date.now(),
parserVersion: PDF_PARSER_VERSION,
};
await updateParseStateForUsers({
documentId: input.documentId,
userIds: [input.userId],
parseState: stringifyDocumentParseState(readyState),
parsedJsonKey: reusableParsed.parsedJsonKey,
});
return;
}
const ready = scopedRows.find(isReadyRow);
if (ready) {
const readyState: DocumentParseState = {
status: 'ready',
progress: null,
updatedAt: Date.now(),
parserVersion: PDF_PARSER_VERSION,
};
await updateParseStateForUsers({
documentId: input.documentId,
@ -201,17 +267,21 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
const coordinator = [...scopedRows].sort((a, b) => a.userId.localeCompare(b.userId))[0];
if (!coordinator) return;
const runningStateData: DocumentParseState = {
status: 'running',
const initialInflightStatus = input.initialStatus === 'running' ? 'running' : 'pending';
const inflightStateData: DocumentParseState = {
status: initialInflightStatus,
progress: null,
updatedAt: Date.now(),
parserVersion: PDF_PARSER_VERSION,
...(activeOpId ? { opId: activeOpId } : {}),
...(activeJobId ? { jobId: activeJobId } : {}),
};
const runningState = stringifyDocumentParseState(runningStateData);
const inflightState = stringifyDocumentParseState(inflightStateData);
const claimRows = (await db
.update(documents)
.set({
parseState: runningState,
parseState: inflightState,
})
.where(
and(
@ -234,12 +304,10 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
await updateParseStateForUsers({
documentId: input.documentId,
userIds: cohortUserIds,
parseState: runningState,
parseState: inflightState,
});
const compute = await getCompute();
let activeOpId: string | undefined;
let activeJobId: string | undefined;
let lastProgressWriteAt = 0;
let lastSnapshotWriteAt = 0;
let lastSnapshotStatus: 'pending' | 'running' | null = null;
@ -279,6 +347,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
status: mappedStatus,
progress: snapshot.progress ?? null,
updatedAt: now,
parserVersion: PDF_PARSER_VERSION,
...(activeOpId ? { opId: activeOpId } : {}),
...(activeJobId ? { jobId: activeJobId } : {}),
};
@ -305,6 +374,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
phase: progress.phase,
},
updatedAt: Date.now(),
parserVersion: PDF_PARSER_VERSION,
...(activeOpId ? { opId: activeOpId } : {}),
...(activeJobId ? { jobId: activeJobId } : {}),
};
@ -327,12 +397,17 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
}
const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey);
const finalScopedRows = await loadScopedRows(input);
const finalUserIds = userIdsFromRows(finalScopedRows);
const readyState: DocumentParseState = {
status: 'ready',
progress: null,
updatedAt: Date.now(),
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
...(activeOpId ? { opId: activeOpId } : {}),
...(activeJobId ? { jobId: activeJobId } : {}),
};
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
await updateParseStateForUsers({
@ -385,6 +460,9 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
progress: null,
updatedAt: Date.now(),
error: message,
parserVersion: PDF_PARSER_VERSION,
...(activeOpId ? { opId: activeOpId } : {}),
...(activeJobId ? { jobId: activeJobId } : {}),
};
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
await updateParseStateForUsers({

View file

@ -11,7 +11,6 @@ export interface BaseDocument {
parsedJsonKey?: string | null;
scope?: 'user';
folderId?: string;
isConverting?: boolean;
}
export interface PDFDocument extends BaseDocument {

View file

@ -1,6 +1,4 @@
import { Page, expect, type TestInfo, type Locator } from '@playwright/test';
import fs from 'fs';
import path from 'path';
import { createHash } from 'crypto';
const DIR = './tests/files/';
@ -10,14 +8,6 @@ function escapeRegExp(input: string) {
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function fixturePath(fileName: string) {
return path.join(__dirname, 'files', fileName);
}
function sha256HexOfFile(filePath: string) {
return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
}
async function waitForPdfViewerReady(page: Page, timeout = 60000) {
await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) });
const loader = page.getByTestId('pdf-status-loader').first();
@ -40,7 +30,7 @@ async function waitForPdfViewerReady(page: Page, timeout = 60000) {
}
/**
* Upload a sample epub or pdf
* Upload a sample document fixture
*/
export async function uploadFile(page: Page, filePath: string) {
const input = page.locator('input[type=file]').first();
@ -50,7 +40,7 @@ export async function uploadFile(page: Page, filePath: string) {
await input.setInputFiles(`${DIR}${filePath}`);
// Wait for the uploader to finish processing. The input is disabled while
// uploading/converting via react-dropzone's `disabled` prop.
// uploading via react-dropzone's `disabled` prop.
// Tolerate extremely fast operations where the disabled state may be missed.
try {
await expect(input).toBeDisabled({ timeout: 2000 });
@ -69,14 +59,8 @@ export async function uploadAndDisplay(page: Page, fileName: string) {
const lower = fileName.toLowerCase();
if (lower.endsWith('.docx')) {
// Best-effort: conversion can complete before we observe this UI state.
try {
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
} catch {
// ignore
}
const expectedId = sha256HexOfFile(fixturePath(fileName));
const targetLink = page.locator(`a[href$="/pdf/${expectedId}"]`).first();
const convertedName = `${fileName.replace(/\.[^.]+$/, '')}.pdf`;
const targetLink = page.getByRole('link', { name: new RegExp(escapeRegExp(convertedName), 'i') }).first();
await expect(targetLink).toBeVisible({ timeout: 15000 });
await dismissOnboardingModals(page);
await targetLink.click();
@ -188,10 +172,11 @@ async function dismissOnboardingModals(page: Page): Promise<void> {
* Wait for the play button to be clickable and click it
*/
export async function waitAndClickPlay(page: Page) {
// Wait for play button selector without disabled attribute
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();
const playButton = page.getByRole('button', { name: 'Play' });
await expect(playButton).toBeVisible();
await expect(playButton).toBeEnabled({ timeout: 15000 });
// Play the TTS by clicking the button
await page.getByRole('button', { name: 'Play' }).click();
await playButton.click();
// Use resilient processing transition helper (tolerates fast completion)
await expectProcessingTransition(page);
}

View file

@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => {
await page.waitForTimeout(750);
// After resize, playback should have paused (Play button visible)
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });
const playButton = page.getByRole('button', { name: 'Play' });
await expect(playButton).toBeVisible({ timeout: 15000 });
await expect(playButton).toBeEnabled({ timeout: 15000 });
// Resume playback and ensure processing -> playing
await page.getByRole('button', { name: 'Play' }).click();
await playButton.click();
await expectProcessingTransition(page);
});
});

View file

@ -0,0 +1,150 @@
import { createHash } from 'node:crypto';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { NextRequest } from 'next/server';
const hoisted = vi.hoisted(() => ({
requireAuthContext: vi.fn(),
registerUploadedDocument: vi.fn(),
convertDocxBufferToPdfBuffer: vi.fn(),
headTempDocumentBlob: vi.fn(),
getTempDocumentBlob: vi.fn(),
getTempDocumentFinalizeReceipt: vi.fn(),
putTempDocumentFinalizeReceipt: vi.fn(),
deleteTempDocumentUpload: vi.fn(),
headDocumentBlob: vi.fn(),
copyTempDocumentBlobToDocument: vi.fn(),
putDocumentBlob: vi.fn(),
}));
vi.mock('@/lib/server/auth/auth', () => ({
requireAuthContext: hoisted.requireAuthContext,
}));
vi.mock('@/lib/server/documents/register-upload', () => ({
registerUploadedDocument: hoisted.registerUploadedDocument,
}));
vi.mock('@/lib/server/documents/docx-convert', () => ({
convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer,
}));
vi.mock('@/lib/server/documents/blobstore', () => ({
TEMP_DOCUMENT_UPLOAD_TTL_MS: 24 * 60 * 60 * 1000,
copyTempDocumentBlobToDocument: hoisted.copyTempDocumentBlobToDocument,
deleteTempDocumentUpload: hoisted.deleteTempDocumentUpload,
getTempDocumentBlob: hoisted.getTempDocumentBlob,
getTempDocumentFinalizeReceipt: hoisted.getTempDocumentFinalizeReceipt,
headDocumentBlob: hoisted.headDocumentBlob,
headTempDocumentBlob: hoisted.headTempDocumentBlob,
isMissingBlobError: vi.fn((error: unknown) => {
const maybe = error as { code?: string } | undefined;
return maybe?.code === 'NoSuchKey';
}),
isPreconditionFailed: vi.fn(() => false),
isValidTempUploadToken: vi.fn(() => true),
putDocumentBlob: hoisted.putDocumentBlob,
putTempDocumentFinalizeReceipt: hoisted.putTempDocumentFinalizeReceipt,
}));
vi.mock('@/lib/server/testing/test-namespace', () => ({
getOpenReaderTestNamespace: vi.fn(() => null),
}));
vi.mock('@/lib/server/storage/s3', () => ({
isS3Configured: vi.fn(() => true),
}));
vi.mock('@/lib/server/logger', () => ({
errorToLog: vi.fn((error: unknown) => error),
serverLogger: {
warn: vi.fn(),
error: vi.fn(),
},
}));
describe('POST /api/documents/blob/upload/finalize DOCX flow', () => {
beforeEach(() => {
hoisted.requireAuthContext.mockReset();
hoisted.registerUploadedDocument.mockReset();
hoisted.convertDocxBufferToPdfBuffer.mockReset();
hoisted.headTempDocumentBlob.mockReset();
hoisted.getTempDocumentBlob.mockReset();
hoisted.getTempDocumentFinalizeReceipt.mockReset();
hoisted.putTempDocumentFinalizeReceipt.mockReset();
hoisted.deleteTempDocumentUpload.mockReset();
hoisted.headDocumentBlob.mockReset();
hoisted.copyTempDocumentBlobToDocument.mockReset();
hoisted.putDocumentBlob.mockReset();
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
hoisted.getTempDocumentFinalizeReceipt.mockResolvedValue(null);
hoisted.headTempDocumentBlob.mockResolvedValue({
contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
contentLength: 12,
lastModified: Date.now(),
});
hoisted.getTempDocumentBlob.mockResolvedValue(Buffer.from('docx-bytes'));
hoisted.convertDocxBufferToPdfBuffer.mockResolvedValue(Buffer.from('pdf-bytes'));
hoisted.headDocumentBlob
.mockRejectedValueOnce({ code: 'NoSuchKey' })
.mockResolvedValue({
contentLength: Buffer.byteLength('pdf-bytes'),
contentType: 'application/pdf',
eTag: 'etag-1',
});
hoisted.registerUploadedDocument.mockImplementation(async (input: { documentId: string; name: string; type: string; size: number; lastModified: number }) => ({
id: input.documentId,
name: input.name,
type: input.type,
size: input.size,
lastModified: input.lastModified,
scope: 'user',
}));
hoisted.putTempDocumentFinalizeReceipt.mockResolvedValue(undefined);
hoisted.deleteTempDocumentUpload.mockResolvedValue(undefined);
});
test('converts raw DOCX during finalize and registers a PDF', async () => {
const { POST } = await import('../../src/app/api/documents/blob/upload/finalize/route');
const request = new NextRequest('http://localhost/api/documents/blob/upload/finalize', {
method: 'POST',
body: JSON.stringify({
uploads: [{
token: '123e4567-e89b-12d3-a456-426614174000',
name: 'Report.docx',
type: 'docx',
lastModified: 1700000000000,
}],
}),
headers: {
'Content-Type': 'application/json',
},
});
const response = await POST(request);
const expectedId = createHash('sha256').update(Buffer.from('pdf-bytes')).digest('hex');
expect(response.status).toBe(200);
await expect(response.json()).resolves.toMatchObject({
stored: [{
id: expectedId,
name: 'Report.pdf',
type: 'pdf',
}],
});
expect(hoisted.convertDocxBufferToPdfBuffer).toHaveBeenCalledTimes(1);
expect(hoisted.putDocumentBlob).toHaveBeenCalledWith(
expectedId,
Buffer.from('pdf-bytes'),
'application/pdf',
null,
{ ifNoneMatch: true },
);
expect(hoisted.copyTempDocumentBlobToDocument).not.toHaveBeenCalled();
expect(hoisted.registerUploadedDocument).toHaveBeenCalledWith(expect.objectContaining({
documentId: expectedId,
name: 'Report.pdf',
type: 'pdf',
}));
});
});

View file

@ -0,0 +1,13 @@
import { describe, expect, test } from 'vitest';
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
import { buildPdfOpKey } from '../../src/lib/server/compute/worker';
describe('pdf worker op key', () => {
test('includes parser version in the reuse boundary', () => {
expect(buildPdfOpKey({
documentId: 'doc-1',
namespace: null,
documentObjectKey: 'documents/doc-1.pdf',
})).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`);
});
});

View file

@ -0,0 +1,84 @@
import { describe, expect, test } from 'vitest';
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
import {
normalizeDocumentParseStateForCurrentParserVersion,
parseDocumentParseState,
stringifyDocumentParseState,
} from '../../src/lib/server/documents/parse-state';
describe('document parse state parser-version handling', () => {
test('preserves parserVersion through stringify/parse', () => {
const state = parseDocumentParseState(stringifyDocumentParseState({
status: 'ready',
progress: null,
updatedAt: 1234,
parserVersion: PDF_PARSER_VERSION,
}));
expect(state).toEqual({
status: 'ready',
progress: null,
updatedAt: 1234,
parserVersion: PDF_PARSER_VERSION,
});
});
test('invalidates ready and inflight states from older parser versions', () => {
expect(normalizeDocumentParseStateForCurrentParserVersion({
status: 'ready',
progress: null,
updatedAt: 1111,
parserVersion: 'old-parser',
opId: 'op-old',
}, 2222)).toEqual({
status: 'pending',
progress: null,
updatedAt: 2222,
});
expect(normalizeDocumentParseStateForCurrentParserVersion({
status: 'running',
progress: {
totalPages: 10,
pagesParsed: 3,
currentPage: 4,
phase: 'infer',
},
updatedAt: 1111,
parserVersion: 'old-parser',
opId: 'op-old',
}, 3333)).toEqual({
status: 'pending',
progress: null,
updatedAt: 3333,
});
});
test('keeps failed states and current-version ready states intact', () => {
expect(normalizeDocumentParseStateForCurrentParserVersion({
status: 'failed',
progress: null,
updatedAt: 1111,
error: 'no text layer',
parserVersion: 'old-parser',
})).toEqual({
status: 'failed',
progress: null,
updatedAt: 1111,
error: 'no text layer',
parserVersion: 'old-parser',
});
expect(normalizeDocumentParseStateForCurrentParserVersion({
status: 'ready',
progress: null,
updatedAt: 1111,
parserVersion: PDF_PARSER_VERSION,
})).toEqual({
status: 'ready',
progress: null,
updatedAt: 1111,
parserVersion: PDF_PARSER_VERSION,
});
});
});

View file

@ -0,0 +1,156 @@
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import { NextRequest } from 'next/server';
const hoisted = vi.hoisted(() => ({
db: null as {
select: ReturnType<typeof vi.fn>;
update: ReturnType<typeof vi.fn>;
} | null,
row: {
id: 'doc-1',
userId: 'user-1',
parseState: null as string | null,
},
requireAuthContext: vi.fn(),
backfillPendingPdfParseOperation: vi.fn(),
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
}));
vi.mock('@/db', () => ({
get db() {
return hoisted.db;
},
}));
vi.mock('@/lib/server/auth/auth', () => ({
requireAuthContext: hoisted.requireAuthContext,
}));
vi.mock('@/lib/server/documents/parse-state-backfill', () => ({
backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation,
}));
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
}));
vi.mock('@/lib/server/documents/blobstore', () => ({
isValidDocumentId: vi.fn(() => true),
}));
vi.mock('@/lib/server/storage/s3', () => ({
isS3Configured: vi.fn(() => true),
}));
vi.mock('@/lib/server/testing/test-namespace', () => ({
getOpenReaderTestNamespace: vi.fn(() => null),
}));
vi.mock('@/lib/server/logger', () => ({
createRequestLogger: vi.fn(() => ({
logger: {
warn: vi.fn(),
info: vi.fn(),
error: vi.fn(),
},
requestId: 'req-test',
})),
hashForLog: vi.fn(() => 'user-hash'),
}));
describe('GET /api/documents/[id]/parsed/events legacy backfill', () => {
const originalFetch = global.fetch;
beforeEach(() => {
process.env.COMPUTE_WORKER_URL = 'http://localhost:4010';
process.env.COMPUTE_WORKER_TOKEN = 'worker-test-token';
hoisted.row = {
id: 'doc-1',
userId: 'user-1',
parseState: null,
};
hoisted.db = {
select: vi.fn(() => ({
from: vi.fn(() => ({
where: vi.fn(async () => ([{ ...hoisted.row }])),
})),
})),
update: vi.fn(() => ({
set: vi.fn((values: { parseState?: string | null }) => ({
where: vi.fn(async () => {
if (typeof values.parseState !== 'undefined') {
hoisted.row.parseState = values.parseState;
}
return [];
}),
})),
})),
};
hoisted.requireAuthContext.mockReset();
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
hoisted.backfillPendingPdfParseOperation.mockReset();
hoisted.healStaleDocumentParseState.mockClear();
global.fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_, reject) => {
const signal = init?.signal;
if (signal) {
signal.addEventListener('abort', () => {
reject(new DOMException('Aborted', 'AbortError'));
}, { once: true });
}
})) as typeof fetch;
});
afterEach(() => {
global.fetch = originalFetch;
});
test('creates a worker op for legacy pending PDFs without opId', async () => {
hoisted.backfillPendingPdfParseOperation.mockResolvedValue({
opId: 'op-legacy-1',
opKey: 'pdf_layout|v1|doc-1||doc-1|',
jobId: 'job-legacy-1',
kind: 'pdf_layout',
status: 'queued',
queuedAt: Date.now(),
progress: null,
result: undefined,
error: undefined,
updatedAt: Date.now(),
});
const { GET } = await import('../../src/app/api/documents/[id]/parsed/events/route');
const controller = new AbortController();
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed/events', {
signal: controller.signal,
});
const response = await GET(request, {
params: Promise.resolve({ id: 'doc-1' }),
});
expect(response.status).toBe(200);
const reader = response.body?.getReader();
expect(reader).toBeDefined();
const first = await reader!.read();
const chunk = new TextDecoder().decode(first.value);
expect(chunk).toContain('event: snapshot');
expect(chunk).toContain('"parseStatus":"pending"');
expect(chunk).toContain('"opId":"op-legacy-1"');
controller.abort();
await reader!.cancel().catch(() => {});
expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({
documentId: 'doc-1',
userId: 'user-1',
namespace: null,
state: expect.objectContaining({ status: 'pending' }),
}));
const parseState = String(hoisted.row.parseState ?? '');
expect(parseState).toContain('"status":"pending"');
expect(parseState).toContain('"opId":"op-legacy-1"');
expect(parseState).toContain('"jobId":"job-legacy-1"');
});
});

View file

@ -14,8 +14,9 @@ const hoisted = vi.hoisted(() => ({
},
requireAuthContext: vi.fn(),
fetchWorkerOperationState: vi.fn(),
backfillPendingPdfParseOperation: vi.fn(),
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
startPdfParseOperation: vi.fn(),
enqueueParsePdfJob: vi.fn(),
}));
vi.mock('@/db', () => ({
@ -32,14 +33,18 @@ vi.mock('@/lib/server/compute/worker-op-state', () => ({
fetchWorkerOperationState: hoisted.fetchWorkerOperationState,
}));
vi.mock('@/lib/server/documents/parse-state-backfill', () => ({
backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation,
}));
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
}));
vi.mock('@/lib/server/documents/pdf-parse-operation', () => ({
startPdfParseOperation: hoisted.startPdfParseOperation,
}));
vi.mock('@/lib/server/jobs/user-pdf-layout-job', () => ({
enqueueParsePdfJob: hoisted.enqueueParsePdfJob,
}));
vi.mock('@/lib/server/documents/blobstore', () => ({
documentKey: vi.fn(),
getParsedDocumentBlob: vi.fn(),
@ -69,7 +74,7 @@ vi.mock('@/lib/server/logger', () => ({
hashForLog: vi.fn(() => 'user-hash'),
}));
describe('GET /api/documents/[id]/parsed legacy backfill', () => {
describe('GET /api/documents/[id]/parsed pure data fetch', () => {
beforeEach(async () => {
process.env.BASE_URL = 'http://localhost:3003';
process.env.AUTH_SECRET = 'test-secret';
@ -104,44 +109,25 @@ describe('GET /api/documents/[id]/parsed legacy backfill', () => {
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
hoisted.fetchWorkerOperationState.mockReset();
hoisted.fetchWorkerOperationState.mockResolvedValue(null);
hoisted.backfillPendingPdfParseOperation.mockReset();
hoisted.healStaleDocumentParseState.mockClear();
hoisted.startPdfParseOperation.mockReset();
hoisted.enqueueParsePdfJob.mockReset();
});
test('creates a worker op for legacy pending PDFs without opId', async () => {
hoisted.backfillPendingPdfParseOperation.mockResolvedValue({
opId: 'op-legacy-1',
opKey: 'pdf_layout|v1|doc-1||doc-1|',
jobId: 'job-legacy-1',
kind: 'pdf_layout',
status: 'queued',
queuedAt: Date.now(),
progress: null,
result: undefined,
error: undefined,
updatedAt: Date.now(),
});
test('returns non-ready status without creating a worker op for legacy pending PDFs without opId', async () => {
const { GET } = await import('../../src/app/api/documents/[id]/parsed/route');
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed');
const response = await GET(request, {
params: Promise.resolve({ id: 'doc-1' }),
});
expect(response.status).toBe(202);
expect(response.status).toBe(409);
await expect(response.json()).resolves.toMatchObject({
parseStatus: 'pending',
opId: 'op-legacy-1',
opId: null,
});
expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({
documentId: 'doc-1',
userId: 'user-1',
namespace: null,
state: expect.objectContaining({ status: 'pending' }),
}));
const parseState = String(hoisted.row.parseState ?? '');
expect(parseState).toContain('"status":"pending"');
expect(parseState).toContain('"opId":"op-legacy-1"');
expect(parseState).toContain('"jobId":"job-legacy-1"');
expect(hoisted.startPdfParseOperation).not.toHaveBeenCalled();
expect(hoisted.enqueueParsePdfJob).not.toHaveBeenCalled();
expect(hoisted.row.parseState).toBeNull();
});
});

View file

@ -1,4 +1,5 @@
import { describe, expect, test } from 'vitest';
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
import {
documentParseStateFromWorkerState,
@ -11,7 +12,7 @@ function makeWorkerState(
): WorkerOperationState<PdfLayoutJobResult> {
return {
opId: 'op-123',
opKey: 'pdf_layout|v1|doc-1',
opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`,
kind: 'pdf_layout',
jobId: 'job-123',
status: 'queued',
@ -41,6 +42,7 @@ describe('worker parse state mapping', () => {
status: 'pending',
progress: null,
updatedAt: 1234,
parserVersion: PDF_PARSER_VERSION,
opId: 'op-123',
jobId: 'job-123',
});
@ -66,6 +68,7 @@ describe('worker parse state mapping', () => {
phase: 'infer',
},
updatedAt: 5678,
parserVersion: PDF_PARSER_VERSION,
opId: 'op-123',
jobId: 'job-123',
});
@ -84,6 +87,7 @@ describe('worker parse state mapping', () => {
status: 'failed',
progress: null,
updatedAt: 9999,
parserVersion: PDF_PARSER_VERSION,
opId: 'op-123',
jobId: 'job-123',
error: 'layout model crashed',

View file

@ -30,6 +30,29 @@ test.describe('Document Upload Tests', () => {
await expectDocumentListed(page, 'sample.txt');
});
test('reuses the same canonical id for identical uploads', async ({ page }) => {
await uploadFile(page, 'sample.pdf');
await uploadFile(page, 'sample.pdf');
const result = await page.evaluate(async () => {
const res = await fetch('/api/documents', { cache: 'no-store' });
if (!res.ok) {
return { ok: false as const, reason: `status:${res.status}` };
}
const data = await res.json() as { documents?: Array<{ id: string; name: string }> };
const matching = (data.documents || []).filter((doc) => doc.name === 'sample.pdf');
const uniqueIds = Array.from(new Set(matching.map((doc) => doc.id)));
return { ok: true as const, matchingCount: matching.length, uniqueIds };
});
expect(result.ok).toBeTruthy();
if (!result.ok) {
throw new Error(`Failed to inspect uploaded documents: ${result.reason}`);
}
expect(result.matchingCount).toBe(1);
expect(result.uniqueIds).toHaveLength(1);
});
test('hashes text/HTML docs using UTF-8 encoded stored string', async ({ page }) => {
await uploadFile(page, 'sample.txt');
await expectDocumentListed(page, 'sample.txt');
@ -77,13 +100,7 @@ test.describe('Document Upload Tests', () => {
test('uploads and converts a DOCX document', async ({ page }) => {
await uploadFile(page, 'sample.docx');
// Should see the converting message (best-effort; conversion may complete extremely fast)
try {
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
} catch {
// ignore
}
// After conversion, should see the PDF with the same name
// DOCX uploads are normalized into stored PDFs with the same basename.
await expectDocumentListed(page, 'sample.pdf');
});