Merge pull request #99 from richardr1126/pdf/parse-fixes
## Summary by CodeRabbit * **New Features** * Expose PDF parser version system-wide; parser-version-aware parsing, reuse, and caching. Added tokenized temp upload flow, presign/finalize endpoints, and server-side DOCX→PDF conversion. * Playback readiness propagated to TTS players/pages; unified upload/delete APIs and simplified uploader/sidebar UX. * **Bug Fixes** * Normalize stored parse state to current parser version to avoid stale readiness and improve reuse. * Prevent premature playback starts and redundant preparation. * **Tests** * Added unit and E2E tests for parser-version handling, op-key inclusion, parse-state behavior, and canonical upload reuse.
This commit is contained in:
commit
e2c0240add
46 changed files with 1786 additions and 1049 deletions
|
|
@ -16,6 +16,7 @@ export {
|
|||
} from './config/timeout';
|
||||
export { renderPage } from './pdf/render';
|
||||
export { mergeTextWithRegions } from './pdf/merge';
|
||||
export { PDF_PARSER_VERSION } from './pdf/parser-version';
|
||||
export { stitchCrossPageBlocks } from './pdf/stitch';
|
||||
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { ensureModel } from './model';
|
|||
import { runLayoutModel } from './runLayoutModel';
|
||||
import { mergeTextWithRegions } from './merge';
|
||||
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||
import { PDF_PARSER_VERSION } from './parser-version';
|
||||
import { stitchCrossPageBlocks } from './stitch';
|
||||
import { renderPage } from './render';
|
||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||
|
|
@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
|
|||
const doc: ParsedPdfDocument = {
|
||||
schemaVersion: 1,
|
||||
documentId: input.documentId,
|
||||
parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69',
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
parsedAt: Date.now(),
|
||||
pages,
|
||||
};
|
||||
|
|
|
|||
1
compute/core/src/pdf/parser-version.ts
Normal file
1
compute/core/src/pdf/parser-version.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||
|
|
@ -1,12 +1,15 @@
|
|||
import fs from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import path from 'node:path';
|
||||
|
||||
let cachedStandardFontDataUrl: string | null = null;
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
export function resolvePdfjsStandardFontDataUrl(): string {
|
||||
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
||||
|
||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
||||
const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json'));
|
||||
const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts');
|
||||
|
||||
if (!fs.existsSync(standardFontDir)) {
|
||||
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export default function EPUBPage() {
|
|||
const {
|
||||
setCurrentDocument,
|
||||
currDocName,
|
||||
isPlaybackReady,
|
||||
clearCurrDoc,
|
||||
createFullAudioBook: createEPUBAudioBook,
|
||||
regenerateChapter: regenerateEPUBChapter,
|
||||
|
|
@ -242,7 +243,7 @@ export default function EPUBPage() {
|
|||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<TTSPlayer />
|
||||
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||
)}
|
||||
<DocumentSettings
|
||||
epub
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ export interface EpubDocumentState {
|
|||
currDocPages: number | undefined;
|
||||
currDocPage: number | string;
|
||||
currDocText: string | undefined;
|
||||
isPlaybackReady: boolean;
|
||||
setCurrentDocument: (id: string) => Promise<void>;
|
||||
clearCurrDoc: () => void;
|
||||
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
|
||||
|
|
@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [currDocText, setCurrDocText] = useState<string>();
|
||||
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||
// Mirror state into a ref so resolveEpubLocator (registered once with
|
||||
// TTSContext via a stable callback) can always read the latest page text
|
||||
// without forcing re-registration on every page turn.
|
||||
|
|
@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
setCurrDocData(undefined);
|
||||
setCurrDocName(undefined);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
setCurrDocPages(undefined);
|
||||
isEPUBSetOnce.current = false;
|
||||
shouldPauseRef.current = true;
|
||||
|
|
@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
*/
|
||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||
try {
|
||||
setIsPlaybackReady(false);
|
||||
const meta = await getDocumentMetadata(id);
|
||||
if (!meta) {
|
||||
clearCurrDoc();
|
||||
|
|
@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
*/
|
||||
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
|
||||
try {
|
||||
setIsPlaybackReady(false);
|
||||
const location = rendition?.location;
|
||||
if (!location) return '';
|
||||
const { start, end } = location;
|
||||
|
|
@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
nextText: continuationPreview
|
||||
});
|
||||
setCurrDocText(textContent);
|
||||
setIsPlaybackReady(true);
|
||||
|
||||
return textContent;
|
||||
} catch (error) {
|
||||
|
|
@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
clearCurrDoc,
|
||||
extractPageText,
|
||||
walkUpcomingRenderedLocations,
|
||||
|
|
@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
clearCurrDoc,
|
||||
extractPageText,
|
||||
walkUpcomingRenderedLocations,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export default function HTMLPage() {
|
|||
setCurrentDocument,
|
||||
currDocData,
|
||||
currDocName,
|
||||
isPlaybackReady,
|
||||
blocks,
|
||||
isTxt,
|
||||
clearCurrDoc,
|
||||
|
|
@ -233,7 +234,7 @@ export default function HTMLPage() {
|
|||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<TTSPlayer />
|
||||
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||
)}
|
||||
<DocumentSettings
|
||||
html
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ export interface HtmlDocumentState {
|
|||
currDocData: string | undefined;
|
||||
currDocName: string | undefined;
|
||||
currDocText: string | undefined;
|
||||
isPlaybackReady: boolean;
|
||||
blocks: HtmlBlock[];
|
||||
isTxt: boolean;
|
||||
setCurrentDocument: (id: string) => Promise<void>;
|
||||
|
|
@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
|
||||
const [currDocData, setCurrDocData] = useState<string>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||
|
||||
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
|
||||
const blocks = useMemo(
|
||||
|
|
@ -91,22 +93,43 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
// sentence splitting + sequential advancement from there.
|
||||
const lastFedDocRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (!currDocText) return;
|
||||
const key = `${currDocData ?? ''}::${currDocText.length}`;
|
||||
if (lastFedDocRef.current === key) return;
|
||||
if (currDocData === undefined) {
|
||||
lastFedDocRef.current = null;
|
||||
setTTSText('');
|
||||
setIsPlaybackReady(false);
|
||||
return;
|
||||
}
|
||||
if (!currDocText) {
|
||||
lastFedDocRef.current = null;
|
||||
setTTSText('');
|
||||
setIsPlaybackReady(true);
|
||||
return;
|
||||
}
|
||||
const key = `${currDocName ?? ''}::${currDocData ?? ''}::${currDocText.length}`;
|
||||
if (lastFedDocRef.current === key) {
|
||||
setIsPlaybackReady(true);
|
||||
return;
|
||||
}
|
||||
setIsPlaybackReady(false);
|
||||
lastFedDocRef.current = key;
|
||||
setTTSText(currDocText);
|
||||
}, [currDocText, currDocData, setTTSText]);
|
||||
setIsPlaybackReady(true);
|
||||
}, [currDocName, currDocText, currDocData, setTTSText]);
|
||||
|
||||
const clearCurrDoc = useCallback(() => {
|
||||
setCurrDocData(undefined);
|
||||
setCurrDocName(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastFedDocRef.current = null;
|
||||
setTTSText('');
|
||||
stop();
|
||||
}, [stop]);
|
||||
}, [setTTSText, stop]);
|
||||
|
||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||
try {
|
||||
setIsPlaybackReady(false);
|
||||
lastFedDocRef.current = null;
|
||||
setTTSText('');
|
||||
const meta = await getDocumentMetadata(id);
|
||||
if (!meta) {
|
||||
console.error('Document not found on server');
|
||||
|
|
@ -125,7 +148,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
console.error('Failed to get HTML document:', error);
|
||||
clearCurrDoc();
|
||||
}
|
||||
}, [clearCurrDoc]);
|
||||
}, [clearCurrDoc, setTTSText]);
|
||||
|
||||
const audiobookAdapter = useMemo(
|
||||
() =>
|
||||
|
|
@ -207,6 +230,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
currDocData,
|
||||
currDocName,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
blocks,
|
||||
isTxt,
|
||||
setCurrentDocument,
|
||||
|
|
@ -218,6 +242,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
currDocData,
|
||||
currDocName,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
blocks,
|
||||
isTxt,
|
||||
setCurrentDocument,
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ export default function PDFViewerPage() {
|
|||
clearCurrDoc,
|
||||
currDocPage,
|
||||
currDocPages,
|
||||
isPlaybackReady,
|
||||
parseStatus,
|
||||
parseProgress,
|
||||
documentSettings,
|
||||
|
|
@ -487,7 +488,7 @@ export default function PDFViewerPage() {
|
|||
</div>
|
||||
</div>
|
||||
) : isParseReady ? (
|
||||
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} />
|
||||
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} isPlaybackReady={isPlaybackReady} />
|
||||
) : null}
|
||||
<DocumentSettings
|
||||
isOpen={activeSidebar === 'settings'}
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ export interface PdfDocumentState {
|
|||
currDocPages: number | undefined;
|
||||
currDocPage: number;
|
||||
currDocText: string | undefined;
|
||||
isPlaybackReady: boolean;
|
||||
pdfDocument: PDFDocumentProxy | undefined;
|
||||
parsedDocument: ParsedPdfDocument | null;
|
||||
parseStatus: PdfParseStatus | null;
|
||||
|
|
@ -142,6 +143,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [currDocText, setCurrDocText] = useState<string>();
|
||||
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
||||
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
||||
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
||||
|
|
@ -167,52 +169,29 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
// Guards for setCurrentDocument to prevent stale loads from overwriting newer selections.
|
||||
const docLoadSeqRef = useRef(0);
|
||||
const docLoadAbortRef = useRef<AbortController | null>(null);
|
||||
const parsePollAbortRef = useRef<AbortController | null>(null);
|
||||
const parseStreamAbortRef = useRef<AbortController | null>(null);
|
||||
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
||||
const lastPreparedPlaybackPageRef = useRef<number | null>(null);
|
||||
|
||||
const fetchParsedDocument = useCallback(async (
|
||||
const loadParsedDocumentOnce = useCallback(async (
|
||||
documentId: string,
|
||||
initialStatus: PdfParseStatus | null,
|
||||
signal: AbortSignal,
|
||||
initialOpId?: string | null,
|
||||
): Promise<void> => {
|
||||
// Legacy PDFs may have null parseStatus; treat as pending so opening the
|
||||
// document backfills parse output via the parsed endpoint polling path.
|
||||
const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending';
|
||||
setParseStatus(effectiveInitialStatus);
|
||||
if (signal.aborted) return;
|
||||
const parsed = await getParsedPdfDocument(documentId, { signal });
|
||||
if (signal.aborted) return;
|
||||
setParsedDocument(parsed);
|
||||
setParseStatus('ready');
|
||||
setParseProgress(null);
|
||||
const delayMs = 1200;
|
||||
const retryFailed = effectiveInitialStatus === 'failed';
|
||||
let attempt = 0;
|
||||
let effectiveOpId = initialOpId?.trim() || null;
|
||||
while (!signal.aborted) {
|
||||
if (signal.aborted) return;
|
||||
const result = await getParsedPdfDocument(documentId, {
|
||||
signal,
|
||||
retryFailed: retryFailed && attempt === 0,
|
||||
...(effectiveOpId ? { opId: effectiveOpId } : {}),
|
||||
});
|
||||
if (result.status === 'ready') {
|
||||
setParsedDocument(result.parsed);
|
||||
setParseStatus('ready');
|
||||
setParseProgress(null);
|
||||
setActiveParseOpId(null);
|
||||
return;
|
||||
}
|
||||
if ('opId' in result && typeof result.opId === 'string' && result.opId.trim()) {
|
||||
effectiveOpId = result.opId.trim();
|
||||
setActiveParseOpId(effectiveOpId);
|
||||
}
|
||||
setParseStatus(result.status);
|
||||
setParseProgress(result.parseProgress ?? null);
|
||||
if (result.status === 'failed') {
|
||||
setParsedDocument(null);
|
||||
setActiveParseOpId(null);
|
||||
return;
|
||||
}
|
||||
attempt += 1;
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
}
|
||||
setActiveParseOpId(null);
|
||||
}, []);
|
||||
|
||||
const resetParsedDocumentState = useCallback(() => {
|
||||
setParsedDocument(null);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
pageTextCacheRef.current.clear();
|
||||
}, []);
|
||||
|
||||
const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise<void> => {
|
||||
|
|
@ -226,42 +205,57 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
}
|
||||
}, []);
|
||||
|
||||
const startParsedPolling = useCallback((documentId: string, initialOpId?: string | null) => {
|
||||
parsePollAbortRef.current?.abort();
|
||||
const startParsedEventStream = useCallback((documentId: string, initialOpId?: string | null) => {
|
||||
parseStreamAbortRef.current?.abort();
|
||||
parseSseCloseRef.current?.();
|
||||
parseSseCloseRef.current = null;
|
||||
setParseProgress(null);
|
||||
setActiveParseOpId(initialOpId?.trim() || null);
|
||||
const controller = new AbortController();
|
||||
parsePollAbortRef.current = controller;
|
||||
parseStreamAbortRef.current = controller;
|
||||
let isResolvingTerminalState = false;
|
||||
|
||||
const closeSse = subscribeParsedPdfDocumentEvents(documentId, {
|
||||
opId: initialOpId?.trim() || null,
|
||||
}, {
|
||||
onSnapshot: (snapshot) => {
|
||||
if (controller.signal.aborted) return;
|
||||
if (isResolvingTerminalState) return;
|
||||
if (typeof snapshot.opId === 'string' && snapshot.opId.trim()) {
|
||||
setActiveParseOpId(snapshot.opId.trim());
|
||||
}
|
||||
setParseStatus(snapshot.parseStatus);
|
||||
setParseProgress(snapshot.parseProgress);
|
||||
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
|
||||
if (snapshot.parseStatus === 'failed') {
|
||||
setParsedDocument(null);
|
||||
setActiveParseOpId(null);
|
||||
} else {
|
||||
void fetchParsedDocument(
|
||||
documentId,
|
||||
'ready',
|
||||
controller.signal,
|
||||
typeof snapshot.opId === 'string' ? snapshot.opId : (initialOpId ?? null),
|
||||
);
|
||||
}
|
||||
if (snapshot.parseStatus === 'ready') {
|
||||
isResolvingTerminalState = true;
|
||||
void (async () => {
|
||||
try {
|
||||
await loadParsedDocumentOnce(documentId, controller.signal);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') return;
|
||||
console.error('Failed to load parsed PDF after ready status:', error);
|
||||
resetParsedDocumentState();
|
||||
} finally {
|
||||
if (parseSseCloseRef.current === closeSse) {
|
||||
closeSse();
|
||||
parseSseCloseRef.current = null;
|
||||
}
|
||||
if (parseStreamAbortRef.current === controller) {
|
||||
parseStreamAbortRef.current = null;
|
||||
}
|
||||
}
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (snapshot.parseStatus === 'failed') {
|
||||
isResolvingTerminalState = true;
|
||||
closeSse();
|
||||
parseSseCloseRef.current = null;
|
||||
if (parsePollAbortRef.current === controller) {
|
||||
parsePollAbortRef.current = null;
|
||||
if (parseStreamAbortRef.current === controller) {
|
||||
parseStreamAbortRef.current = null;
|
||||
}
|
||||
resetParsedDocumentState();
|
||||
setActiveParseOpId(null);
|
||||
return;
|
||||
}
|
||||
},
|
||||
|
|
@ -281,11 +275,11 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
if (parseSseCloseRef.current === closeSse) {
|
||||
parseSseCloseRef.current = null;
|
||||
}
|
||||
if (parsePollAbortRef.current === controller) {
|
||||
parsePollAbortRef.current = null;
|
||||
if (parseStreamAbortRef.current === controller) {
|
||||
parseStreamAbortRef.current = null;
|
||||
}
|
||||
}, { once: true });
|
||||
}, [fetchParsedDocument, setActiveParseOpId]);
|
||||
}, [loadParsedDocumentOnce, resetParsedDocumentState, setActiveParseOpId]);
|
||||
|
||||
useEffect(() => {
|
||||
pdfDocumentRef.current = pdfDocument;
|
||||
|
|
@ -297,6 +291,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
|
||||
useEffect(() => {
|
||||
setCurrDocPage(currDocPageNumber);
|
||||
setIsPlaybackReady(false);
|
||||
}, [currDocPageNumber]);
|
||||
|
||||
/**
|
||||
|
|
@ -324,11 +319,13 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
if (!currentPdf) return;
|
||||
const seq = ++loadSeqRef.current;
|
||||
const pageNumber = currDocPageNumber;
|
||||
setIsPlaybackReady(false);
|
||||
|
||||
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
|
||||
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
|
||||
|
||||
if (parseStatus !== 'ready' || !parsedDocument) {
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setCurrDocText(undefined);
|
||||
setTTSText('', { location: currDocPageNumber });
|
||||
return;
|
||||
|
|
@ -401,7 +398,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
return;
|
||||
}
|
||||
|
||||
if (text !== currDocText || text === '') {
|
||||
const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber;
|
||||
if (shouldPreparePlayback) {
|
||||
setCurrDocText(text);
|
||||
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
|
||||
setTTSText(text, {
|
||||
|
|
@ -414,6 +412,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
|
||||
});
|
||||
}
|
||||
lastPreparedPlaybackPageRef.current = currDocPageNumber;
|
||||
setIsPlaybackReady(true);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
|
|
@ -461,14 +461,16 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
// or fast refresh.
|
||||
pdfDocGenerationRef.current += 1;
|
||||
loadSeqRef.current += 1;
|
||||
parsePollAbortRef.current?.abort();
|
||||
parsePollAbortRef.current = null;
|
||||
parseStreamAbortRef.current?.abort();
|
||||
parseStreamAbortRef.current = null;
|
||||
parseSseCloseRef.current?.();
|
||||
parseSseCloseRef.current = null;
|
||||
pageTextCacheRef.current.clear();
|
||||
setPdfDocument(undefined);
|
||||
setCurrDocPages(undefined);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setCurrDocId(id);
|
||||
setCurrDocName(undefined);
|
||||
setCurrDocData(undefined);
|
||||
|
|
@ -489,7 +491,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
setParseStatus(initialParseStatus);
|
||||
setParseProgress(null);
|
||||
setActiveParseOpId(null);
|
||||
startParsedPolling(id, null);
|
||||
startParsedEventStream(id, null);
|
||||
void fetchDocumentSettings(id, controller.signal);
|
||||
}
|
||||
|
||||
|
|
@ -524,7 +526,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
setCurrDocText,
|
||||
setPdfDocument,
|
||||
fetchDocumentSettings,
|
||||
startParsedPolling,
|
||||
startParsedEventStream,
|
||||
]);
|
||||
|
||||
const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise<void> => {
|
||||
|
|
@ -546,14 +548,16 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
pageTextCacheRef.current.clear();
|
||||
setParsedDocument(null);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setParseStatus(forced.status);
|
||||
setParseProgress(null);
|
||||
setActiveParseOpId(forced.opId ?? null);
|
||||
startParsedPolling(currDocId, forced.opId ?? null);
|
||||
startParsedEventStream(currDocId, forced.opId ?? null);
|
||||
} catch (error) {
|
||||
console.error('Failed to force PDF reparse:', error);
|
||||
}
|
||||
}, [currDocId, startParsedPolling]);
|
||||
}, [currDocId, startParsedEventStream]);
|
||||
|
||||
/**
|
||||
* Clears the current document state
|
||||
|
|
@ -567,14 +571,15 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
docLoadSeqRef.current += 1;
|
||||
docLoadAbortRef.current?.abort();
|
||||
docLoadAbortRef.current = null;
|
||||
parsePollAbortRef.current?.abort();
|
||||
parsePollAbortRef.current = null;
|
||||
parseStreamAbortRef.current?.abort();
|
||||
parseStreamAbortRef.current = null;
|
||||
parseSseCloseRef.current?.();
|
||||
parseSseCloseRef.current = null;
|
||||
setCurrDocId(undefined);
|
||||
setCurrDocName(undefined);
|
||||
setCurrDocData(undefined);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
setCurrDocPages(undefined);
|
||||
setPdfDocument(undefined);
|
||||
setParsedDocument(null);
|
||||
|
|
@ -582,6 +587,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
setParseProgress(null);
|
||||
setActiveParseOpId(null);
|
||||
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
pageTextCacheRef.current.clear();
|
||||
stop();
|
||||
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
|
||||
|
|
@ -681,6 +687,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
parsedDocument,
|
||||
parseStatus,
|
||||
parseProgress,
|
||||
|
|
@ -708,6 +715,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
parsedDocument,
|
||||
parseStatus,
|
||||
parseProgress,
|
||||
|
|
|
|||
|
|
@ -5,10 +5,15 @@ import { documents } from '@/db/schema';
|
|||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker';
|
||||
import { isAbortLikeError } from '@/lib/server/compute/abort-like-error';
|
||||
import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/server/compute/worker-parse-state';
|
||||
import {
|
||||
isWorkerOperationStateStale,
|
||||
mergeNonReadyParseSnapshot,
|
||||
snapshotFromWorkerState,
|
||||
} from '@/lib/server/compute/worker-parse-state';
|
||||
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
|
|
@ -67,7 +72,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
|
|||
const state = await healStaleDocumentParseState({
|
||||
documentId: row.id,
|
||||
userId: row.userId,
|
||||
state: parseDocumentParseState(row.parseState),
|
||||
state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)),
|
||||
});
|
||||
const parseStatus = normalizeParseStatus(state.status);
|
||||
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
|
||||
|
|
@ -82,13 +87,20 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
|
|||
&& workerState.opId === opId
|
||||
&& !isWorkerOperationStateStale(workerState, opStaleMs)
|
||||
) {
|
||||
const merged = mergeNonReadyParseSnapshot({
|
||||
parseStatus,
|
||||
parseProgress: state.progress ?? null,
|
||||
workerState,
|
||||
});
|
||||
const workerSnapshot = snapshotFromWorkerState(workerState);
|
||||
const fromWorker = workerSnapshot.parseStatus === 'pending' || workerSnapshot.parseStatus === 'running';
|
||||
return {
|
||||
snapshot: {
|
||||
...snapshotFromWorkerState(workerState),
|
||||
...merged,
|
||||
opId: workerState.opId,
|
||||
},
|
||||
opId: workerState.opId,
|
||||
fromWorker: true,
|
||||
fromWorker,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -199,7 +211,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
|
||||
let initialState = await toSnapshotState(row, requestedOpId);
|
||||
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
|
||||
const state = parseDocumentParseState(row.parseState);
|
||||
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
const created = await backfillPendingPdfParseOperation({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
|
|
@ -261,22 +273,19 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
let current = initialState.snapshot;
|
||||
let signature = JSON.stringify(current);
|
||||
let currentOpId = requestedOpId ?? initialState.opId;
|
||||
let currentFromWorker = initialState.fromWorker;
|
||||
let lastEventId: number | null = null;
|
||||
let loggedMissingOpId = false;
|
||||
const pinnedRequestedOp = requestedOpId ?? null;
|
||||
const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot, fromWorker: boolean): boolean => {
|
||||
const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot): boolean => {
|
||||
const isTerminal = snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed';
|
||||
if (!isTerminal) return false;
|
||||
// If caller pinned an opId, keep streaming until worker confirms that
|
||||
// op state. DB fallback can report stale terminal status for other rows.
|
||||
if (pinnedRequestedOp && snapshot.opId === pinnedRequestedOp && !fromWorker) return false;
|
||||
if (pinnedRequestedOp && snapshot.opId !== pinnedRequestedOp) return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
writeSnapshot(current);
|
||||
|
||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
||||
if (shouldCloseForTerminalSnapshot(current)) {
|
||||
closeStream();
|
||||
return;
|
||||
}
|
||||
|
|
@ -303,7 +312,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
}
|
||||
current = next.snapshot;
|
||||
signature = next.signature;
|
||||
currentFromWorker = next.fromWorker;
|
||||
if (!requestedOpId && next.opId !== currentOpId) {
|
||||
currentOpId = next.opId;
|
||||
lastEventId = null;
|
||||
|
|
@ -312,7 +320,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
workerAbort = null;
|
||||
}
|
||||
}
|
||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
||||
if (shouldCloseForTerminalSnapshot(current)) {
|
||||
closeStream();
|
||||
}
|
||||
}).catch((error) => {
|
||||
|
|
@ -349,7 +357,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
}
|
||||
current = next.snapshot;
|
||||
signature = next.signature;
|
||||
currentFromWorker = next.fromWorker;
|
||||
currentOpId = requestedOpId ?? next.opId;
|
||||
if (!currentOpId) {
|
||||
if (!loggedMissingOpId) {
|
||||
|
|
@ -357,7 +364,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
logger.warn({
|
||||
event: 'documents.parsed.events.missing_opid_non_terminal',
|
||||
degraded: true,
|
||||
step: 'poll_without_worker_op',
|
||||
step: 'missing_opid_fallback',
|
||||
documentId: id,
|
||||
storageUserIdHash,
|
||||
parseStatus: current.parseStatus,
|
||||
|
|
@ -367,7 +374,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
} else if (loggedMissingOpId) {
|
||||
loggedMissingOpId = false;
|
||||
}
|
||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
||||
if (shouldCloseForTerminalSnapshot(current)) {
|
||||
closeStream();
|
||||
return;
|
||||
}
|
||||
|
|
@ -471,19 +478,23 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
);
|
||||
if (!workerSnapshot || workerSnapshot.opId !== currentOpId) continue;
|
||||
|
||||
const mergedSnapshot = mergeNonReadyParseSnapshot({
|
||||
parseStatus: current.parseStatus,
|
||||
parseProgress: current.parseProgress,
|
||||
workerState: workerSnapshot,
|
||||
});
|
||||
const nextSnapshot: ParsedSnapshot = {
|
||||
...snapshotFromWorkerState(workerSnapshot),
|
||||
...mergedSnapshot,
|
||||
opId: workerSnapshot.opId,
|
||||
};
|
||||
const nextSignature = JSON.stringify(nextSnapshot);
|
||||
if (nextSignature !== signature) {
|
||||
current = nextSnapshot;
|
||||
signature = nextSignature;
|
||||
currentFromWorker = true;
|
||||
writeSnapshot(current);
|
||||
}
|
||||
|
||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
||||
if (shouldCloseForTerminalSnapshot(current)) {
|
||||
closeStream();
|
||||
return;
|
||||
}
|
||||
|
|
@ -522,12 +533,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
}
|
||||
current = next.snapshot;
|
||||
signature = next.signature;
|
||||
currentFromWorker = next.fromWorker;
|
||||
if (!requestedOpId && next.opId !== currentOpId) {
|
||||
currentOpId = next.opId;
|
||||
lastEventId = null;
|
||||
}
|
||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
||||
if (shouldCloseForTerminalSnapshot(current)) {
|
||||
closeStream();
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,39 +4,36 @@ import { and, eq, inArray } from 'drizzle-orm';
|
|||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
|
||||
import {
|
||||
documentParseStateFromWorkerState,
|
||||
isWorkerOperationStateStale,
|
||||
snapshotFromWorkerState,
|
||||
} from '@/lib/server/compute/worker-parse-state';
|
||||
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
||||
import {
|
||||
documentKey,
|
||||
getParsedDocumentBlob,
|
||||
getParsedDocumentBlobByKey,
|
||||
isMissingBlobError,
|
||||
isValidDocumentId,
|
||||
putParsedDocumentBlob,
|
||||
} from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill';
|
||||
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||
import { checkJobRate, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||
import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger';
|
||||
import { createRequestLogger, hashForLog } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
import { logDegraded } from '@/lib/server/errors/logging';
|
||||
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
import type { PdfLayoutJobResult } from '@openreader/compute-core/api-contracts';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -100,102 +97,12 @@ async function writeParseRowState(input: {
|
|||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||
}
|
||||
|
||||
async function finalizeFromWorkerState(input: {
|
||||
workerState: WorkerOperationState<PdfLayoutJobResult>;
|
||||
row: ParseRow;
|
||||
namespace: string | null;
|
||||
logger: ServerLogger;
|
||||
}): Promise<NextResponse> {
|
||||
const snapshot = snapshotFromWorkerState(input.workerState);
|
||||
const workerParseState = documentParseStateFromWorkerState(input.workerState);
|
||||
|
||||
if (snapshot.parseStatus === 'pending' || snapshot.parseStatus === 'running') {
|
||||
await writeParseRowState({
|
||||
documentId: input.row.id,
|
||||
userId: input.row.userId,
|
||||
parseState: stringifyDocumentParseState(workerParseState),
|
||||
});
|
||||
return NextResponse.json({
|
||||
parseStatus: snapshot.parseStatus,
|
||||
parseProgress: snapshot.parseProgress,
|
||||
opId: input.workerState.opId,
|
||||
}, { status: 202 });
|
||||
}
|
||||
|
||||
if (snapshot.parseStatus === 'failed') {
|
||||
await writeParseRowState({
|
||||
documentId: input.row.id,
|
||||
userId: input.row.userId,
|
||||
parseState: stringifyDocumentParseState(workerParseState),
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
parseStatus: 'failed',
|
||||
parseProgress: null,
|
||||
opId: input.workerState.opId,
|
||||
error: input.workerState.error?.message ?? 'Worker parse failed',
|
||||
}, { status: 202 });
|
||||
}
|
||||
|
||||
let parsedJsonKey: string | null = null;
|
||||
if (input.workerState.result && 'parsedObjectKey' in input.workerState.result) {
|
||||
parsedJsonKey = typeof input.workerState.result.parsedObjectKey === 'string'
|
||||
? input.workerState.result.parsedObjectKey
|
||||
: null;
|
||||
}
|
||||
|
||||
if (!parsedJsonKey && input.workerState.result && 'parsed' in input.workerState.result && input.workerState.result.parsed) {
|
||||
const parsedJson = Buffer.from(JSON.stringify(input.workerState.result.parsed));
|
||||
parsedJsonKey = await putParsedDocumentBlob(input.row.id, parsedJson, input.namespace);
|
||||
}
|
||||
|
||||
if (!parsedJsonKey) {
|
||||
return errorResponse(new Error('Worker completed without parsed output'), {
|
||||
apiErrorMessage: 'Worker completed without parsed output',
|
||||
normalize: { code: 'DOCUMENTS_PARSED_WORKER_OUTPUT_MISSING', errorClass: 'upstream' },
|
||||
});
|
||||
}
|
||||
|
||||
await writeParseRowState({
|
||||
documentId: input.row.id,
|
||||
userId: input.row.userId,
|
||||
parseState: stringifyDocumentParseState(workerParseState),
|
||||
parsedJsonKey,
|
||||
});
|
||||
|
||||
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||
let parsedDoc: ParsedPdfDocument | null = null;
|
||||
try {
|
||||
parsedDoc = JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
|
||||
} catch {
|
||||
parsedDoc = null;
|
||||
}
|
||||
|
||||
if (!hasAnyParsedBlocks(parsedDoc)) {
|
||||
input.logger.warn({
|
||||
event: 'documents.parsed.no_blocks_in_output',
|
||||
documentId: input.row.id,
|
||||
userIdHash: hashForLog(input.row.userId),
|
||||
parsedJsonKey,
|
||||
}, 'Worker output parsed successfully but contained no blocks');
|
||||
}
|
||||
|
||||
return new NextResponse(new Uint8Array(json), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||
const { logger } = createRequestLogger({
|
||||
route: '/api/documents/[id]/parsed',
|
||||
request: req,
|
||||
});
|
||||
try {
|
||||
const opStaleMs = getComputeOpStaleMs();
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const authCtxOrRes = await requireAuthContext(req);
|
||||
|
|
@ -204,8 +111,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
|
||||
const params = await ctx.params;
|
||||
const id = (params.id || '').trim().toLowerCase();
|
||||
const retryFailed = req.nextUrl.searchParams.get('retry') === '1';
|
||||
const requestedOpId = normalizeOpId(req.nextUrl.searchParams.get('opId'));
|
||||
if (!isValidDocumentId(id)) {
|
||||
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
||||
}
|
||||
|
|
@ -220,112 +125,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
if (requestedOpId) {
|
||||
const workerState = await fetchWorkerOperationState<PdfLayoutJobResult>(requestedOpId);
|
||||
if (workerState && workerState.opId === requestedOpId) {
|
||||
return finalizeFromWorkerState({
|
||||
workerState,
|
||||
row,
|
||||
namespace: testNamespace,
|
||||
logger,
|
||||
});
|
||||
}
|
||||
logDegraded(logger, {
|
||||
event: 'documents.parsed.requested_op_unavailable',
|
||||
msg: 'Requested worker operation id was unavailable',
|
||||
step: 'requested_op_lookup',
|
||||
context: {
|
||||
documentId: id,
|
||||
userIdHash: hashForLog(row.userId),
|
||||
opId: requestedOpId,
|
||||
error: {
|
||||
name: 'RequestedOperationUnavailable',
|
||||
message: `Requested worker operation ${requestedOpId} was unavailable`,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
let state = parseDocumentParseState(row.parseState);
|
||||
state = await healStaleDocumentParseState({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
state,
|
||||
});
|
||||
|
||||
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
const effectiveStatus = normalizeParseStatus(state.status);
|
||||
const effectiveProgress = state.progress ?? null;
|
||||
const effectiveOpId = normalizeOpId(state.opId);
|
||||
|
||||
if (effectiveOpId && effectiveStatus !== 'ready') {
|
||||
const workerState = await fetchWorkerOperationState<PdfLayoutJobResult>(effectiveOpId);
|
||||
if (
|
||||
workerState
|
||||
&& workerState.opId === effectiveOpId
|
||||
&& !isWorkerOperationStateStale(workerState, opStaleMs)
|
||||
) {
|
||||
return finalizeFromWorkerState({
|
||||
workerState,
|
||||
row,
|
||||
namespace: testNamespace,
|
||||
logger,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!effectiveOpId && (effectiveStatus === 'pending' || effectiveStatus === 'running')) {
|
||||
const created = await backfillPendingPdfParseOperation({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
namespace: testNamespace,
|
||||
state,
|
||||
});
|
||||
if (created) {
|
||||
const snapshot = snapshotFromWorkerState(created);
|
||||
await writeParseRowState({
|
||||
documentId: row.id,
|
||||
userId: row.userId,
|
||||
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
|
||||
});
|
||||
return NextResponse.json({
|
||||
parseStatus: snapshot.parseStatus,
|
||||
parseProgress: snapshot.parseProgress,
|
||||
opId: created.opId,
|
||||
}, { status: 202 });
|
||||
}
|
||||
}
|
||||
|
||||
if (effectiveStatus === 'failed' && retryFailed) {
|
||||
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
||||
const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig);
|
||||
if (!rateDecision.allowed) {
|
||||
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
|
||||
}
|
||||
const created = await createOrReusePdfWorkerOperation({
|
||||
documentId: id,
|
||||
namespace: testNamespace,
|
||||
documentObjectKey: documentKey(id, testNamespace),
|
||||
});
|
||||
await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig);
|
||||
const snapshot = snapshotFromWorkerState(created);
|
||||
await writeParseRowState({
|
||||
documentId: row.id,
|
||||
userId: row.userId,
|
||||
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
|
||||
});
|
||||
return NextResponse.json({
|
||||
parseStatus: snapshot.parseStatus,
|
||||
parseProgress: snapshot.parseProgress,
|
||||
opId: created.opId,
|
||||
}, { status: 202 });
|
||||
}
|
||||
|
||||
if (effectiveStatus !== 'ready') {
|
||||
return NextResponse.json({
|
||||
parseStatus: effectiveStatus,
|
||||
parseProgress: effectiveProgress,
|
||||
opId: effectiveOpId,
|
||||
}, { status: 202 });
|
||||
}, { status: 409 });
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
@ -409,7 +219,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
let state = parseDocumentParseState(row.parseState);
|
||||
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
state = await healStaleDocumentParseState({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
|
|
@ -441,25 +251,33 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
|
||||
}
|
||||
|
||||
const created = await createOrReusePdfWorkerOperation({
|
||||
const forceToken = randomUUID();
|
||||
const startedParse = await startPdfParseOperation({
|
||||
documentId: id,
|
||||
userId: authCtxOrRes.userId,
|
||||
namespace: testNamespace,
|
||||
documentObjectKey: documentKey(id, testNamespace),
|
||||
forceToken: randomUUID(),
|
||||
forceToken,
|
||||
});
|
||||
await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig);
|
||||
|
||||
const snapshot = snapshotFromWorkerState(created);
|
||||
const snapshot = snapshotFromWorkerState(startedParse.workerState);
|
||||
await writeParseRowState({
|
||||
documentId: row.id,
|
||||
userId: row.userId,
|
||||
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
|
||||
parseState: stringifyDocumentParseState(startedParse.parseState),
|
||||
});
|
||||
enqueueParsePdfJob({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
namespace: testNamespace,
|
||||
forceToken,
|
||||
initialOpId: startedParse.workerState.opId,
|
||||
initialJobId: startedParse.workerState.jobId,
|
||||
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
parseStatus: snapshot.parseStatus,
|
||||
parseProgress: snapshot.parseProgress,
|
||||
opId: created.opId,
|
||||
opId: startedParse.workerState.opId,
|
||||
}, { status: 202 });
|
||||
} catch (error) {
|
||||
return errorResponse(error, {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { isValidTempUploadToken, putTempDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
|
|
@ -26,11 +26,12 @@ export async function PUT(req: NextRequest) {
|
|||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
const url = new URL(req.url);
|
||||
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
|
||||
if (!isValidDocumentId(id)) {
|
||||
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
||||
const token = (url.searchParams.get('token') || '').trim().toLowerCase();
|
||||
if (!isValidTempUploadToken(token)) {
|
||||
return NextResponse.json({ error: 'Invalid upload token' }, { status: 400 });
|
||||
}
|
||||
|
||||
const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream';
|
||||
|
|
@ -86,7 +87,7 @@ export async function PUT(req: NextRequest) {
|
|||
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||
|
||||
try {
|
||||
await putDocumentBlob(id, body, contentType, namespace);
|
||||
await putTempDocumentBlob(token, ctxOrRes.userId, body, contentType, namespace);
|
||||
} catch (error) {
|
||||
if (!isPreconditionFailed(error)) {
|
||||
throw error;
|
||||
|
|
@ -97,12 +98,12 @@ export async function PUT(req: NextRequest) {
|
|||
event: 'documents.blob.upload.fallback.proxy_used',
|
||||
degraded: true,
|
||||
fallbackPath: 'upload_proxy',
|
||||
documentId: id,
|
||||
uploadToken: token,
|
||||
contentType,
|
||||
bytes: body.byteLength,
|
||||
}, 'Document upload fallback proxy used');
|
||||
|
||||
return NextResponse.json({ success: true, id });
|
||||
return NextResponse.json({ success: true, token });
|
||||
} catch (error) {
|
||||
serverLogger.error({
|
||||
event: 'documents.blob.upload.fallback.failed',
|
||||
|
|
|
|||
257
src/app/api/documents/blob/upload/finalize/route.ts
Normal file
257
src/app/api/documents/blob/upload/finalize/route.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import path from 'path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import {
|
||||
TEMP_DOCUMENT_UPLOAD_TTL_MS,
|
||||
copyTempDocumentBlobToDocument,
|
||||
deleteTempDocumentUpload,
|
||||
getTempDocumentBlob,
|
||||
getTempDocumentFinalizeReceipt,
|
||||
headDocumentBlob,
|
||||
headTempDocumentBlob,
|
||||
isMissingBlobError,
|
||||
isPreconditionFailed,
|
||||
isValidTempUploadToken,
|
||||
putDocumentBlob,
|
||||
putTempDocumentFinalizeReceipt,
|
||||
} from '@/lib/server/documents/blobstore';
|
||||
import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert';
|
||||
import { registerUploadedDocument } from '@/lib/server/documents/register-upload';
|
||||
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type FinalizeUpload = {
|
||||
token: string;
|
||||
name: string;
|
||||
type: DocumentType;
|
||||
lastModified: number;
|
||||
};
|
||||
|
||||
type FinalizeReceipt = {
|
||||
stored: BaseDocument;
|
||||
};
|
||||
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType {
|
||||
if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') {
|
||||
return rawType;
|
||||
}
|
||||
return toDocumentTypeFromName(safeName);
|
||||
}
|
||||
|
||||
function normalizeLastModified(value: unknown): number {
|
||||
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
|
||||
}
|
||||
|
||||
function parseFinalizePayload(body: unknown): FinalizeUpload[] {
|
||||
if (!body || typeof body !== 'object') return [];
|
||||
const rawUploads = (body as { uploads?: unknown }).uploads;
|
||||
if (!Array.isArray(rawUploads)) return [];
|
||||
|
||||
const uploads: FinalizeUpload[] = [];
|
||||
for (const rawUpload of rawUploads) {
|
||||
if (!rawUpload || typeof rawUpload !== 'object') continue;
|
||||
const rec = rawUpload as Record<string, unknown>;
|
||||
const token = typeof rec.token === 'string' ? rec.token.trim().toLowerCase() : '';
|
||||
if (!isValidTempUploadToken(token)) continue;
|
||||
const fallbackName = `upload-${token}.txt`;
|
||||
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
|
||||
uploads.push({
|
||||
token,
|
||||
name,
|
||||
type: normalizeDocumentType(rec.type, name),
|
||||
lastModified: normalizeLastModified(rec.lastModified),
|
||||
});
|
||||
}
|
||||
return uploads;
|
||||
}
|
||||
|
||||
async function loadTempUpload(input: {
|
||||
token: string;
|
||||
userId: string;
|
||||
namespace: string | null;
|
||||
}): Promise<{ contentType: string; size: number; lastModified: number; body: Buffer }> {
|
||||
const RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 500;
|
||||
|
||||
let lastError: unknown = null;
|
||||
for (let attempt = 0; attempt < RETRIES; attempt += 1) {
|
||||
try {
|
||||
const head = await headTempDocumentBlob(input.token, input.userId, input.namespace);
|
||||
const body = await getTempDocumentBlob(input.token, input.userId, input.namespace);
|
||||
return {
|
||||
contentType: head.contentType ?? 'application/octet-stream',
|
||||
size: head.contentLength > 0 ? head.contentLength : body.byteLength,
|
||||
lastModified: head.lastModified ?? Date.now(),
|
||||
body,
|
||||
};
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (isMissingBlobError(error) && attempt < RETRIES - 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error('Temporary upload is unavailable');
|
||||
}
|
||||
|
||||
async function finalizeOne(input: {
|
||||
upload: FinalizeUpload;
|
||||
userId: string;
|
||||
namespace: string | null;
|
||||
}): Promise<BaseDocument> {
|
||||
const existingReceipt = await getTempDocumentFinalizeReceipt<FinalizeReceipt>(
|
||||
input.upload.token,
|
||||
input.userId,
|
||||
input.namespace,
|
||||
);
|
||||
if (existingReceipt?.stored) {
|
||||
return existingReceipt.stored;
|
||||
}
|
||||
|
||||
const temp = await loadTempUpload({
|
||||
token: input.upload.token,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
});
|
||||
|
||||
if (Date.now() - temp.lastModified > TEMP_DOCUMENT_UPLOAD_TTL_MS) {
|
||||
await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch(() => undefined);
|
||||
throw new Error('Temporary upload expired before finalize');
|
||||
}
|
||||
|
||||
const isDocxUpload = input.upload.type === 'docx';
|
||||
const finalizedType: DocumentType = isDocxUpload ? 'pdf' : input.upload.type;
|
||||
const finalizedBody = isDocxUpload
|
||||
? await convertDocxBufferToPdfBuffer(temp.body)
|
||||
: temp.body;
|
||||
const finalizedContentType = finalizedType === 'pdf'
|
||||
? 'application/pdf'
|
||||
: temp.contentType;
|
||||
const finalizedName = finalizedType === 'pdf' && input.upload.type === 'docx'
|
||||
? safeDocumentName(`${path.parse(input.upload.name).name}.pdf`, 'upload.pdf')
|
||||
: input.upload.name;
|
||||
const documentId = createHash('sha256').update(finalizedBody).digest('hex');
|
||||
|
||||
try {
|
||||
await headDocumentBlob(documentId, input.namespace);
|
||||
} catch (error) {
|
||||
if (!isMissingBlobError(error)) throw error;
|
||||
if (!isDocxUpload) {
|
||||
try {
|
||||
await copyTempDocumentBlobToDocument(
|
||||
input.upload.token,
|
||||
input.userId,
|
||||
documentId,
|
||||
input.namespace,
|
||||
finalizedContentType,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
} catch (copyError) {
|
||||
if (!isPreconditionFailed(copyError)) throw copyError;
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await putDocumentBlob(
|
||||
documentId,
|
||||
finalizedBody,
|
||||
finalizedContentType,
|
||||
input.namespace,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
} catch (putError) {
|
||||
if (!isPreconditionFailed(putError)) throw putError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const canonicalHead = await headDocumentBlob(documentId, input.namespace);
|
||||
const stored = await registerUploadedDocument({
|
||||
documentId,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
name: finalizedName,
|
||||
type: finalizedType,
|
||||
size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength,
|
||||
lastModified: input.upload.lastModified,
|
||||
});
|
||||
|
||||
await putTempDocumentFinalizeReceipt(
|
||||
input.upload.token,
|
||||
input.userId,
|
||||
input.namespace,
|
||||
Buffer.from(JSON.stringify({ stored }), 'utf8'),
|
||||
);
|
||||
|
||||
await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch((error) => {
|
||||
serverLogger.warn({
|
||||
event: 'documents.blob.upload.finalize.temp_delete_failed',
|
||||
degraded: true,
|
||||
fallbackPath: 'leave_temp_upload',
|
||||
documentId,
|
||||
token: input.upload.token,
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to delete temp upload after finalize');
|
||||
});
|
||||
|
||||
return stored;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
const userId = ctxOrRes.userId;
|
||||
|
||||
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||
const uploads = parseFinalizePayload(await req.json().catch(() => null));
|
||||
if (uploads.length === 0) {
|
||||
return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 });
|
||||
}
|
||||
|
||||
const stored = await Promise.all(
|
||||
uploads.map((upload) => finalizeOne({
|
||||
upload,
|
||||
userId,
|
||||
namespace,
|
||||
})),
|
||||
);
|
||||
|
||||
return NextResponse.json({ stored });
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'Temporary upload expired before finalize') {
|
||||
return NextResponse.json({ error: error.message }, { status: 410 });
|
||||
}
|
||||
if (isMissingBlobError(error)) {
|
||||
return NextResponse.json({ error: 'Temporary upload missing. Upload bytes again and retry finalize.' }, { status: 409 });
|
||||
}
|
||||
|
||||
serverLogger.error({
|
||||
event: 'documents.blob.upload.finalize.failed',
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to finalize uploaded documents');
|
||||
return errorResponse(error, {
|
||||
apiErrorMessage: 'Failed to finalize uploaded documents',
|
||||
normalize: { code: 'DOCUMENTS_BLOB_UPLOAD_FINALIZE_FAILED', errorClass: 'storage' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,11 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
TEMP_DOCUMENT_UPLOAD_TTL_MS,
|
||||
deleteExpiredTempDocumentUploads,
|
||||
presignTempPut,
|
||||
} from '@/lib/server/documents/blobstore';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
|
|
@ -10,7 +15,6 @@ import { errorResponse } from '@/lib/server/errors/next-response';
|
|||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type PresignUpload = {
|
||||
id: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
};
|
||||
|
|
@ -24,14 +28,12 @@ function parseUploads(body: unknown): PresignUpload[] {
|
|||
for (const raw of rawUploads) {
|
||||
if (!raw || typeof raw !== 'object') continue;
|
||||
const rec = raw as Record<string, unknown>;
|
||||
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
|
||||
if (!isValidDocumentId(id)) continue;
|
||||
const contentType =
|
||||
typeof rec.contentType === 'string' && rec.contentType.trim()
|
||||
? rec.contentType.trim()
|
||||
: 'application/octet-stream';
|
||||
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
||||
uploads.push({ id, contentType, size });
|
||||
uploads.push({ contentType, size });
|
||||
}
|
||||
return uploads;
|
||||
}
|
||||
|
|
@ -47,6 +49,8 @@ export async function POST(req: NextRequest) {
|
|||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
const userId = ctxOrRes.userId;
|
||||
|
||||
const body = await req.json().catch(() => null);
|
||||
const uploads = parseUploads(body);
|
||||
|
|
@ -68,13 +72,16 @@ export async function POST(req: NextRequest) {
|
|||
}
|
||||
|
||||
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||
await deleteExpiredTempDocumentUploads(userId, namespace, Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS)
|
||||
.catch(() => undefined);
|
||||
const signed = await Promise.all(
|
||||
uploads.map(async (upload) => {
|
||||
const res = await presignPut(upload.id, upload.contentType, namespace, {
|
||||
const token = randomUUID();
|
||||
const res = await presignTempPut(token, userId, upload.contentType, namespace, {
|
||||
contentLength: upload.size,
|
||||
});
|
||||
return {
|
||||
id: upload.id,
|
||||
token,
|
||||
url: res.url,
|
||||
headers: res.headers,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,209 +0,0 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { writeFile, mkdir, readFile, readdir, rm, stat } from 'fs/promises';
|
||||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { randomUUID, createHash } from 'crypto';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { safeDocumentName } from '@/lib/server/documents/utils';
|
||||
import { enqueueDocumentPreview } from '@/lib/server/documents/previews';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||
import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||
|
||||
async function ensureTempDir() {
|
||||
if (!existsSync(DOCSTORE_DIR)) {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
}
|
||||
if (!existsSync(TEMP_DIR)) {
|
||||
await mkdir(TEMP_DIR, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args: string[] = [];
|
||||
if (profileDir) {
|
||||
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
|
||||
}
|
||||
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
|
||||
const proc = spawn('soffice', args);
|
||||
|
||||
proc.on('error', (error) => reject(error));
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100): Promise<string> {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) {
|
||||
const files = await readdir(dir);
|
||||
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
|
||||
if (pdf) {
|
||||
const pdfPath = path.join(dir, pdf);
|
||||
try {
|
||||
const first = await stat(pdfPath);
|
||||
await new Promise((res) => setTimeout(res, intervalMs));
|
||||
const second = await stat(pdfPath);
|
||||
if (second.size > 0 && second.size === first.size) {
|
||||
return pdfPath;
|
||||
}
|
||||
} catch {
|
||||
// ignore transient errors
|
||||
}
|
||||
}
|
||||
await new Promise((res) => setTimeout(res, intervalMs));
|
||||
}
|
||||
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
const storageUserId = ctxOrRes.userId;
|
||||
|
||||
await ensureTempDir();
|
||||
|
||||
const formData = await req.formData();
|
||||
const file = formData.get('file');
|
||||
if (!(file instanceof File)) {
|
||||
return NextResponse.json({ error: 'No file provided' }, { status: 400 });
|
||||
}
|
||||
|
||||
if (!file.name.toLowerCase().endsWith('.docx')) {
|
||||
return NextResponse.json({ error: 'File must be a .docx document' }, { status: 400 });
|
||||
}
|
||||
|
||||
const docxBytes = Buffer.from(await file.arrayBuffer());
|
||||
// Keep stable IDs tied to source bytes.
|
||||
const id = createHash('sha256').update(docxBytes).digest('hex');
|
||||
|
||||
const tempId = randomUUID();
|
||||
const jobDir = path.join(TEMP_DIR, tempId);
|
||||
await mkdir(jobDir, { recursive: true });
|
||||
const profileDir = path.join(jobDir, 'lo-profile');
|
||||
await mkdir(profileDir, { recursive: true });
|
||||
const inputPath = path.join(jobDir, 'input.docx');
|
||||
|
||||
await writeFile(inputPath, docxBytes);
|
||||
|
||||
try {
|
||||
await convertDocxToPdf(inputPath, jobDir, profileDir);
|
||||
const pdfPath = await waitForPdfReady(jobDir);
|
||||
const pdfContent = await readFile(pdfPath);
|
||||
|
||||
try {
|
||||
await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace);
|
||||
} catch (error) {
|
||||
// Idempotent behavior: if blob already exists for this sha, continue.
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined;
|
||||
const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed';
|
||||
if (!isPreconditionFailed) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
|
||||
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
id,
|
||||
userId: storageUserId,
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
||||
parsedJsonKey: null,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
set: {
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
||||
parsedJsonKey: null,
|
||||
},
|
||||
});
|
||||
|
||||
await enqueueDocumentPreview(
|
||||
{
|
||||
id,
|
||||
type: 'pdf',
|
||||
lastModified,
|
||||
},
|
||||
testNamespace,
|
||||
).catch((error) => {
|
||||
serverLogger.warn({
|
||||
event: 'documents.docx.preview_enqueue.failed',
|
||||
degraded: true,
|
||||
fallbackPath: 'skip_preview_enqueue',
|
||||
documentId: id,
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to enqueue preview for converted DOCX');
|
||||
});
|
||||
|
||||
// Record upload-driven parse load (see register route for rationale).
|
||||
const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
||||
await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig);
|
||||
enqueueParsePdfJob({
|
||||
documentId: id,
|
||||
userId: storageUserId,
|
||||
namespace: testNamespace,
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
stored: {
|
||||
id,
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
} catch (error) {
|
||||
serverLogger.error({
|
||||
event: 'documents.docx.convert_upload.failed',
|
||||
error: errorToLog(error),
|
||||
}, 'Failed converting/uploading DOCX');
|
||||
return errorResponse(error, {
|
||||
apiErrorMessage: 'Failed to convert document',
|
||||
normalize: { code: 'DOCUMENTS_DOCX_CONVERT_UPLOAD_FAILED', errorClass: 'upstream' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +1,20 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { and, count, eq, inArray } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
||||
import { toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
import {
|
||||
cleanupDocumentPreviewArtifacts,
|
||||
deleteDocumentPreviewRows,
|
||||
enqueueDocumentPreview,
|
||||
} from '@/lib/server/documents/previews';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||
import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
|
|
@ -27,14 +22,6 @@ import type { BaseDocument, DocumentType } from '@/types/documents';
|
|||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type RegisterDocument = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: DocumentType;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
};
|
||||
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
|
|
@ -49,175 +36,6 @@ function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType
|
|||
return toDocumentTypeFromName(safeName);
|
||||
}
|
||||
|
||||
function normalizeLastModified(value: unknown): number {
|
||||
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
|
||||
}
|
||||
|
||||
function parseDocumentPayload(body: unknown): RegisterDocument[] {
|
||||
if (!body || typeof body !== 'object') return [];
|
||||
const rawDocs = (body as { documents?: unknown }).documents;
|
||||
if (!Array.isArray(rawDocs)) return [];
|
||||
|
||||
const docs: RegisterDocument[] = [];
|
||||
for (const rawDoc of rawDocs) {
|
||||
if (!rawDoc || typeof rawDoc !== 'object') continue;
|
||||
const rec = rawDoc as Record<string, unknown>;
|
||||
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
|
||||
if (!isValidDocumentId(id)) continue;
|
||||
const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`;
|
||||
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
|
||||
const type = normalizeDocumentType(rec.type, name);
|
||||
const lastModified = normalizeLastModified(rec.lastModified);
|
||||
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
||||
docs.push({ id, name, type, size, lastModified });
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const storageUserId = ctxOrRes.userId;
|
||||
|
||||
const body = await req.json().catch(() => null);
|
||||
const documentsData = parseDocumentPayload(body);
|
||||
if (documentsData.length === 0) {
|
||||
return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 });
|
||||
}
|
||||
|
||||
const stored: BaseDocument[] = [];
|
||||
|
||||
// Resolve the parse rate-limit config once (only when a PDF is present).
|
||||
const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf')
|
||||
? getPdfLayoutRateConfig(await getResolvedRuntimeConfig())
|
||||
: null;
|
||||
|
||||
for (const doc of documentsData) {
|
||||
let headSize = doc.size;
|
||||
|
||||
// Retry HEAD check to handle S3 read-after-write propagation delays.
|
||||
// The client uploads bytes directly to S3 via presigned URL, then
|
||||
// immediately calls this endpoint. On serverless platforms the HEAD
|
||||
// request may reach S3 before the PUT is visible.
|
||||
const HEAD_RETRIES = 3;
|
||||
const HEAD_RETRY_DELAY_MS = 500;
|
||||
let headError: unknown = null;
|
||||
for (let attempt = 0; attempt < HEAD_RETRIES; attempt++) {
|
||||
headError = null;
|
||||
try {
|
||||
const head = await headDocumentBlob(doc.id, testNamespace);
|
||||
if (head.contentLength > 0) headSize = head.contentLength;
|
||||
break;
|
||||
} catch (error) {
|
||||
if (isMissingBlobError(error)) {
|
||||
headError = error;
|
||||
if (attempt < HEAD_RETRIES - 1) {
|
||||
await new Promise((r) => setTimeout(r, HEAD_RETRY_DELAY_MS));
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (headError && isMissingBlobError(headError)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`,
|
||||
},
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
id: doc.id,
|
||||
userId: storageUserId,
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: headSize,
|
||||
lastModified: doc.lastModified,
|
||||
filePath: doc.id,
|
||||
parseState: doc.type === 'pdf'
|
||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
||||
: null,
|
||||
parsedJsonKey: null,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
set: {
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: headSize,
|
||||
lastModified: doc.lastModified,
|
||||
filePath: doc.id,
|
||||
parseState: doc.type === 'pdf'
|
||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
||||
: null,
|
||||
parsedJsonKey: null,
|
||||
},
|
||||
});
|
||||
|
||||
stored.push({
|
||||
id: doc.id,
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: headSize,
|
||||
lastModified: doc.lastModified,
|
||||
scope: 'user',
|
||||
});
|
||||
|
||||
await enqueueDocumentPreview(
|
||||
{
|
||||
id: doc.id,
|
||||
type: doc.type,
|
||||
lastModified: doc.lastModified,
|
||||
},
|
||||
testNamespace,
|
||||
).catch((error) => {
|
||||
serverLogger.warn({
|
||||
event: 'documents.preview.enqueue.failed',
|
||||
degraded: true,
|
||||
fallbackPath: 'skip_preview_enqueue',
|
||||
documentId: doc.id,
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to enqueue document preview');
|
||||
});
|
||||
|
||||
if (doc.type === 'pdf') {
|
||||
// Account for upload-driven parse load in the same ledger the explicit
|
||||
// re-parse limiter reads. We record (not reject) here so a legitimate
|
||||
// bulk upload always parses; the recorded load still throttles
|
||||
// subsequent loopable re-parse spam via /parsed.
|
||||
await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] });
|
||||
enqueueParsePdfJob({
|
||||
documentId: doc.id,
|
||||
userId: storageUserId,
|
||||
namespace: testNamespace,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({ success: true, stored });
|
||||
} catch (error) {
|
||||
serverLogger.error({
|
||||
event: 'documents.register.failed',
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to register documents');
|
||||
return errorResponse(error, {
|
||||
apiErrorMessage: 'Failed to register documents',
|
||||
normalize: { code: 'DOCUMENTS_REGISTER_FAILED', errorClass: 'db' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
|
@ -260,13 +78,16 @@ export async function GET(req: NextRequest) {
|
|||
|
||||
const results: BaseDocument[] = rows.map((doc) => {
|
||||
const type = normalizeDocumentType(doc.type, doc.name);
|
||||
const parseState = type === 'pdf'
|
||||
? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState))
|
||||
: null;
|
||||
return {
|
||||
id: doc.id,
|
||||
name: doc.name,
|
||||
size: Number(doc.size),
|
||||
lastModified: Number(doc.lastModified),
|
||||
type,
|
||||
parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null,
|
||||
parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null,
|
||||
parsedJsonKey: doc.parsedJsonKey,
|
||||
scope: 'user',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -122,16 +122,14 @@ interface DocumentListInnerProps {
|
|||
function SidebarUploadLoader({
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase,
|
||||
currentFileName,
|
||||
}: {
|
||||
totalFiles: number;
|
||||
completedFiles: number;
|
||||
phase: 'uploading' | 'converting';
|
||||
phase: 'uploading';
|
||||
currentFileName: string | null;
|
||||
}) {
|
||||
const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0;
|
||||
const label = phase === 'converting' ? 'Converting' : 'Uploading';
|
||||
const radius = 7;
|
||||
const stroke = 2;
|
||||
const size = 18;
|
||||
|
|
@ -143,12 +141,12 @@ function SidebarUploadLoader({
|
|||
<div className="rounded-md border border-line bg-surface-sunken px-2 py-1.5">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="min-w-0 flex items-center gap-1.5 text-[11px] leading-tight">
|
||||
<span className="font-medium text-foreground">{label}</span>
|
||||
<span className="font-medium text-foreground">Uploading</span>
|
||||
<span className="shrink-0 tabular-nums text-soft">{completedFiles}/{totalFiles}</span>
|
||||
</div>
|
||||
<div className="shrink-0 flex items-center gap-1 text-accent" aria-label={`Upload progress ${progress}%`}>
|
||||
<span className="text-[10px] tabular-nums text-soft">{progress}%</span>
|
||||
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`} className={phase === 'converting' ? 'animate-spin' : ''}>
|
||||
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`}>
|
||||
<circle
|
||||
cx={size / 2}
|
||||
cy={size / 2}
|
||||
|
|
@ -228,13 +226,11 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
|
||||
const {
|
||||
pdfDocs,
|
||||
removePDFDocument,
|
||||
isPDFLoading,
|
||||
epubDocs,
|
||||
removeEPUBDocument,
|
||||
isEPUBLoading,
|
||||
htmlDocs,
|
||||
removeHTMLDocument,
|
||||
deleteDocument,
|
||||
isHTMLLoading,
|
||||
} = useDocuments();
|
||||
|
||||
|
|
@ -436,9 +432,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
const handleDelete = useCallback(async () => {
|
||||
if (!documentToDelete) return;
|
||||
try {
|
||||
if (documentToDelete.type === 'pdf') await removePDFDocument(documentToDelete.id);
|
||||
else if (documentToDelete.type === 'epub') await removeEPUBDocument(documentToDelete.id);
|
||||
else if (documentToDelete.type === 'html') await removeHTMLDocument(documentToDelete.id);
|
||||
await deleteDocument(documentToDelete.id);
|
||||
setFolders((prev) =>
|
||||
prev.map((f) => ({
|
||||
...f,
|
||||
|
|
@ -451,7 +445,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
} catch (err) {
|
||||
console.error('Failed to remove document:', err);
|
||||
}
|
||||
}, [documentToDelete, removePDFDocument, removeEPUBDocument, removeHTMLDocument]);
|
||||
}, [deleteDocument, documentToDelete]);
|
||||
|
||||
const handleDeleteDoc = useCallback((doc: DocumentListDocument) => {
|
||||
setDocumentToDelete({ id: doc.id, name: doc.name, type: doc.type });
|
||||
|
|
@ -577,10 +571,8 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
if (batches.length === 0) return null;
|
||||
const totalFiles = batches.reduce((sum, batch) => sum + batch.totalFiles, 0);
|
||||
const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0);
|
||||
const convertingBatch = batches.find((batch) => batch.phase === 'converting');
|
||||
const phase: 'uploading' | 'converting' = convertingBatch ? 'converting' : 'uploading';
|
||||
const currentFileName = convertingBatch?.currentFileName ?? batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
|
||||
return { totalFiles, completedFiles, phase, currentFileName };
|
||||
const currentFileName = batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
|
||||
return { totalFiles, completedFiles, phase: 'uploading' as const, currentFileName };
|
||||
}, [activeUploadBatches]);
|
||||
|
||||
const fallbackViewMode: ViewMode = viewMode;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import { useState, useCallback, useId, type ReactNode } from 'react';
|
|||
import { useDropzone } from 'react-dropzone';
|
||||
import { UploadIcon } from '@/components/icons/Icons';
|
||||
import { useDocuments } from '@/contexts/DocumentContext';
|
||||
import { uploadDocxAsPdf } from '@/lib/client/api/documents';
|
||||
import { useFeatureFlag } from '@/contexts/RuntimeConfigContext';
|
||||
import { dropzoneSurfaceClass } from '@/components/ui';
|
||||
|
||||
|
|
@ -20,7 +19,7 @@ export interface UploadBatchState {
|
|||
isActive: boolean;
|
||||
totalFiles: number;
|
||||
completedFiles: number;
|
||||
phase: 'uploading' | 'converting';
|
||||
phase: 'uploading';
|
||||
currentFileName: string | null;
|
||||
}
|
||||
|
||||
|
|
@ -33,13 +32,9 @@ export function DocumentUploader({
|
|||
const uploaderId = useId();
|
||||
const enableDocx = useFeatureFlag('enableDocxConversion');
|
||||
const {
|
||||
addPDFDocument: addPDF,
|
||||
addEPUBDocument: addEPUB,
|
||||
addHTMLDocument: addHTML,
|
||||
refreshDocuments,
|
||||
uploadDocuments,
|
||||
} = useDocuments();
|
||||
const [isUploading, setIsUploading] = useState(false);
|
||||
const [isConverting, setIsConverting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const emitBatchState = useCallback((state: Omit<UploadBatchState, 'uploaderId'>) => {
|
||||
|
|
@ -63,72 +58,13 @@ export function DocumentUploader({
|
|||
});
|
||||
|
||||
try {
|
||||
for (const file of acceptedFiles) {
|
||||
if (file.type === 'application/pdf') {
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: file.name,
|
||||
});
|
||||
await addPDF(file);
|
||||
completedFiles += 1;
|
||||
} else if (file.type === 'application/epub+zip') {
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: file.name,
|
||||
});
|
||||
await addEPUB(file);
|
||||
completedFiles += 1;
|
||||
} else if (file.type === 'text/plain' || file.type === 'text/markdown' || file.name.endsWith('.md')) {
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: file.name,
|
||||
});
|
||||
await addHTML(file);
|
||||
completedFiles += 1;
|
||||
} else if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
|
||||
// Preserve prior UX: show "Converting DOCX..." state rather than generic uploading.
|
||||
setIsUploading(false);
|
||||
setIsConverting(true);
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'converting',
|
||||
currentFileName: file.name,
|
||||
});
|
||||
// Convert+upload directly on the server. Use sha(docx) as stable ID to avoid duplicates.
|
||||
await uploadDocxAsPdf(file);
|
||||
await refreshDocuments();
|
||||
setIsConverting(false);
|
||||
setIsUploading(true);
|
||||
completedFiles += 1;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: null,
|
||||
});
|
||||
}
|
||||
await uploadDocuments(acceptedFiles);
|
||||
completedFiles = acceptedFiles.length;
|
||||
} catch (err) {
|
||||
setError('Failed to upload file. Please try again.');
|
||||
console.error('Upload error:', err);
|
||||
} finally {
|
||||
setIsUploading(false);
|
||||
setIsConverting(false);
|
||||
emitBatchState({
|
||||
isActive: false,
|
||||
totalFiles,
|
||||
|
|
@ -137,7 +73,7 @@ export function DocumentUploader({
|
|||
currentFileName: null,
|
||||
});
|
||||
}
|
||||
}, [addHTML, addPDF, addEPUB, refreshDocuments, enableDocx, emitBatchState]);
|
||||
}, [uploadDocuments, emitBatchState]);
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive } = useDropzone({
|
||||
onDrop,
|
||||
|
|
@ -151,12 +87,12 @@ export function DocumentUploader({
|
|||
} : {})
|
||||
},
|
||||
multiple: true,
|
||||
disabled: isUploading || isConverting,
|
||||
disabled: isUploading,
|
||||
noClick: variant === 'overlay',
|
||||
noKeyboard: variant === 'overlay'
|
||||
});
|
||||
|
||||
const isDisabled = isUploading || isConverting;
|
||||
const isDisabled = isUploading;
|
||||
|
||||
if (variant === 'overlay') {
|
||||
const rootProps = getRootProps();
|
||||
|
|
@ -209,8 +145,6 @@ export function DocumentUploader({
|
|||
<UploadIcon className="w-3.5 h-3.5 text-soft group-hover:text-accent shrink-0 transition-colors duration-base" />
|
||||
{isUploading ? (
|
||||
<p className="text-[12px] font-medium truncate flex-1">Uploading…</p>
|
||||
) : isConverting ? (
|
||||
<p className="text-[12px] font-medium truncate flex-1">Converting DOCX…</p>
|
||||
) : (
|
||||
<div className="flex items-center gap-2 min-w-0 flex-1">
|
||||
<p className="text-[12px] truncate flex-1">
|
||||
|
|
@ -225,8 +159,6 @@ export function DocumentUploader({
|
|||
<UploadIcon className="w-7 h-7 sm:w-10 sm:h-10 mb-2 text-soft" />
|
||||
{isUploading ? (
|
||||
<p className="text-sm sm:text-lg font-semibold text-foreground">Uploading file...</p>
|
||||
) : isConverting ? (
|
||||
<p className="text-sm sm:text-lg font-semibold text-foreground">Converting DOCX to PDF...</p>
|
||||
) : (
|
||||
<>
|
||||
<p className="mb-2 text-sm sm:text-lg font-semibold text-foreground">
|
||||
|
|
|
|||
|
|
@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl';
|
|||
import { Navigator } from '@/components/player/Navigator';
|
||||
import { IconButton } from '@/components/ui';
|
||||
|
||||
export default function TTSPlayer({ currentPage, numPages }: {
|
||||
export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: {
|
||||
currentPage?: number;
|
||||
numPages?: number | undefined;
|
||||
isPlaybackReady?: boolean;
|
||||
}) {
|
||||
const {
|
||||
isPlaying,
|
||||
|
|
@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
|||
<IconButton
|
||||
onClick={skipBackward}
|
||||
aria-label="Skip backward"
|
||||
disabled={isProcessing}
|
||||
disabled={isProcessing || !isPlaybackReady}
|
||||
className="relative"
|
||||
>
|
||||
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
|
||||
|
|
@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
|||
<IconButton
|
||||
onClick={togglePlay}
|
||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||
disabled={isProcessing && !isPlaying}
|
||||
disabled={!isPlaying && (!isPlaybackReady || isProcessing)}
|
||||
className="relative"
|
||||
>
|
||||
{isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />}
|
||||
{!isPlaying && !isPlaybackReady
|
||||
? <LoadingSpinner />
|
||||
: (isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />)}
|
||||
</IconButton>
|
||||
|
||||
<IconButton
|
||||
onClick={skipForward}
|
||||
aria-label="Skip forward"
|
||||
disabled={isProcessing}
|
||||
disabled={isProcessing || !isPlaybackReady}
|
||||
className="relative"
|
||||
>
|
||||
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}
|
||||
|
|
|
|||
|
|
@ -3,32 +3,53 @@
|
|||
import { createContext, useCallback, useContext, useEffect, useMemo, ReactNode } from 'react';
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import type { BaseDocument } from '@/types/documents';
|
||||
import { listDocuments, uploadDocuments, deleteDocuments } from '@/lib/client/api/documents';
|
||||
import { putCachedEpub, putCachedHtml, putCachedPdf, evictCachedEpub, evictCachedHtml, evictCachedPdf } from '@/lib/client/cache/documents';
|
||||
import {
|
||||
deleteDocuments as deleteServerDocuments,
|
||||
listDocuments,
|
||||
uploadDocuments as uploadServerDocuments,
|
||||
} from '@/lib/client/api/documents';
|
||||
import { cacheStoredDocumentFromBytes, evictCachedDocument } from '@/lib/client/cache/documents';
|
||||
import { useAuthSession } from '@/hooks/useAuthSession';
|
||||
|
||||
interface DocumentContextType {
|
||||
pdfDocs: Array<BaseDocument & { type: 'pdf' }>;
|
||||
addPDFDocument: (file: File) => Promise<string>;
|
||||
removePDFDocument: (id: string) => Promise<void>;
|
||||
isPDFLoading: boolean;
|
||||
|
||||
epubDocs: Array<BaseDocument & { type: 'epub' }>;
|
||||
addEPUBDocument: (file: File) => Promise<string>;
|
||||
removeEPUBDocument: (id: string) => Promise<void>;
|
||||
isEPUBLoading: boolean;
|
||||
|
||||
htmlDocs: Array<BaseDocument & { type: 'html' }>;
|
||||
addHTMLDocument: (file: File) => Promise<string>;
|
||||
removeHTMLDocument: (id: string) => Promise<void>;
|
||||
isHTMLLoading: boolean;
|
||||
|
||||
uploadDocuments: (files: File[]) => Promise<BaseDocument[]>;
|
||||
deleteDocument: (id: string) => Promise<void>;
|
||||
refreshDocuments: () => Promise<void>;
|
||||
}
|
||||
|
||||
const DocumentContext = createContext<DocumentContextType | undefined>(undefined);
|
||||
const DOCUMENTS_QUERY_KEY = 'documents';
|
||||
|
||||
type SupportedDocument = BaseDocument & { type: 'pdf' | 'epub' | 'html' };
|
||||
|
||||
function mergeStoredDocuments(
|
||||
previous: SupportedDocument[] | undefined,
|
||||
uploaded: BaseDocument[],
|
||||
): SupportedDocument[] {
|
||||
const next = [...(previous ?? [])];
|
||||
|
||||
for (let index = uploaded.length - 1; index >= 0; index -= 1) {
|
||||
const stored = uploaded[index];
|
||||
if (stored.type !== 'pdf' && stored.type !== 'epub' && stored.type !== 'html') {
|
||||
continue;
|
||||
}
|
||||
const supportedStored = stored as SupportedDocument;
|
||||
const withoutExisting = next.filter((document) => document.id !== supportedStored.id);
|
||||
next.splice(0, next.length, supportedStored, ...withoutExisting);
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
export function DocumentProvider({ children }: { children: ReactNode }) {
|
||||
const queryClient = useQueryClient();
|
||||
const { data: sessionData, isPending: isSessionPending } = useAuthSession();
|
||||
|
|
@ -81,67 +102,63 @@ export function DocumentProvider({ children }: { children: ReactNode }) {
|
|||
return { pdfDocs, epubDocs, htmlDocs };
|
||||
}, [docs]);
|
||||
|
||||
const cacheUploaded = useCallback(async (stored: BaseDocument, file: File) => {
|
||||
try {
|
||||
if (stored.type === 'pdf') {
|
||||
await putCachedPdf(stored, await file.arrayBuffer());
|
||||
} else if (stored.type === 'epub') {
|
||||
await putCachedEpub(stored, await file.arrayBuffer());
|
||||
} else if (stored.type === 'html') {
|
||||
const buf = await file.arrayBuffer();
|
||||
const decoded = new TextDecoder().decode(new Uint8Array(buf));
|
||||
await putCachedHtml(stored, decoded);
|
||||
}
|
||||
} catch (err) {
|
||||
// Cache failures should not block uploads.
|
||||
console.warn('Failed to cache uploaded document:', stored.id, err);
|
||||
}
|
||||
}, []);
|
||||
const uploadDocuments = useCallback(async (files: File[]): Promise<BaseDocument[]> => {
|
||||
if (files.length === 0) return [];
|
||||
|
||||
const addDocument = useCallback(async (file: File): Promise<string> => {
|
||||
const [stored] = await uploadDocuments([file]);
|
||||
if (!stored) throw new Error('Upload succeeded but returned no document');
|
||||
await cacheUploaded(stored, file);
|
||||
const isSupported = stored.type === 'pdf' || stored.type === 'epub' || stored.type === 'html';
|
||||
if (!isSupported) return stored.id;
|
||||
const supportedStored = stored as BaseDocument & { type: 'pdf' | 'epub' | 'html' };
|
||||
queryClient.setQueryData<Array<BaseDocument & { type: 'pdf' | 'epub' | 'html' }>>(documentsQueryKey, (prev = []) => {
|
||||
const next = prev.filter((d) => d.id !== supportedStored.id);
|
||||
return [supportedStored, ...next];
|
||||
});
|
||||
return stored.id;
|
||||
}, [cacheUploaded, queryClient, documentsQueryKey]);
|
||||
|
||||
const addPDFDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
|
||||
const addEPUBDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
|
||||
const addHTMLDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
|
||||
|
||||
const removeById = useCallback(async (id: string) => {
|
||||
await deleteDocuments({ ids: [id] });
|
||||
await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]);
|
||||
queryClient.setQueryData<Array<BaseDocument & { type: 'pdf' | 'epub' | 'html' }>>(documentsQueryKey, (prev = []) =>
|
||||
prev.filter((d) => d.id !== id),
|
||||
const stored = await uploadServerDocuments(files);
|
||||
await Promise.allSettled(
|
||||
stored.map(async (document, index) => {
|
||||
const file = files[index];
|
||||
if (!file) return;
|
||||
const sourceType = file.name
|
||||
? (
|
||||
file.name.toLowerCase().endsWith('.pdf')
|
||||
? 'pdf'
|
||||
: file.name.toLowerCase().endsWith('.epub')
|
||||
? 'epub'
|
||||
: file.name.toLowerCase().endsWith('.docx')
|
||||
? 'docx'
|
||||
: 'html'
|
||||
)
|
||||
: (
|
||||
file.type === 'application/pdf'
|
||||
? 'pdf'
|
||||
: file.type === 'application/epub+zip'
|
||||
? 'epub'
|
||||
: file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
? 'docx'
|
||||
: 'html'
|
||||
);
|
||||
if (document.type !== sourceType) return;
|
||||
await cacheStoredDocumentFromBytes(document, await file.arrayBuffer());
|
||||
}),
|
||||
);
|
||||
}, [queryClient, documentsQueryKey]);
|
||||
|
||||
const removePDFDocument = useCallback(async (id: string) => removeById(id), [removeById]);
|
||||
const removeEPUBDocument = useCallback(async (id: string) => removeById(id), [removeById]);
|
||||
const removeHTMLDocument = useCallback(async (id: string) => removeById(id), [removeById]);
|
||||
queryClient.setQueryData<SupportedDocument[]>(documentsQueryKey, (previous) =>
|
||||
mergeStoredDocuments(previous, stored),
|
||||
);
|
||||
|
||||
return stored;
|
||||
}, [documentsQueryKey, queryClient]);
|
||||
|
||||
const deleteDocument = useCallback(async (id: string) => {
|
||||
await deleteServerDocuments({ ids: [id] });
|
||||
await evictCachedDocument(id);
|
||||
queryClient.setQueryData<SupportedDocument[]>(documentsQueryKey, (previous = []) =>
|
||||
previous.filter((document) => document.id !== id),
|
||||
);
|
||||
}, [documentsQueryKey, queryClient]);
|
||||
|
||||
return (
|
||||
<DocumentContext.Provider value={{
|
||||
pdfDocs: docsByType.pdfDocs,
|
||||
addPDFDocument,
|
||||
removePDFDocument,
|
||||
isPDFLoading: isLoading,
|
||||
epubDocs: docsByType.epubDocs,
|
||||
addEPUBDocument,
|
||||
removeEPUBDocument,
|
||||
isEPUBLoading: isLoading,
|
||||
htmlDocs: docsByType.htmlDocs,
|
||||
addHTMLDocument,
|
||||
removeHTMLDocument,
|
||||
isHTMLLoading: isLoading,
|
||||
uploadDocuments,
|
||||
deleteDocument,
|
||||
refreshDocuments,
|
||||
|
||||
}}>
|
||||
|
|
|
|||
|
|
@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
||||
const isPlayingRef = useRef(false);
|
||||
const pauseEpochRef = useRef(0);
|
||||
const pendingPlaybackStartRef = useRef(false);
|
||||
const sentencesRef = useRef<string[]>([]);
|
||||
const currentIndexRef = useRef(0);
|
||||
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
|
||||
|
|
@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const recordManualPause = useCallback(() => {
|
||||
// Cancel any queued auto-resume intent and mark an explicit user pause.
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
pendingPlaybackStartRef.current = false;
|
||||
pauseEpochRef.current += 1;
|
||||
}, []);
|
||||
|
||||
|
|
@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
* Used for external control of playback state
|
||||
*/
|
||||
const pause = useCallback(() => {
|
||||
pendingPlaybackStartRef.current = false;
|
||||
recordManualPause();
|
||||
clearPendingEpubJump();
|
||||
abortPendingTtsRequests(true);
|
||||
|
|
@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const shouldPause = normalizedOptions.shouldPause ?? false;
|
||||
const pauseEpochAtStart = pauseEpochRef.current;
|
||||
const pendingAutoResume = resumeAfterLocationChangeRef.current;
|
||||
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume);
|
||||
if (shouldPause || pendingAutoResume) {
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
}
|
||||
const pendingPlaybackStart = pendingPlaybackStartRef.current;
|
||||
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart);
|
||||
|
||||
// Keep track of previous state and pause playback
|
||||
invalidatePlaybackRun();
|
||||
|
|
@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
try {
|
||||
if (newSentences.length === 0) {
|
||||
console.warn('No sentences found in text');
|
||||
if (shouldPause || pendingAutoResume) {
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
}
|
||||
if (pendingPlaybackStart) {
|
||||
pendingPlaybackStartRef.current = false;
|
||||
}
|
||||
setIsProcessing(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (shouldPause || pendingAutoResume) {
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
}
|
||||
if (pendingPlaybackStart) {
|
||||
pendingPlaybackStartRef.current = false;
|
||||
}
|
||||
|
||||
if (!isEPUB && typeof resolvedLocation === 'number') {
|
||||
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
|
||||
if (firstFingerprint) {
|
||||
|
|
@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
// Ensure audio is unlocked while we're still in the click/tap handler.
|
||||
unlockPlaybackOnUserGesture();
|
||||
|
||||
const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]);
|
||||
if (!activeHowl && !hasPreparedSentence) {
|
||||
pendingPlaybackStartRef.current = true;
|
||||
setIsProcessing(true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Resume current sentence if we already have a paused Howl.
|
||||
if (activeHowl) {
|
||||
applyPlaybackRateToHowl(activeHowl);
|
||||
|
|
@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
setIsPlaying(true);
|
||||
}, [
|
||||
activeHowl,
|
||||
currentIndex,
|
||||
applyPlaybackRateToHowl,
|
||||
isPlaying,
|
||||
pauseActiveHowl,
|
||||
playbackSegments,
|
||||
recordManualPause,
|
||||
clearPendingEpubJump,
|
||||
abortPendingTtsRequests,
|
||||
sentences,
|
||||
unlockPlaybackOnUserGesture,
|
||||
]);
|
||||
|
||||
|
|
@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
abortAudio();
|
||||
clearWarmAudioCache();
|
||||
playbackInFlightRef.current = false;
|
||||
pendingPlaybackStartRef.current = false;
|
||||
pendingJumpTargetRef.current = null;
|
||||
clearPendingEpubJump();
|
||||
bumpEpubPreloadGeneration();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
import { sha256HexFromArrayBuffer } from '@/lib/client/sha256';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
import type { DocumentSettings } from '@/types/document-settings';
|
||||
|
||||
export type UploadSource = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: DocumentType;
|
||||
size: number;
|
||||
|
|
@ -23,8 +21,12 @@ function toUploadBody(body: UploadSource['body']): BodyInit {
|
|||
return body as unknown as BodyInit;
|
||||
}
|
||||
|
||||
async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise<void> {
|
||||
const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, {
|
||||
async function uploadDocumentSourceViaProxy(
|
||||
source: UploadSource,
|
||||
token: string,
|
||||
options?: UploadOptions,
|
||||
): Promise<void> {
|
||||
const res = await fetch(`/api/documents/blob/upload/fallback?token=${encodeURIComponent(token)}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': source.contentType || 'application/octet-stream' },
|
||||
body: toUploadBody(source.body),
|
||||
|
|
@ -45,6 +47,14 @@ function documentTypeForName(name: string): DocumentType {
|
|||
return 'html';
|
||||
}
|
||||
|
||||
function documentTypeForMime(contentType: string): DocumentType | null {
|
||||
const normalized = contentType.trim().toLowerCase();
|
||||
if (normalized === 'application/pdf') return 'pdf';
|
||||
if (normalized === 'application/epub+zip') return 'epub';
|
||||
if (normalized === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') return 'docx';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string {
|
||||
if (doc.type === 'pdf') return 'application/pdf';
|
||||
if (doc.type === 'epub') return 'application/epub+zip';
|
||||
|
|
@ -80,31 +90,20 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
|
|||
|
||||
export async function getParsedPdfDocument(
|
||||
id: string,
|
||||
options?: { signal?: AbortSignal; retryFailed?: boolean; opId?: string },
|
||||
): Promise<
|
||||
| { status: 'ready'; parsed: ParsedPdfDocument }
|
||||
| { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null; opId?: string | null }
|
||||
> {
|
||||
const params = new URLSearchParams();
|
||||
if (options?.retryFailed) params.set('retry', '1');
|
||||
if (options?.opId) params.set('opId', options.opId);
|
||||
const query = params.size > 0 ? `?${params.toString()}` : '';
|
||||
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, {
|
||||
options?: { signal?: AbortSignal },
|
||||
): Promise<ParsedPdfDocument> {
|
||||
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed`, {
|
||||
signal: options?.signal,
|
||||
cache: 'no-store',
|
||||
});
|
||||
|
||||
if (res.status === 202) {
|
||||
if (res.status === 409) {
|
||||
const data = (await res.json().catch(() => null)) as {
|
||||
parseStatus?: string;
|
||||
parseProgress?: PdfParseProgress | null;
|
||||
opId?: string | null;
|
||||
} | null;
|
||||
const parseStatus = data?.parseStatus;
|
||||
if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') {
|
||||
return { status: parseStatus, parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null };
|
||||
}
|
||||
return { status: 'pending', parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null };
|
||||
throw new Error(data?.parseStatus ? `Parsed PDF is not ready (${data.parseStatus})` : 'Parsed PDF is not ready');
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
|
|
@ -112,8 +111,7 @@ export async function getParsedPdfDocument(
|
|||
throw new Error(data?.error || 'Failed to load parsed PDF');
|
||||
}
|
||||
|
||||
const parsed = (await res.json()) as ParsedPdfDocument;
|
||||
return { status: 'ready', parsed };
|
||||
return (await res.json()) as ParsedPdfDocument;
|
||||
}
|
||||
|
||||
export function subscribeParsedPdfDocumentEvents(
|
||||
|
|
@ -241,7 +239,6 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
|||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
uploads: sources.map((source) => ({
|
||||
id: source.id,
|
||||
contentType: source.contentType,
|
||||
size: source.size,
|
||||
})),
|
||||
|
|
@ -255,14 +252,18 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
|||
}
|
||||
|
||||
const presigned = (await presignRes.json()) as {
|
||||
uploads?: Array<{ id: string; url: string; headers?: Record<string, string> }>;
|
||||
uploads?: Array<{ token: string; url: string; headers?: Record<string, string> }>;
|
||||
};
|
||||
const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload]));
|
||||
const uploads = presigned.uploads || [];
|
||||
if (uploads.length !== sources.length) {
|
||||
throw new Error('Upload preparation returned an unexpected number of temp uploads');
|
||||
}
|
||||
|
||||
for (const source of sources) {
|
||||
const upload = byId.get(source.id);
|
||||
if (!upload?.url) {
|
||||
throw new Error(`Missing presigned upload for document ${source.id}`);
|
||||
for (let index = 0; index < sources.length; index += 1) {
|
||||
const source = sources[index];
|
||||
const upload = uploads[index];
|
||||
if (!upload?.url || !upload.token) {
|
||||
throw new Error(`Missing presigned upload for document ${source.name}`);
|
||||
}
|
||||
|
||||
let putError: unknown = null;
|
||||
|
|
@ -285,7 +286,7 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
|||
}
|
||||
|
||||
try {
|
||||
await uploadDocumentSourceViaProxy(source, options);
|
||||
await uploadDocumentSourceViaProxy(source, upload.token, options);
|
||||
} catch (proxyError) {
|
||||
const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error';
|
||||
const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error';
|
||||
|
|
@ -293,27 +294,26 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
|||
}
|
||||
}
|
||||
|
||||
const registerRes = await fetch('/api/documents', {
|
||||
const finalizeRes = await fetch('/api/documents/blob/upload/finalize', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
documents: sources.map((source) => ({
|
||||
id: source.id,
|
||||
uploads: sources.map((source, index) => ({
|
||||
token: uploads[index]?.token,
|
||||
name: source.name,
|
||||
type: source.type,
|
||||
size: source.size,
|
||||
lastModified: source.lastModified,
|
||||
})),
|
||||
}),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!registerRes.ok) {
|
||||
const data = (await registerRes.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to register uploaded documents');
|
||||
if (!finalizeRes.ok) {
|
||||
const data = (await finalizeRes.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to finalize uploaded documents');
|
||||
}
|
||||
|
||||
const data = (await registerRes.json()) as { stored: BaseDocument[] };
|
||||
const data = (await finalizeRes.json()) as { stored: BaseDocument[] };
|
||||
return data.stored || [];
|
||||
}
|
||||
|
||||
|
|
@ -322,14 +322,14 @@ export async function uploadDocuments(files: File[], options?: UploadOptions): P
|
|||
|
||||
const sources: UploadSource[] = [];
|
||||
for (const file of files) {
|
||||
const bytes = await file.arrayBuffer();
|
||||
const id = await sha256HexFromArrayBuffer(bytes);
|
||||
const type = documentTypeForName(file.name);
|
||||
const name = file.name || `${id}.${type}`;
|
||||
const contentType = file.type || mimeTypeForDoc({ name, type });
|
||||
const name = file.name || '';
|
||||
const type = name
|
||||
? documentTypeForName(name)
|
||||
: (documentTypeForMime(file.type) ?? 'html');
|
||||
const resolvedName = name || `upload.${type}`;
|
||||
const contentType = file.type || mimeTypeForDoc({ name: resolvedName, type });
|
||||
sources.push({
|
||||
id,
|
||||
name,
|
||||
name: resolvedName,
|
||||
type,
|
||||
size: file.size,
|
||||
lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(),
|
||||
|
|
@ -504,23 +504,3 @@ export async function getDocumentPreviewStatus(
|
|||
|
||||
throw new Error(`Failed to load preview status (status ${res.status})`);
|
||||
}
|
||||
|
||||
export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise<BaseDocument> {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
|
||||
const res = await fetch('/api/documents/docx-to-pdf/upload', {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to convert DOCX');
|
||||
}
|
||||
|
||||
const data = (await res.json()) as { stored: BaseDocument };
|
||||
if (!data?.stored) throw new Error('DOCX conversion succeeded but returned no document');
|
||||
return data.stored;
|
||||
}
|
||||
|
|
|
|||
4
src/lib/client/cache/documents.ts
vendored
4
src/lib/client/cache/documents.ts
vendored
|
|
@ -107,6 +107,10 @@ export async function evictCachedHtml(id: string): Promise<void> {
|
|||
await removeHtmlDocument(id);
|
||||
}
|
||||
|
||||
export async function evictCachedDocument(id: string): Promise<void> {
|
||||
await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]);
|
||||
}
|
||||
|
||||
export async function clearDocumentCache(): Promise<void> {
|
||||
const { clearPdfDocuments, clearEpubDocuments, clearHtmlDocuments } = await import('@/lib/client/dexie');
|
||||
await Promise.all([clearPdfDocuments(), clearEpubDocuments(), clearHtmlDocuments()]);
|
||||
|
|
|
|||
|
|
@ -971,11 +971,9 @@ export async function syncDocumentsToServer(
|
|||
|
||||
for (const doc of pdfDocs) {
|
||||
const bytes = new Uint8Array(doc.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: doc.id,
|
||||
source: {
|
||||
id,
|
||||
name: doc.name,
|
||||
type: 'pdf',
|
||||
size: bytes.byteLength,
|
||||
|
|
@ -992,11 +990,9 @@ export async function syncDocumentsToServer(
|
|||
|
||||
for (const doc of epubDocs) {
|
||||
const bytes = new Uint8Array(doc.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: doc.id,
|
||||
source: {
|
||||
id,
|
||||
name: doc.name,
|
||||
type: 'epub',
|
||||
size: bytes.byteLength,
|
||||
|
|
@ -1013,11 +1009,9 @@ export async function syncDocumentsToServer(
|
|||
|
||||
for (const doc of htmlDocs) {
|
||||
const encoded = textEncoder.encode(doc.data);
|
||||
const id = await sha256HexFromBytes(encoded);
|
||||
uploads.push({
|
||||
oldId: doc.id,
|
||||
source: {
|
||||
id,
|
||||
name: doc.name,
|
||||
type: 'html',
|
||||
size: encoded.byteLength,
|
||||
|
|
@ -1036,11 +1030,13 @@ export async function syncDocumentsToServer(
|
|||
onProgress(50, 'Uploading to server...');
|
||||
}
|
||||
|
||||
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||
const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||
|
||||
for (const entry of uploads) {
|
||||
if (entry.oldId === entry.source.id) continue;
|
||||
await applyDocumentIdMapping(entry.oldId, entry.source.id);
|
||||
for (let index = 0; index < uploads.length; index += 1) {
|
||||
const entry = uploads[index];
|
||||
const nextId = stored[index]?.id;
|
||||
if (!nextId || entry.oldId === nextId) continue;
|
||||
await applyDocumentIdMapping(entry.oldId, nextId);
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
|
|
@ -1064,11 +1060,9 @@ export async function syncSelectedDocumentsToServer(
|
|||
const data = await getPdfDocument(doc.id);
|
||||
if (data) {
|
||||
const bytes = new Uint8Array(data.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: data.id,
|
||||
source: {
|
||||
id,
|
||||
name: data.name,
|
||||
type: 'pdf',
|
||||
size: bytes.byteLength,
|
||||
|
|
@ -1082,11 +1076,9 @@ export async function syncSelectedDocumentsToServer(
|
|||
const data = await getEpubDocument(doc.id);
|
||||
if (data) {
|
||||
const bytes = new Uint8Array(data.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: data.id,
|
||||
source: {
|
||||
id,
|
||||
name: data.name,
|
||||
type: 'epub',
|
||||
size: bytes.byteLength,
|
||||
|
|
@ -1100,11 +1092,9 @@ export async function syncSelectedDocumentsToServer(
|
|||
const data = await getHtmlDocument(doc.id);
|
||||
if (data) {
|
||||
const bytes = textEncoder.encode(data.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: data.id,
|
||||
source: {
|
||||
id,
|
||||
name: data.name,
|
||||
type: 'html',
|
||||
size: bytes.byteLength,
|
||||
|
|
@ -1121,11 +1111,13 @@ export async function syncSelectedDocumentsToServer(
|
|||
}
|
||||
|
||||
if (onProgress) onProgress(50, 'Uploading to server...');
|
||||
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||
const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||
|
||||
for (const entry of uploads) {
|
||||
if (entry.oldId === entry.source.id) continue;
|
||||
await applyDocumentIdMapping(entry.oldId, entry.source.id);
|
||||
for (let index = 0; index < uploads.length; index += 1) {
|
||||
const entry = uploads[index];
|
||||
const nextId = stored[index]?.id;
|
||||
if (!nextId || entry.oldId === nextId) continue;
|
||||
await applyDocumentIdMapping(entry.oldId, nextId);
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
|
|
@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState(
|
|||
? parseProgress
|
||||
: null,
|
||||
updatedAt: nowMs,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
|
||||
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
|
||||
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
|
||||
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
|
||||
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core';
|
||||
import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||
import type {
|
||||
|
|
@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string {
|
|||
return [
|
||||
'pdf_layout',
|
||||
'v1',
|
||||
PDF_PARSER_VERSION,
|
||||
input.documentId,
|
||||
input.namespace ?? '',
|
||||
input.documentObjectKey ?? '',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import {
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
DeleteObjectsCommand,
|
||||
GetObjectCommand,
|
||||
|
|
@ -11,6 +12,9 @@ import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage
|
|||
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const TEMP_UPLOAD_TOKEN_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const TEMP_UPLOAD_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/;
|
||||
export const TEMP_DOCUMENT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
function sanitizeNamespace(namespace: string | null): string | null {
|
||||
if (!namespace) return null;
|
||||
|
|
@ -69,6 +73,23 @@ export function isValidDocumentId(id: string): boolean {
|
|||
return DOCUMENT_ID_REGEX.test(id);
|
||||
}
|
||||
|
||||
export function isPreconditionFailed(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
|
||||
}
|
||||
|
||||
export function isValidTempUploadToken(token: string): boolean {
|
||||
return TEMP_UPLOAD_TOKEN_REGEX.test(token);
|
||||
}
|
||||
|
||||
function sanitizeTempUploadUserId(userId: string): string {
|
||||
if (!TEMP_UPLOAD_USER_ID_REGEX.test(userId)) {
|
||||
throw new Error(`Invalid temp upload user id: ${userId}`);
|
||||
}
|
||||
return encodeURIComponent(userId);
|
||||
}
|
||||
|
||||
export function documentKey(id: string, namespace: string | null): string {
|
||||
if (!isValidDocumentId(id)) {
|
||||
throw new Error(`Invalid document id: ${id}`);
|
||||
|
|
@ -90,6 +111,27 @@ export function documentParsedKey(id: string, namespace: string | null): string
|
|||
return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`;
|
||||
}
|
||||
|
||||
export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string {
|
||||
const cfg = getS3Config();
|
||||
const ns = sanitizeNamespace(namespace);
|
||||
const nsSegment = ns ? `ns/${ns}/` : '';
|
||||
return `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}users/${sanitizeTempUploadUserId(userId)}/`;
|
||||
}
|
||||
|
||||
export function tempDocumentUploadKey(token: string, userId: string, namespace: string | null): string {
|
||||
if (!isValidTempUploadToken(token)) {
|
||||
throw new Error(`Invalid temp upload token: ${token}`);
|
||||
}
|
||||
return `${tempDocumentUploadPrefix(userId, namespace)}${token}.bin`;
|
||||
}
|
||||
|
||||
export function tempDocumentUploadReceiptKey(token: string, userId: string, namespace: string | null): string {
|
||||
if (!isValidTempUploadToken(token)) {
|
||||
throw new Error(`Invalid temp upload token: ${token}`);
|
||||
}
|
||||
return `${tempDocumentUploadPrefix(userId, namespace)}${token}.receipt.json`;
|
||||
}
|
||||
|
||||
function legacyDocumentParsedKey(id: string, namespace: string | null): string {
|
||||
if (!isValidDocumentId(id)) {
|
||||
throw new Error(`Invalid document id: ${id}`);
|
||||
|
|
@ -138,6 +180,40 @@ export async function presignPut(
|
|||
};
|
||||
}
|
||||
|
||||
export async function presignTempPut(
|
||||
token: string,
|
||||
userId: string,
|
||||
contentType: string,
|
||||
namespace: string | null,
|
||||
options?: { contentLength?: number },
|
||||
): Promise<{ url: string; headers: Record<string, string> }> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||
const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream';
|
||||
const contentLength =
|
||||
typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0
|
||||
? Math.floor(options.contentLength)
|
||||
: undefined;
|
||||
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
ContentType: normalizedType,
|
||||
ServerSideEncryption: 'AES256',
|
||||
...(contentLength !== undefined ? { ContentLength: contentLength } : {}),
|
||||
});
|
||||
const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 });
|
||||
|
||||
return {
|
||||
url,
|
||||
headers: {
|
||||
'Content-Type': normalizedType,
|
||||
'x-amz-server-side-encryption': 'AES256',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function headDocumentBlob(
|
||||
id: string,
|
||||
namespace: string | null,
|
||||
|
|
@ -153,6 +229,23 @@ export async function headDocumentBlob(
|
|||
};
|
||||
}
|
||||
|
||||
export async function headTempDocumentBlob(
|
||||
token: string,
|
||||
userId: string,
|
||||
namespace: string | null,
|
||||
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null; lastModified: number | null }> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
return {
|
||||
contentLength: Number(res.ContentLength ?? 0),
|
||||
contentType: res.ContentType ?? null,
|
||||
eTag: res.ETag ?? null,
|
||||
lastModified: res.LastModified?.getTime() ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getDocumentRange(
|
||||
id: string,
|
||||
start: number,
|
||||
|
|
@ -185,6 +278,23 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro
|
|||
return bodyToBuffer(res.Body);
|
||||
}
|
||||
|
||||
export async function getTempDocumentBlob(
|
||||
token: string,
|
||||
userId: string,
|
||||
namespace: string | null,
|
||||
): Promise<Buffer> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||
const res = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
}),
|
||||
);
|
||||
return bodyToBuffer(res.Body);
|
||||
}
|
||||
|
||||
export async function getDocumentBlobStream(id: string, namespace: string | null): Promise<DocumentBlobBody> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
|
|
@ -198,6 +308,29 @@ export async function getDocumentBlobStream(id: string, namespace: string | null
|
|||
return res.Body as DocumentBlobBody;
|
||||
}
|
||||
|
||||
export async function getTempDocumentFinalizeReceipt<T>(
|
||||
token: string,
|
||||
userId: string,
|
||||
namespace: string | null,
|
||||
): Promise<T | null> {
|
||||
try {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
const key = tempDocumentUploadReceiptKey(token, userId, namespace);
|
||||
const res = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
}),
|
||||
);
|
||||
const body = await bodyToBuffer(res.Body);
|
||||
return JSON.parse(body.toString('utf8')) as T;
|
||||
} catch (error) {
|
||||
if (isMissingBlobError(error)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
|
|
@ -241,6 +374,26 @@ export async function putParsedDocumentBlob(id: string, body: Buffer, namespace:
|
|||
return key;
|
||||
}
|
||||
|
||||
export async function putTempDocumentFinalizeReceipt(
|
||||
token: string,
|
||||
userId: string,
|
||||
namespace: string | null,
|
||||
body: Buffer,
|
||||
): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
const key = tempDocumentUploadReceiptKey(token, userId, namespace);
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: 'application/json',
|
||||
ServerSideEncryption: 'AES256',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function presignGet(
|
||||
id: string,
|
||||
namespace: string | null,
|
||||
|
|
@ -264,6 +417,7 @@ export async function putDocumentBlob(
|
|||
body: Buffer,
|
||||
contentType: string,
|
||||
namespace: string | null,
|
||||
options?: { ifNoneMatch?: boolean },
|
||||
): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
|
|
@ -275,6 +429,51 @@ export async function putDocumentBlob(
|
|||
Body: body,
|
||||
ContentType: contentType,
|
||||
ServerSideEncryption: 'AES256',
|
||||
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function putTempDocumentBlob(
|
||||
token: string,
|
||||
userId: string,
|
||||
body: Buffer,
|
||||
contentType: string,
|
||||
namespace: string | null,
|
||||
): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
ServerSideEncryption: 'AES256',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function copyTempDocumentBlobToDocument(
|
||||
token: string,
|
||||
userId: string,
|
||||
documentId: string,
|
||||
namespace: string | null,
|
||||
contentType: string,
|
||||
options?: { ifNoneMatch?: boolean },
|
||||
): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
await client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: documentKey(documentId, namespace),
|
||||
CopySource: `${cfg.bucket}/${tempDocumentUploadKey(token, userId, namespace)}`,
|
||||
ContentType: contentType,
|
||||
MetadataDirective: 'REPLACE',
|
||||
ServerSideEncryption: 'AES256',
|
||||
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
|
@ -292,6 +491,18 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
|
|||
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
|
||||
}
|
||||
|
||||
export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadKey(token, userId, namespace) }));
|
||||
}
|
||||
|
||||
export async function deleteTempDocumentFinalizeReceipt(token: string, userId: string, namespace: string | null): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadReceiptKey(token, userId, namespace) }));
|
||||
}
|
||||
|
||||
export function isMissingBlobError(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
|
||||
|
|
@ -339,3 +550,53 @@ export async function deleteDocumentPrefix(prefix: string): Promise<number> {
|
|||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
export async function deleteExpiredTempDocumentUploads(
|
||||
userId: string,
|
||||
namespace: string | null,
|
||||
olderThanMs: number,
|
||||
): Promise<number> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3ProxyClient();
|
||||
const prefix = tempDocumentUploadPrefix(userId, namespace);
|
||||
let continuationToken: string | undefined;
|
||||
const keys: string[] = [];
|
||||
|
||||
do {
|
||||
const listRes = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: cfg.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
}),
|
||||
);
|
||||
|
||||
for (const item of listRes.Contents ?? []) {
|
||||
const key = item.Key;
|
||||
const lastModified = item.LastModified?.getTime() ?? 0;
|
||||
if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue;
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
if (keys.length === 0) return 0;
|
||||
|
||||
let deleted = 0;
|
||||
for (let i = 0; i < keys.length; i += 1000) {
|
||||
const batch = keys.slice(i, i + 1000);
|
||||
const deleteRes = await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Delete: {
|
||||
Objects: batch.map((Key) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
deleted += deleteRes.Deleted?.length ?? 0;
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
|
|
|||
94
src/lib/server/documents/docx-convert.ts
Normal file
94
src/lib/server/documents/docx-convert.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import { spawn } from 'child_process';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { pathToFileURL } from 'url';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||
const DEFAULT_DOCX_CONVERSION_TIMEOUT_MS = 60_000;
|
||||
|
||||
async function ensureTempDir(): Promise<void> {
|
||||
if (!existsSync(DOCSTORE_DIR)) {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
}
|
||||
if (!existsSync(TEMP_DIR)) {
|
||||
await mkdir(TEMP_DIR, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args: string[] = [];
|
||||
if (profileDir) {
|
||||
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
|
||||
}
|
||||
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
|
||||
const proc = spawn('soffice', args);
|
||||
let settled = false;
|
||||
const timeout = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
proc.kill();
|
||||
reject(new Error(`LibreOffice conversion timed out after ${DEFAULT_DOCX_CONVERSION_TIMEOUT_MS}ms`));
|
||||
}, DEFAULT_DOCX_CONVERSION_TIMEOUT_MS);
|
||||
|
||||
proc.on('error', (error) => {
|
||||
clearTimeout(timeout);
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
reject(error);
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
clearTimeout(timeout);
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForPdfReady(dir: string, timeoutMs = 20_000, intervalMs = 100): Promise<string> {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) {
|
||||
const files = await readdir(dir);
|
||||
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
|
||||
if (pdf) {
|
||||
const pdfPath = path.join(dir, pdf);
|
||||
try {
|
||||
const first = await stat(pdfPath);
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
const second = await stat(pdfPath);
|
||||
if (second.size > 0 && second.size === first.size) {
|
||||
return pdfPath;
|
||||
}
|
||||
} catch {
|
||||
// Ignore transient filesystem races while LibreOffice is still writing.
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
export async function convertDocxBufferToPdfBuffer(docxBytes: Buffer): Promise<Buffer> {
|
||||
await ensureTempDir();
|
||||
|
||||
const tempId = randomUUID();
|
||||
const jobDir = path.join(TEMP_DIR, tempId);
|
||||
await mkdir(jobDir, { recursive: true });
|
||||
const profileDir = path.join(jobDir, 'lo-profile');
|
||||
await mkdir(profileDir, { recursive: true });
|
||||
const inputPath = path.join(jobDir, 'input.docx');
|
||||
|
||||
try {
|
||||
await writeFile(inputPath, docxBytes);
|
||||
await convertDocxToPdf(inputPath, jobDir, profileDir);
|
||||
const pdfPath = await waitForPdfReady(jobDir);
|
||||
return await readFile(pdfPath);
|
||||
} finally {
|
||||
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,5 @@
|
|||
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
|
||||
import { documentKey } from '@/lib/server/documents/blobstore';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||
import { normalizeParseStatus, type DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
|
||||
|
|
@ -20,13 +18,20 @@ export async function backfillPendingPdfParseOperation(input: {
|
|||
if (parseStatus === 'ready' || parseStatus === 'failed') return null;
|
||||
if (normalizeOpId(input.state.opId)) return null;
|
||||
|
||||
const created = await createOrReusePdfWorkerOperation({
|
||||
const startedParse = await startPdfParseOperation({
|
||||
documentId: input.documentId,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
documentObjectKey: documentKey(input.documentId, input.namespace),
|
||||
});
|
||||
|
||||
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
||||
await recordJobEvent(input.userId, 'pdf_layout', created.opId, rateConfig);
|
||||
return created;
|
||||
if (startedParse.parseState.status === 'pending' || startedParse.parseState.status === 'running') {
|
||||
enqueueParsePdfJob({
|
||||
documentId: input.documentId,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
initialOpId: startedParse.workerState.opId,
|
||||
initialJobId: startedParse.workerState.jobId,
|
||||
initialStatus: startedParse.parseState.status,
|
||||
});
|
||||
}
|
||||
return startedParse.workerState;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: {
|
|||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||
...(input.state.opId ? { opId: input.state.opId } : {}),
|
||||
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
|
||||
...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}),
|
||||
}));
|
||||
|
||||
await db
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
|
||||
export interface DocumentParseState {
|
||||
status: PdfParseStatus;
|
||||
|
|
@ -7,6 +8,7 @@ export interface DocumentParseState {
|
|||
error?: string | null;
|
||||
opId?: string;
|
||||
jobId?: string;
|
||||
parserVersion?: string;
|
||||
}
|
||||
|
||||
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
|
||||
|
|
@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars
|
|||
return 'pending';
|
||||
}
|
||||
|
||||
export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean {
|
||||
return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION;
|
||||
}
|
||||
|
||||
export function normalizeDocumentParseStateForCurrentParserVersion(
|
||||
state: DocumentParseState,
|
||||
nowMs = Date.now(),
|
||||
): DocumentParseState {
|
||||
const status = normalizeParseStatus(state.status);
|
||||
if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) {
|
||||
return state;
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: nowMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string {
|
||||
const parserVersion = parsed?.parserVersion?.trim();
|
||||
return parserVersion || PDF_PARSER_VERSION;
|
||||
}
|
||||
|
||||
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
||||
if (!progress || typeof progress !== 'object') return null;
|
||||
const rec = progress as Record<string, unknown>;
|
||||
|
|
@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
|||
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
||||
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
||||
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
||||
const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined;
|
||||
return {
|
||||
status,
|
||||
progress,
|
||||
|
|
@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
|||
...(error ? { error } : {}),
|
||||
...(opId ? { opId } : {}),
|
||||
...(jobId ? { jobId } : {}),
|
||||
...(parserVersion ? { parserVersion } : {}),
|
||||
};
|
||||
} catch {
|
||||
return { status: 'pending', progress: null };
|
||||
|
|
@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string {
|
|||
...(state.error ? { error: state.error } : {}),
|
||||
...(state.opId ? { opId: state.opId } : {}),
|
||||
...(state.jobId ? { jobId: state.jobId } : {}),
|
||||
...(state.parserVersion ? { parserVersion: state.parserVersion } : {}),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { eq } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
parseDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
|
||||
type ReadyParsedPdfResult = {
|
||||
parsedJsonKey: string;
|
||||
};
|
||||
|
||||
export async function findReusableParsedPdfResult(
|
||||
documentId: string,
|
||||
): Promise<ReadyParsedPdfResult | null> {
|
||||
const rows = await db
|
||||
.select({
|
||||
parseState: documents.parseState,
|
||||
parsedJsonKey: documents.parsedJsonKey,
|
||||
})
|
||||
.from(documents)
|
||||
.where(eq(documents.id, documentId));
|
||||
|
||||
for (const row of rows) {
|
||||
const parsedJsonKey = row.parsedJsonKey?.trim();
|
||||
if (!parsedJsonKey) continue;
|
||||
const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
if (parseState.status !== 'ready') continue;
|
||||
return { parsedJsonKey };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
30
src/lib/server/documents/pdf-parse-operation.ts
Normal file
30
src/lib/server/documents/pdf-parse-operation.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
|
||||
import { documentParseStateFromWorkerState } from '@/lib/server/compute/worker-parse-state';
|
||||
import { documentKey } from '@/lib/server/documents/blobstore';
|
||||
import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
|
||||
export async function startPdfParseOperation(input: {
|
||||
documentId: string;
|
||||
userId: string;
|
||||
namespace: string | null;
|
||||
forceToken?: string;
|
||||
}): Promise<{
|
||||
workerState: WorkerOperationState<PdfLayoutJobResult>;
|
||||
parseState: DocumentParseState;
|
||||
}> {
|
||||
const workerState = await createOrReusePdfWorkerOperation({
|
||||
documentId: input.documentId,
|
||||
namespace: input.namespace,
|
||||
documentObjectKey: documentKey(input.documentId, input.namespace),
|
||||
...(input.forceToken ? { forceToken: input.forceToken } : {}),
|
||||
});
|
||||
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
||||
await recordJobEvent(input.userId, 'pdf_layout', workerState.opId, rateConfig);
|
||||
return {
|
||||
workerState,
|
||||
parseState: documentParseStateFromWorkerState(workerState),
|
||||
};
|
||||
}
|
||||
106
src/lib/server/documents/register-upload.ts
Normal file
106
src/lib/server/documents/register-upload.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import {
|
||||
enqueueDocumentPreview,
|
||||
} from '@/lib/server/documents/previews';
|
||||
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
type RegisterUploadedDocumentInput = {
|
||||
documentId: string;
|
||||
userId: string;
|
||||
namespace: string | null;
|
||||
name: string;
|
||||
type: DocumentType;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
};
|
||||
|
||||
export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> {
|
||||
const reusableParsedPdf = input.type === 'pdf'
|
||||
? await findReusableParsedPdfResult(input.documentId)
|
||||
: null;
|
||||
const startedParse = input.type === 'pdf' && !reusableParsedPdf
|
||||
? await startPdfParseOperation({
|
||||
documentId: input.documentId,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
})
|
||||
: null;
|
||||
const parsedJsonKey = reusableParsedPdf?.parsedJsonKey ?? null;
|
||||
const parseState = input.type === 'pdf'
|
||||
? stringifyDocumentParseState(
|
||||
reusableParsedPdf
|
||||
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||
: (startedParse?.parseState ?? { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }),
|
||||
)
|
||||
: null;
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
id: input.documentId,
|
||||
userId: input.userId,
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
size: input.size,
|
||||
lastModified: input.lastModified,
|
||||
filePath: input.documentId,
|
||||
parseState,
|
||||
parsedJsonKey,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
set: {
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
size: input.size,
|
||||
lastModified: input.lastModified,
|
||||
filePath: input.documentId,
|
||||
parseState,
|
||||
parsedJsonKey,
|
||||
},
|
||||
});
|
||||
|
||||
await enqueueDocumentPreview(
|
||||
{
|
||||
id: input.documentId,
|
||||
type: input.type,
|
||||
lastModified: input.lastModified,
|
||||
},
|
||||
input.namespace,
|
||||
).catch((error) => {
|
||||
serverLogger.warn({
|
||||
event: 'documents.preview.enqueue.failed',
|
||||
degraded: true,
|
||||
fallbackPath: 'skip_preview_enqueue',
|
||||
documentId: input.documentId,
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to enqueue document preview');
|
||||
});
|
||||
|
||||
if (startedParse) {
|
||||
enqueueParsePdfJob({
|
||||
documentId: input.documentId,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
initialOpId: startedParse.workerState.opId,
|
||||
initialJobId: startedParse.workerState.jobId,
|
||||
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
id: input.documentId,
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
size: input.size,
|
||||
lastModified: input.lastModified,
|
||||
scope: 'user',
|
||||
};
|
||||
}
|
||||
|
|
@ -1,11 +1,14 @@
|
|||
import { and, eq, inArray, isNull } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||
import { serverLogger } from '@/lib/server/logger';
|
||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
parseDocumentParseState,
|
||||
resolveParsedPdfParserVersion,
|
||||
stringifyDocumentParseState,
|
||||
type DocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
|
|
@ -13,15 +16,20 @@ import { getCompute } from '@/lib/server/compute';
|
|||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
|
||||
import type {
|
||||
ParsedPdfDocument,
|
||||
PdfLayoutJobBase,
|
||||
PdfLayoutJobResult,
|
||||
PdfLayoutProgress,
|
||||
WorkerOperationState,
|
||||
} from '@openreader/compute-core/api-contracts';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
|
||||
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
|
||||
userId: string;
|
||||
forceToken?: string;
|
||||
initialOpId?: string;
|
||||
initialJobId?: string;
|
||||
initialStatus?: 'pending' | 'running';
|
||||
};
|
||||
|
||||
const running = new Set<string>();
|
||||
|
|
@ -41,7 +49,12 @@ function keyFor(input: UserPdfLayoutJobRequest): string {
|
|||
if (forceToken) {
|
||||
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
|
||||
}
|
||||
return `shared:${input.documentId}:${input.namespace || ''}`;
|
||||
return `shared:${input.documentId}`;
|
||||
}
|
||||
|
||||
function normalizeOpId(value: string | undefined): string | undefined {
|
||||
const normalized = value?.trim();
|
||||
return normalized ? normalized : undefined;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
|
|
@ -84,7 +97,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise<ParseRow[
|
|||
|
||||
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
|
||||
if (!row.parsedJsonKey) return false;
|
||||
return parseDocumentParseState(row.parseState).status === 'ready';
|
||||
return normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)).status === 'ready';
|
||||
}
|
||||
|
||||
function userIdsFromRows(rows: ParseRow[]): string[] {
|
||||
|
|
@ -95,6 +108,20 @@ function parseStateMatchCondition(expected: string | null) {
|
|||
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
|
||||
}
|
||||
|
||||
async function loadParsedDocumentForResult(
|
||||
parsed: ParsedPdfDocument | undefined,
|
||||
parsedJsonKey: string,
|
||||
): Promise<ParsedPdfDocument | null> {
|
||||
if (parsed) return parsed;
|
||||
|
||||
try {
|
||||
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||
return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateParseStateForUsers(input: {
|
||||
documentId: string;
|
||||
userIds: string[];
|
||||
|
|
@ -125,6 +152,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||
if (reusableParsed) {
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
userIds: [input.userId],
|
||||
parseState: stringifyDocumentParseState(readyState),
|
||||
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = await loadScopedRows(input);
|
||||
const ready = rows.find(isReadyRow);
|
||||
if (ready) {
|
||||
|
|
@ -132,6 +176,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
|
|
@ -142,7 +187,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
return;
|
||||
}
|
||||
|
||||
const statuses = rows.map((row) => parseDocumentParseState(row.parseState));
|
||||
const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)));
|
||||
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
|
||||
const failedState = statuses.find((state) => state.status === 'failed');
|
||||
if (failedState && !hasInFlight) {
|
||||
|
|
@ -151,6 +196,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
...(failedState.error ? { error: failedState.error } : {}),
|
||||
...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}),
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
|
|
@ -174,6 +220,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
}
|
||||
running.add(key);
|
||||
|
||||
let activeOpId = normalizeOpId(input.initialOpId);
|
||||
let activeJobId = normalizeOpId(input.initialJobId);
|
||||
try {
|
||||
const scopedRows = await loadScopedRows(input);
|
||||
const scopedUserIds = userIdsFromRows(scopedRows);
|
||||
|
|
@ -181,12 +229,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
// Non-force jobs can short-circuit by reusing existing ready output from
|
||||
// any row in the same document scope.
|
||||
if (!input.forceToken?.trim()) {
|
||||
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||
if (reusableParsed) {
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
userIds: [input.userId],
|
||||
parseState: stringifyDocumentParseState(readyState),
|
||||
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const ready = scopedRows.find(isReadyRow);
|
||||
if (ready) {
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
|
|
@ -201,17 +267,21 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
const coordinator = [...scopedRows].sort((a, b) => a.userId.localeCompare(b.userId))[0];
|
||||
if (!coordinator) return;
|
||||
|
||||
const runningStateData: DocumentParseState = {
|
||||
status: 'running',
|
||||
const initialInflightStatus = input.initialStatus === 'running' ? 'running' : 'pending';
|
||||
const inflightStateData: DocumentParseState = {
|
||||
status: initialInflightStatus,
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
const runningState = stringifyDocumentParseState(runningStateData);
|
||||
const inflightState = stringifyDocumentParseState(inflightStateData);
|
||||
|
||||
const claimRows = (await db
|
||||
.update(documents)
|
||||
.set({
|
||||
parseState: runningState,
|
||||
parseState: inflightState,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
|
|
@ -234,12 +304,10 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
userIds: cohortUserIds,
|
||||
parseState: runningState,
|
||||
parseState: inflightState,
|
||||
});
|
||||
|
||||
const compute = await getCompute();
|
||||
let activeOpId: string | undefined;
|
||||
let activeJobId: string | undefined;
|
||||
let lastProgressWriteAt = 0;
|
||||
let lastSnapshotWriteAt = 0;
|
||||
let lastSnapshotStatus: 'pending' | 'running' | null = null;
|
||||
|
|
@ -279,6 +347,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
status: mappedStatus,
|
||||
progress: snapshot.progress ?? null,
|
||||
updatedAt: now,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
|
|
@ -305,6 +374,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
phase: progress.phase,
|
||||
},
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
|
|
@ -327,12 +397,17 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
|
||||
}
|
||||
|
||||
const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey);
|
||||
|
||||
const finalScopedRows = await loadScopedRows(input);
|
||||
const finalUserIds = userIdsFromRows(finalScopedRows);
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
|
||||
await updateParseStateForUsers({
|
||||
|
|
@ -385,6 +460,9 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
error: message,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
|
||||
await updateParseStateForUsers({
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ export interface BaseDocument {
|
|||
parsedJsonKey?: string | null;
|
||||
scope?: 'user';
|
||||
folderId?: string;
|
||||
isConverting?: boolean;
|
||||
}
|
||||
|
||||
export interface PDFDocument extends BaseDocument {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
import { Page, expect, type TestInfo, type Locator } from '@playwright/test';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
const DIR = './tests/files/';
|
||||
|
|
@ -10,14 +8,6 @@ function escapeRegExp(input: string) {
|
|||
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function fixturePath(fileName: string) {
|
||||
return path.join(__dirname, 'files', fileName);
|
||||
}
|
||||
|
||||
function sha256HexOfFile(filePath: string) {
|
||||
return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
||||
}
|
||||
|
||||
async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
||||
await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) });
|
||||
const loader = page.getByTestId('pdf-status-loader').first();
|
||||
|
|
@ -40,7 +30,7 @@ async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Upload a sample epub or pdf
|
||||
* Upload a sample document fixture
|
||||
*/
|
||||
export async function uploadFile(page: Page, filePath: string) {
|
||||
const input = page.locator('input[type=file]').first();
|
||||
|
|
@ -50,7 +40,7 @@ export async function uploadFile(page: Page, filePath: string) {
|
|||
await input.setInputFiles(`${DIR}${filePath}`);
|
||||
|
||||
// Wait for the uploader to finish processing. The input is disabled while
|
||||
// uploading/converting via react-dropzone's `disabled` prop.
|
||||
// uploading via react-dropzone's `disabled` prop.
|
||||
// Tolerate extremely fast operations where the disabled state may be missed.
|
||||
try {
|
||||
await expect(input).toBeDisabled({ timeout: 2000 });
|
||||
|
|
@ -69,14 +59,8 @@ export async function uploadAndDisplay(page: Page, fileName: string) {
|
|||
const lower = fileName.toLowerCase();
|
||||
|
||||
if (lower.endsWith('.docx')) {
|
||||
// Best-effort: conversion can complete before we observe this UI state.
|
||||
try {
|
||||
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
const expectedId = sha256HexOfFile(fixturePath(fileName));
|
||||
const targetLink = page.locator(`a[href$="/pdf/${expectedId}"]`).first();
|
||||
const convertedName = `${fileName.replace(/\.[^.]+$/, '')}.pdf`;
|
||||
const targetLink = page.getByRole('link', { name: new RegExp(escapeRegExp(convertedName), 'i') }).first();
|
||||
await expect(targetLink).toBeVisible({ timeout: 15000 });
|
||||
await dismissOnboardingModals(page);
|
||||
await targetLink.click();
|
||||
|
|
@ -188,10 +172,11 @@ async function dismissOnboardingModals(page: Page): Promise<void> {
|
|||
* Wait for the play button to be clickable and click it
|
||||
*/
|
||||
export async function waitAndClickPlay(page: Page) {
|
||||
// Wait for play button selector without disabled attribute
|
||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();
|
||||
const playButton = page.getByRole('button', { name: 'Play' });
|
||||
await expect(playButton).toBeVisible();
|
||||
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||
// Play the TTS by clicking the button
|
||||
await page.getByRole('button', { name: 'Play' }).click();
|
||||
await playButton.click();
|
||||
// Use resilient processing transition helper (tolerates fast completion)
|
||||
await expectProcessingTransition(page);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => {
|
|||
await page.waitForTimeout(750);
|
||||
|
||||
// After resize, playback should have paused (Play button visible)
|
||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });
|
||||
const playButton = page.getByRole('button', { name: 'Play' });
|
||||
await expect(playButton).toBeVisible({ timeout: 15000 });
|
||||
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||
|
||||
// Resume playback and ensure processing -> playing
|
||||
await page.getByRole('button', { name: 'Play' }).click();
|
||||
await playButton.click();
|
||||
await expectProcessingTransition(page);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
150
tests/unit/blob-upload-finalize-docx.vitest.spec.ts
Normal file
150
tests/unit/blob-upload-finalize-docx.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
requireAuthContext: vi.fn(),
|
||||
registerUploadedDocument: vi.fn(),
|
||||
convertDocxBufferToPdfBuffer: vi.fn(),
|
||||
headTempDocumentBlob: vi.fn(),
|
||||
getTempDocumentBlob: vi.fn(),
|
||||
getTempDocumentFinalizeReceipt: vi.fn(),
|
||||
putTempDocumentFinalizeReceipt: vi.fn(),
|
||||
deleteTempDocumentUpload: vi.fn(),
|
||||
headDocumentBlob: vi.fn(),
|
||||
copyTempDocumentBlobToDocument: vi.fn(),
|
||||
putDocumentBlob: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/auth/auth', () => ({
|
||||
requireAuthContext: hoisted.requireAuthContext,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/register-upload', () => ({
|
||||
registerUploadedDocument: hoisted.registerUploadedDocument,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/docx-convert', () => ({
|
||||
convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||
TEMP_DOCUMENT_UPLOAD_TTL_MS: 24 * 60 * 60 * 1000,
|
||||
copyTempDocumentBlobToDocument: hoisted.copyTempDocumentBlobToDocument,
|
||||
deleteTempDocumentUpload: hoisted.deleteTempDocumentUpload,
|
||||
getTempDocumentBlob: hoisted.getTempDocumentBlob,
|
||||
getTempDocumentFinalizeReceipt: hoisted.getTempDocumentFinalizeReceipt,
|
||||
headDocumentBlob: hoisted.headDocumentBlob,
|
||||
headTempDocumentBlob: hoisted.headTempDocumentBlob,
|
||||
isMissingBlobError: vi.fn((error: unknown) => {
|
||||
const maybe = error as { code?: string } | undefined;
|
||||
return maybe?.code === 'NoSuchKey';
|
||||
}),
|
||||
isPreconditionFailed: vi.fn(() => false),
|
||||
isValidTempUploadToken: vi.fn(() => true),
|
||||
putDocumentBlob: hoisted.putDocumentBlob,
|
||||
putTempDocumentFinalizeReceipt: hoisted.putTempDocumentFinalizeReceipt,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/testing/test-namespace', () => ({
|
||||
getOpenReaderTestNamespace: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/s3', () => ({
|
||||
isS3Configured: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/logger', () => ({
|
||||
errorToLog: vi.fn((error: unknown) => error),
|
||||
serverLogger: {
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
describe('POST /api/documents/blob/upload/finalize DOCX flow', () => {
|
||||
beforeEach(() => {
|
||||
hoisted.requireAuthContext.mockReset();
|
||||
hoisted.registerUploadedDocument.mockReset();
|
||||
hoisted.convertDocxBufferToPdfBuffer.mockReset();
|
||||
hoisted.headTempDocumentBlob.mockReset();
|
||||
hoisted.getTempDocumentBlob.mockReset();
|
||||
hoisted.getTempDocumentFinalizeReceipt.mockReset();
|
||||
hoisted.putTempDocumentFinalizeReceipt.mockReset();
|
||||
hoisted.deleteTempDocumentUpload.mockReset();
|
||||
hoisted.headDocumentBlob.mockReset();
|
||||
hoisted.copyTempDocumentBlobToDocument.mockReset();
|
||||
hoisted.putDocumentBlob.mockReset();
|
||||
|
||||
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||
hoisted.getTempDocumentFinalizeReceipt.mockResolvedValue(null);
|
||||
hoisted.headTempDocumentBlob.mockResolvedValue({
|
||||
contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
contentLength: 12,
|
||||
lastModified: Date.now(),
|
||||
});
|
||||
hoisted.getTempDocumentBlob.mockResolvedValue(Buffer.from('docx-bytes'));
|
||||
hoisted.convertDocxBufferToPdfBuffer.mockResolvedValue(Buffer.from('pdf-bytes'));
|
||||
hoisted.headDocumentBlob
|
||||
.mockRejectedValueOnce({ code: 'NoSuchKey' })
|
||||
.mockResolvedValue({
|
||||
contentLength: Buffer.byteLength('pdf-bytes'),
|
||||
contentType: 'application/pdf',
|
||||
eTag: 'etag-1',
|
||||
});
|
||||
hoisted.registerUploadedDocument.mockImplementation(async (input: { documentId: string; name: string; type: string; size: number; lastModified: number }) => ({
|
||||
id: input.documentId,
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
size: input.size,
|
||||
lastModified: input.lastModified,
|
||||
scope: 'user',
|
||||
}));
|
||||
hoisted.putTempDocumentFinalizeReceipt.mockResolvedValue(undefined);
|
||||
hoisted.deleteTempDocumentUpload.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
test('converts raw DOCX during finalize and registers a PDF', async () => {
|
||||
const { POST } = await import('../../src/app/api/documents/blob/upload/finalize/route');
|
||||
const request = new NextRequest('http://localhost/api/documents/blob/upload/finalize', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
uploads: [{
|
||||
token: '123e4567-e89b-12d3-a456-426614174000',
|
||||
name: 'Report.docx',
|
||||
type: 'docx',
|
||||
lastModified: 1700000000000,
|
||||
}],
|
||||
}),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
const response = await POST(request);
|
||||
const expectedId = createHash('sha256').update(Buffer.from('pdf-bytes')).digest('hex');
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
stored: [{
|
||||
id: expectedId,
|
||||
name: 'Report.pdf',
|
||||
type: 'pdf',
|
||||
}],
|
||||
});
|
||||
expect(hoisted.convertDocxBufferToPdfBuffer).toHaveBeenCalledTimes(1);
|
||||
expect(hoisted.putDocumentBlob).toHaveBeenCalledWith(
|
||||
expectedId,
|
||||
Buffer.from('pdf-bytes'),
|
||||
'application/pdf',
|
||||
null,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
expect(hoisted.copyTempDocumentBlobToDocument).not.toHaveBeenCalled();
|
||||
expect(hoisted.registerUploadedDocument).toHaveBeenCalledWith(expect.objectContaining({
|
||||
documentId: expectedId,
|
||||
name: 'Report.pdf',
|
||||
type: 'pdf',
|
||||
}));
|
||||
});
|
||||
});
|
||||
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import { buildPdfOpKey } from '../../src/lib/server/compute/worker';
|
||||
|
||||
describe('pdf worker op key', () => {
|
||||
test('includes parser version in the reuse boundary', () => {
|
||||
expect(buildPdfOpKey({
|
||||
documentId: 'doc-1',
|
||||
namespace: null,
|
||||
documentObjectKey: 'documents/doc-1.pdf',
|
||||
})).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`);
|
||||
});
|
||||
});
|
||||
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '../../src/lib/server/documents/parse-state';
|
||||
|
||||
describe('document parse state parser-version handling', () => {
|
||||
test('preserves parserVersion through stringify/parse', () => {
|
||||
const state = parseDocumentParseState(stringifyDocumentParseState({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1234,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
}));
|
||||
|
||||
expect(state).toEqual({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1234,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
});
|
||||
});
|
||||
|
||||
test('invalidates ready and inflight states from older parser versions', () => {
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
parserVersion: 'old-parser',
|
||||
opId: 'op-old',
|
||||
}, 2222)).toEqual({
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: 2222,
|
||||
});
|
||||
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'running',
|
||||
progress: {
|
||||
totalPages: 10,
|
||||
pagesParsed: 3,
|
||||
currentPage: 4,
|
||||
phase: 'infer',
|
||||
},
|
||||
updatedAt: 1111,
|
||||
parserVersion: 'old-parser',
|
||||
opId: 'op-old',
|
||||
}, 3333)).toEqual({
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: 3333,
|
||||
});
|
||||
});
|
||||
|
||||
test('keeps failed states and current-version ready states intact', () => {
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
error: 'no text layer',
|
||||
parserVersion: 'old-parser',
|
||||
})).toEqual({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
error: 'no text layer',
|
||||
parserVersion: 'old-parser',
|
||||
});
|
||||
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
})).toEqual({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
db: null as {
|
||||
select: ReturnType<typeof vi.fn>;
|
||||
update: ReturnType<typeof vi.fn>;
|
||||
} | null,
|
||||
row: {
|
||||
id: 'doc-1',
|
||||
userId: 'user-1',
|
||||
parseState: null as string | null,
|
||||
},
|
||||
requireAuthContext: vi.fn(),
|
||||
backfillPendingPdfParseOperation: vi.fn(),
|
||||
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
|
||||
}));
|
||||
|
||||
vi.mock('@/db', () => ({
|
||||
get db() {
|
||||
return hoisted.db;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/auth/auth', () => ({
|
||||
requireAuthContext: hoisted.requireAuthContext,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/parse-state-backfill', () => ({
|
||||
backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
|
||||
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||
isValidDocumentId: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/s3', () => ({
|
||||
isS3Configured: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/testing/test-namespace', () => ({
|
||||
getOpenReaderTestNamespace: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/logger', () => ({
|
||||
createRequestLogger: vi.fn(() => ({
|
||||
logger: {
|
||||
warn: vi.fn(),
|
||||
info: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
requestId: 'req-test',
|
||||
})),
|
||||
hashForLog: vi.fn(() => 'user-hash'),
|
||||
}));
|
||||
|
||||
describe('GET /api/documents/[id]/parsed/events legacy backfill', () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.COMPUTE_WORKER_URL = 'http://localhost:4010';
|
||||
process.env.COMPUTE_WORKER_TOKEN = 'worker-test-token';
|
||||
|
||||
hoisted.row = {
|
||||
id: 'doc-1',
|
||||
userId: 'user-1',
|
||||
parseState: null,
|
||||
};
|
||||
hoisted.db = {
|
||||
select: vi.fn(() => ({
|
||||
from: vi.fn(() => ({
|
||||
where: vi.fn(async () => ([{ ...hoisted.row }])),
|
||||
})),
|
||||
})),
|
||||
update: vi.fn(() => ({
|
||||
set: vi.fn((values: { parseState?: string | null }) => ({
|
||||
where: vi.fn(async () => {
|
||||
if (typeof values.parseState !== 'undefined') {
|
||||
hoisted.row.parseState = values.parseState;
|
||||
}
|
||||
return [];
|
||||
}),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
hoisted.requireAuthContext.mockReset();
|
||||
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||
hoisted.backfillPendingPdfParseOperation.mockReset();
|
||||
hoisted.healStaleDocumentParseState.mockClear();
|
||||
|
||||
global.fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_, reject) => {
|
||||
const signal = init?.signal;
|
||||
if (signal) {
|
||||
signal.addEventListener('abort', () => {
|
||||
reject(new DOMException('Aborted', 'AbortError'));
|
||||
}, { once: true });
|
||||
}
|
||||
})) as typeof fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test('creates a worker op for legacy pending PDFs without opId', async () => {
|
||||
hoisted.backfillPendingPdfParseOperation.mockResolvedValue({
|
||||
opId: 'op-legacy-1',
|
||||
opKey: 'pdf_layout|v1|doc-1||doc-1|',
|
||||
jobId: 'job-legacy-1',
|
||||
kind: 'pdf_layout',
|
||||
status: 'queued',
|
||||
queuedAt: Date.now(),
|
||||
progress: null,
|
||||
result: undefined,
|
||||
error: undefined,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
|
||||
const { GET } = await import('../../src/app/api/documents/[id]/parsed/events/route');
|
||||
const controller = new AbortController();
|
||||
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed/events', {
|
||||
signal: controller.signal,
|
||||
});
|
||||
const response = await GET(request, {
|
||||
params: Promise.resolve({ id: 'doc-1' }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const reader = response.body?.getReader();
|
||||
expect(reader).toBeDefined();
|
||||
|
||||
const first = await reader!.read();
|
||||
const chunk = new TextDecoder().decode(first.value);
|
||||
expect(chunk).toContain('event: snapshot');
|
||||
expect(chunk).toContain('"parseStatus":"pending"');
|
||||
expect(chunk).toContain('"opId":"op-legacy-1"');
|
||||
|
||||
controller.abort();
|
||||
await reader!.cancel().catch(() => {});
|
||||
|
||||
expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({
|
||||
documentId: 'doc-1',
|
||||
userId: 'user-1',
|
||||
namespace: null,
|
||||
state: expect.objectContaining({ status: 'pending' }),
|
||||
}));
|
||||
const parseState = String(hoisted.row.parseState ?? '');
|
||||
expect(parseState).toContain('"status":"pending"');
|
||||
expect(parseState).toContain('"opId":"op-legacy-1"');
|
||||
expect(parseState).toContain('"jobId":"job-legacy-1"');
|
||||
});
|
||||
});
|
||||
|
|
@ -14,8 +14,9 @@ const hoisted = vi.hoisted(() => ({
|
|||
},
|
||||
requireAuthContext: vi.fn(),
|
||||
fetchWorkerOperationState: vi.fn(),
|
||||
backfillPendingPdfParseOperation: vi.fn(),
|
||||
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
|
||||
startPdfParseOperation: vi.fn(),
|
||||
enqueueParsePdfJob: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/db', () => ({
|
||||
|
|
@ -32,14 +33,18 @@ vi.mock('@/lib/server/compute/worker-op-state', () => ({
|
|||
fetchWorkerOperationState: hoisted.fetchWorkerOperationState,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/parse-state-backfill', () => ({
|
||||
backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
|
||||
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/pdf-parse-operation', () => ({
|
||||
startPdfParseOperation: hoisted.startPdfParseOperation,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/jobs/user-pdf-layout-job', () => ({
|
||||
enqueueParsePdfJob: hoisted.enqueueParsePdfJob,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||
documentKey: vi.fn(),
|
||||
getParsedDocumentBlob: vi.fn(),
|
||||
|
|
@ -69,7 +74,7 @@ vi.mock('@/lib/server/logger', () => ({
|
|||
hashForLog: vi.fn(() => 'user-hash'),
|
||||
}));
|
||||
|
||||
describe('GET /api/documents/[id]/parsed legacy backfill', () => {
|
||||
describe('GET /api/documents/[id]/parsed pure data fetch', () => {
|
||||
beforeEach(async () => {
|
||||
process.env.BASE_URL = 'http://localhost:3003';
|
||||
process.env.AUTH_SECRET = 'test-secret';
|
||||
|
|
@ -104,44 +109,25 @@ describe('GET /api/documents/[id]/parsed legacy backfill', () => {
|
|||
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||
hoisted.fetchWorkerOperationState.mockReset();
|
||||
hoisted.fetchWorkerOperationState.mockResolvedValue(null);
|
||||
hoisted.backfillPendingPdfParseOperation.mockReset();
|
||||
hoisted.healStaleDocumentParseState.mockClear();
|
||||
hoisted.startPdfParseOperation.mockReset();
|
||||
hoisted.enqueueParsePdfJob.mockReset();
|
||||
});
|
||||
|
||||
test('creates a worker op for legacy pending PDFs without opId', async () => {
|
||||
hoisted.backfillPendingPdfParseOperation.mockResolvedValue({
|
||||
opId: 'op-legacy-1',
|
||||
opKey: 'pdf_layout|v1|doc-1||doc-1|',
|
||||
jobId: 'job-legacy-1',
|
||||
kind: 'pdf_layout',
|
||||
status: 'queued',
|
||||
queuedAt: Date.now(),
|
||||
progress: null,
|
||||
result: undefined,
|
||||
error: undefined,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
|
||||
test('returns non-ready status without creating a worker op for legacy pending PDFs without opId', async () => {
|
||||
const { GET } = await import('../../src/app/api/documents/[id]/parsed/route');
|
||||
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed');
|
||||
const response = await GET(request, {
|
||||
params: Promise.resolve({ id: 'doc-1' }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(202);
|
||||
expect(response.status).toBe(409);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
parseStatus: 'pending',
|
||||
opId: 'op-legacy-1',
|
||||
opId: null,
|
||||
});
|
||||
expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({
|
||||
documentId: 'doc-1',
|
||||
userId: 'user-1',
|
||||
namespace: null,
|
||||
state: expect.objectContaining({ status: 'pending' }),
|
||||
}));
|
||||
const parseState = String(hoisted.row.parseState ?? '');
|
||||
expect(parseState).toContain('"status":"pending"');
|
||||
expect(parseState).toContain('"opId":"op-legacy-1"');
|
||||
expect(parseState).toContain('"jobId":"job-legacy-1"');
|
||||
expect(hoisted.startPdfParseOperation).not.toHaveBeenCalled();
|
||||
expect(hoisted.enqueueParsePdfJob).not.toHaveBeenCalled();
|
||||
expect(hoisted.row.parseState).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
import {
|
||||
documentParseStateFromWorkerState,
|
||||
|
|
@ -11,7 +12,7 @@ function makeWorkerState(
|
|||
): WorkerOperationState<PdfLayoutJobResult> {
|
||||
return {
|
||||
opId: 'op-123',
|
||||
opKey: 'pdf_layout|v1|doc-1',
|
||||
opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`,
|
||||
kind: 'pdf_layout',
|
||||
jobId: 'job-123',
|
||||
status: 'queued',
|
||||
|
|
@ -41,6 +42,7 @@ describe('worker parse state mapping', () => {
|
|||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: 1234,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
opId: 'op-123',
|
||||
jobId: 'job-123',
|
||||
});
|
||||
|
|
@ -66,6 +68,7 @@ describe('worker parse state mapping', () => {
|
|||
phase: 'infer',
|
||||
},
|
||||
updatedAt: 5678,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
opId: 'op-123',
|
||||
jobId: 'job-123',
|
||||
});
|
||||
|
|
@ -84,6 +87,7 @@ describe('worker parse state mapping', () => {
|
|||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: 9999,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
opId: 'op-123',
|
||||
jobId: 'job-123',
|
||||
error: 'layout model crashed',
|
||||
|
|
|
|||
|
|
@ -30,6 +30,29 @@ test.describe('Document Upload Tests', () => {
|
|||
await expectDocumentListed(page, 'sample.txt');
|
||||
});
|
||||
|
||||
test('reuses the same canonical id for identical uploads', async ({ page }) => {
|
||||
await uploadFile(page, 'sample.pdf');
|
||||
await uploadFile(page, 'sample.pdf');
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const res = await fetch('/api/documents', { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
return { ok: false as const, reason: `status:${res.status}` };
|
||||
}
|
||||
const data = await res.json() as { documents?: Array<{ id: string; name: string }> };
|
||||
const matching = (data.documents || []).filter((doc) => doc.name === 'sample.pdf');
|
||||
const uniqueIds = Array.from(new Set(matching.map((doc) => doc.id)));
|
||||
return { ok: true as const, matchingCount: matching.length, uniqueIds };
|
||||
});
|
||||
|
||||
expect(result.ok).toBeTruthy();
|
||||
if (!result.ok) {
|
||||
throw new Error(`Failed to inspect uploaded documents: ${result.reason}`);
|
||||
}
|
||||
expect(result.matchingCount).toBe(1);
|
||||
expect(result.uniqueIds).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('hashes text/HTML docs using UTF-8 encoded stored string', async ({ page }) => {
|
||||
await uploadFile(page, 'sample.txt');
|
||||
await expectDocumentListed(page, 'sample.txt');
|
||||
|
|
@ -77,13 +100,7 @@ test.describe('Document Upload Tests', () => {
|
|||
|
||||
test('uploads and converts a DOCX document', async ({ page }) => {
|
||||
await uploadFile(page, 'sample.docx');
|
||||
// Should see the converting message (best-effort; conversion may complete extremely fast)
|
||||
try {
|
||||
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
// After conversion, should see the PDF with the same name
|
||||
// DOCX uploads are normalized into stored PDFs with the same basename.
|
||||
await expectDocumentListed(page, 'sample.pdf');
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue