Merge pull request #99 from richardr1126/pdf/parse-fixes
## Summary by CodeRabbit * **New Features** * Expose PDF parser version system-wide; parser-version-aware parsing, reuse, and caching. Added tokenized temp upload flow, presign/finalize endpoints, and server-side DOCX→PDF conversion. * Playback readiness propagated to TTS players/pages; unified upload/delete APIs and simplified uploader/sidebar UX. * **Bug Fixes** * Normalize stored parse state to current parser version to avoid stale readiness and improve reuse. * Prevent premature playback starts and redundant preparation. * **Tests** * Added unit and E2E tests for parser-version handling, op-key inclusion, parse-state behavior, and canonical upload reuse.
This commit is contained in:
commit
e2c0240add
46 changed files with 1786 additions and 1049 deletions
|
|
@ -16,6 +16,7 @@ export {
|
||||||
} from './config/timeout';
|
} from './config/timeout';
|
||||||
export { renderPage } from './pdf/render';
|
export { renderPage } from './pdf/render';
|
||||||
export { mergeTextWithRegions } from './pdf/merge';
|
export { mergeTextWithRegions } from './pdf/merge';
|
||||||
|
export { PDF_PARSER_VERSION } from './pdf/parser-version';
|
||||||
export { stitchCrossPageBlocks } from './pdf/stitch';
|
export { stitchCrossPageBlocks } from './pdf/stitch';
|
||||||
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
||||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import { ensureModel } from './model';
|
||||||
import { runLayoutModel } from './runLayoutModel';
|
import { runLayoutModel } from './runLayoutModel';
|
||||||
import { mergeTextWithRegions } from './merge';
|
import { mergeTextWithRegions } from './merge';
|
||||||
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||||
|
import { PDF_PARSER_VERSION } from './parser-version';
|
||||||
import { stitchCrossPageBlocks } from './stitch';
|
import { stitchCrossPageBlocks } from './stitch';
|
||||||
import { renderPage } from './render';
|
import { renderPage } from './render';
|
||||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||||
|
|
@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
|
||||||
const doc: ParsedPdfDocument = {
|
const doc: ParsedPdfDocument = {
|
||||||
schemaVersion: 1,
|
schemaVersion: 1,
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69',
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
parsedAt: Date.now(),
|
parsedAt: Date.now(),
|
||||||
pages,
|
pages,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
1
compute/core/src/pdf/parser-version.ts
Normal file
1
compute/core/src/pdf/parser-version.ts
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||||
|
|
@ -1,12 +1,15 @@
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
let cachedStandardFontDataUrl: string | null = null;
|
let cachedStandardFontDataUrl: string | null = null;
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
export function resolvePdfjsStandardFontDataUrl(): string {
|
export function resolvePdfjsStandardFontDataUrl(): string {
|
||||||
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
||||||
|
|
||||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json'));
|
||||||
|
const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts');
|
||||||
|
|
||||||
if (!fs.existsSync(standardFontDir)) {
|
if (!fs.existsSync(standardFontDir)) {
|
||||||
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ export default function EPUBPage() {
|
||||||
const {
|
const {
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
currDocName,
|
currDocName,
|
||||||
|
isPlaybackReady,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
createFullAudioBook: createEPUBAudioBook,
|
createFullAudioBook: createEPUBAudioBook,
|
||||||
regenerateChapter: regenerateEPUBChapter,
|
regenerateChapter: regenerateEPUBChapter,
|
||||||
|
|
@ -242,7 +243,7 @@ export default function EPUBPage() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<TTSPlayer />
|
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||||
)}
|
)}
|
||||||
<DocumentSettings
|
<DocumentSettings
|
||||||
epub
|
epub
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,7 @@ export interface EpubDocumentState {
|
||||||
currDocPages: number | undefined;
|
currDocPages: number | undefined;
|
||||||
currDocPage: number | string;
|
currDocPage: number | string;
|
||||||
currDocText: string | undefined;
|
currDocText: string | undefined;
|
||||||
|
isPlaybackReady: boolean;
|
||||||
setCurrentDocument: (id: string) => Promise<void>;
|
setCurrentDocument: (id: string) => Promise<void>;
|
||||||
clearCurrDoc: () => void;
|
clearCurrDoc: () => void;
|
||||||
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
|
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
|
||||||
|
|
@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
const [currDocText, setCurrDocText] = useState<string>();
|
const [currDocText, setCurrDocText] = useState<string>();
|
||||||
|
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||||
// Mirror state into a ref so resolveEpubLocator (registered once with
|
// Mirror state into a ref so resolveEpubLocator (registered once with
|
||||||
// TTSContext via a stable callback) can always read the latest page text
|
// TTSContext via a stable callback) can always read the latest page text
|
||||||
// without forcing re-registration on every page turn.
|
// without forcing re-registration on every page turn.
|
||||||
|
|
@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
isEPUBSetOnce.current = false;
|
isEPUBSetOnce.current = false;
|
||||||
shouldPauseRef.current = true;
|
shouldPauseRef.current = true;
|
||||||
|
|
@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
*/
|
*/
|
||||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||||
try {
|
try {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
const meta = await getDocumentMetadata(id);
|
const meta = await getDocumentMetadata(id);
|
||||||
if (!meta) {
|
if (!meta) {
|
||||||
clearCurrDoc();
|
clearCurrDoc();
|
||||||
|
|
@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
*/
|
*/
|
||||||
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
|
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
|
||||||
try {
|
try {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
const location = rendition?.location;
|
const location = rendition?.location;
|
||||||
if (!location) return '';
|
if (!location) return '';
|
||||||
const { start, end } = location;
|
const { start, end } = location;
|
||||||
|
|
@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
nextText: continuationPreview
|
nextText: continuationPreview
|
||||||
});
|
});
|
||||||
setCurrDocText(textContent);
|
setCurrDocText(textContent);
|
||||||
|
setIsPlaybackReady(true);
|
||||||
|
|
||||||
return textContent;
|
return textContent;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
extractPageText,
|
extractPageText,
|
||||||
walkUpcomingRenderedLocations,
|
walkUpcomingRenderedLocations,
|
||||||
|
|
@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
extractPageText,
|
extractPageText,
|
||||||
walkUpcomingRenderedLocations,
|
walkUpcomingRenderedLocations,
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ export default function HTMLPage() {
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocName,
|
currDocName,
|
||||||
|
isPlaybackReady,
|
||||||
blocks,
|
blocks,
|
||||||
isTxt,
|
isTxt,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
|
|
@ -233,7 +234,7 @@ export default function HTMLPage() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<TTSPlayer />
|
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||||
)}
|
)}
|
||||||
<DocumentSettings
|
<DocumentSettings
|
||||||
html
|
html
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ export interface HtmlDocumentState {
|
||||||
currDocData: string | undefined;
|
currDocData: string | undefined;
|
||||||
currDocName: string | undefined;
|
currDocName: string | undefined;
|
||||||
currDocText: string | undefined;
|
currDocText: string | undefined;
|
||||||
|
isPlaybackReady: boolean;
|
||||||
blocks: HtmlBlock[];
|
blocks: HtmlBlock[];
|
||||||
isTxt: boolean;
|
isTxt: boolean;
|
||||||
setCurrentDocument: (id: string) => Promise<void>;
|
setCurrentDocument: (id: string) => Promise<void>;
|
||||||
|
|
@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
|
|
||||||
const [currDocData, setCurrDocData] = useState<string>();
|
const [currDocData, setCurrDocData] = useState<string>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
|
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||||
|
|
||||||
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
|
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
|
||||||
const blocks = useMemo(
|
const blocks = useMemo(
|
||||||
|
|
@ -91,22 +93,43 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
// sentence splitting + sequential advancement from there.
|
// sentence splitting + sequential advancement from there.
|
||||||
const lastFedDocRef = useRef<string | null>(null);
|
const lastFedDocRef = useRef<string | null>(null);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!currDocText) return;
|
if (currDocData === undefined) {
|
||||||
const key = `${currDocData ?? ''}::${currDocText.length}`;
|
lastFedDocRef.current = null;
|
||||||
if (lastFedDocRef.current === key) return;
|
setTTSText('');
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!currDocText) {
|
||||||
|
lastFedDocRef.current = null;
|
||||||
|
setTTSText('');
|
||||||
|
setIsPlaybackReady(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const key = `${currDocName ?? ''}::${currDocData ?? ''}::${currDocText.length}`;
|
||||||
|
if (lastFedDocRef.current === key) {
|
||||||
|
setIsPlaybackReady(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setIsPlaybackReady(false);
|
||||||
lastFedDocRef.current = key;
|
lastFedDocRef.current = key;
|
||||||
setTTSText(currDocText);
|
setTTSText(currDocText);
|
||||||
}, [currDocText, currDocData, setTTSText]);
|
setIsPlaybackReady(true);
|
||||||
|
}, [currDocName, currDocText, currDocData, setTTSText]);
|
||||||
|
|
||||||
const clearCurrDoc = useCallback(() => {
|
const clearCurrDoc = useCallback(() => {
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
lastFedDocRef.current = null;
|
lastFedDocRef.current = null;
|
||||||
|
setTTSText('');
|
||||||
stop();
|
stop();
|
||||||
}, [stop]);
|
}, [setTTSText, stop]);
|
||||||
|
|
||||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||||
try {
|
try {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
lastFedDocRef.current = null;
|
||||||
|
setTTSText('');
|
||||||
const meta = await getDocumentMetadata(id);
|
const meta = await getDocumentMetadata(id);
|
||||||
if (!meta) {
|
if (!meta) {
|
||||||
console.error('Document not found on server');
|
console.error('Document not found on server');
|
||||||
|
|
@ -125,7 +148,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
console.error('Failed to get HTML document:', error);
|
console.error('Failed to get HTML document:', error);
|
||||||
clearCurrDoc();
|
clearCurrDoc();
|
||||||
}
|
}
|
||||||
}, [clearCurrDoc]);
|
}, [clearCurrDoc, setTTSText]);
|
||||||
|
|
||||||
const audiobookAdapter = useMemo(
|
const audiobookAdapter = useMemo(
|
||||||
() =>
|
() =>
|
||||||
|
|
@ -207,6 +230,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocName,
|
currDocName,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
blocks,
|
blocks,
|
||||||
isTxt,
|
isTxt,
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
|
|
@ -218,6 +242,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocName,
|
currDocName,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
blocks,
|
blocks,
|
||||||
isTxt,
|
isTxt,
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,7 @@ export default function PDFViewerPage() {
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocPages,
|
currDocPages,
|
||||||
|
isPlaybackReady,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
parseProgress,
|
parseProgress,
|
||||||
documentSettings,
|
documentSettings,
|
||||||
|
|
@ -487,7 +488,7 @@ export default function PDFViewerPage() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : isParseReady ? (
|
) : isParseReady ? (
|
||||||
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} />
|
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} isPlaybackReady={isPlaybackReady} />
|
||||||
) : null}
|
) : null}
|
||||||
<DocumentSettings
|
<DocumentSettings
|
||||||
isOpen={activeSidebar === 'settings'}
|
isOpen={activeSidebar === 'settings'}
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,7 @@ export interface PdfDocumentState {
|
||||||
currDocPages: number | undefined;
|
currDocPages: number | undefined;
|
||||||
currDocPage: number;
|
currDocPage: number;
|
||||||
currDocText: string | undefined;
|
currDocText: string | undefined;
|
||||||
|
isPlaybackReady: boolean;
|
||||||
pdfDocument: PDFDocumentProxy | undefined;
|
pdfDocument: PDFDocumentProxy | undefined;
|
||||||
parsedDocument: ParsedPdfDocument | null;
|
parsedDocument: ParsedPdfDocument | null;
|
||||||
parseStatus: PdfParseStatus | null;
|
parseStatus: PdfParseStatus | null;
|
||||||
|
|
@ -142,6 +143,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
const [currDocText, setCurrDocText] = useState<string>();
|
const [currDocText, setCurrDocText] = useState<string>();
|
||||||
|
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||||
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
||||||
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
||||||
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
||||||
|
|
@ -167,52 +169,29 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
// Guards for setCurrentDocument to prevent stale loads from overwriting newer selections.
|
// Guards for setCurrentDocument to prevent stale loads from overwriting newer selections.
|
||||||
const docLoadSeqRef = useRef(0);
|
const docLoadSeqRef = useRef(0);
|
||||||
const docLoadAbortRef = useRef<AbortController | null>(null);
|
const docLoadAbortRef = useRef<AbortController | null>(null);
|
||||||
const parsePollAbortRef = useRef<AbortController | null>(null);
|
const parseStreamAbortRef = useRef<AbortController | null>(null);
|
||||||
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
||||||
|
const lastPreparedPlaybackPageRef = useRef<number | null>(null);
|
||||||
|
|
||||||
const fetchParsedDocument = useCallback(async (
|
const loadParsedDocumentOnce = useCallback(async (
|
||||||
documentId: string,
|
documentId: string,
|
||||||
initialStatus: PdfParseStatus | null,
|
|
||||||
signal: AbortSignal,
|
signal: AbortSignal,
|
||||||
initialOpId?: string | null,
|
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
// Legacy PDFs may have null parseStatus; treat as pending so opening the
|
if (signal.aborted) return;
|
||||||
// document backfills parse output via the parsed endpoint polling path.
|
const parsed = await getParsedPdfDocument(documentId, { signal });
|
||||||
const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending';
|
if (signal.aborted) return;
|
||||||
setParseStatus(effectiveInitialStatus);
|
setParsedDocument(parsed);
|
||||||
|
setParseStatus('ready');
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
const delayMs = 1200;
|
setActiveParseOpId(null);
|
||||||
const retryFailed = effectiveInitialStatus === 'failed';
|
}, []);
|
||||||
let attempt = 0;
|
|
||||||
let effectiveOpId = initialOpId?.trim() || null;
|
const resetParsedDocumentState = useCallback(() => {
|
||||||
while (!signal.aborted) {
|
setParsedDocument(null);
|
||||||
if (signal.aborted) return;
|
setCurrDocText(undefined);
|
||||||
const result = await getParsedPdfDocument(documentId, {
|
setIsPlaybackReady(false);
|
||||||
signal,
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
retryFailed: retryFailed && attempt === 0,
|
pageTextCacheRef.current.clear();
|
||||||
...(effectiveOpId ? { opId: effectiveOpId } : {}),
|
|
||||||
});
|
|
||||||
if (result.status === 'ready') {
|
|
||||||
setParsedDocument(result.parsed);
|
|
||||||
setParseStatus('ready');
|
|
||||||
setParseProgress(null);
|
|
||||||
setActiveParseOpId(null);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if ('opId' in result && typeof result.opId === 'string' && result.opId.trim()) {
|
|
||||||
effectiveOpId = result.opId.trim();
|
|
||||||
setActiveParseOpId(effectiveOpId);
|
|
||||||
}
|
|
||||||
setParseStatus(result.status);
|
|
||||||
setParseProgress(result.parseProgress ?? null);
|
|
||||||
if (result.status === 'failed') {
|
|
||||||
setParsedDocument(null);
|
|
||||||
setActiveParseOpId(null);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
attempt += 1;
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
||||||
}
|
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise<void> => {
|
const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise<void> => {
|
||||||
|
|
@ -226,42 +205,57 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const startParsedPolling = useCallback((documentId: string, initialOpId?: string | null) => {
|
const startParsedEventStream = useCallback((documentId: string, initialOpId?: string | null) => {
|
||||||
parsePollAbortRef.current?.abort();
|
parseStreamAbortRef.current?.abort();
|
||||||
parseSseCloseRef.current?.();
|
parseSseCloseRef.current?.();
|
||||||
parseSseCloseRef.current = null;
|
parseSseCloseRef.current = null;
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
setActiveParseOpId(initialOpId?.trim() || null);
|
setActiveParseOpId(initialOpId?.trim() || null);
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
parsePollAbortRef.current = controller;
|
parseStreamAbortRef.current = controller;
|
||||||
|
let isResolvingTerminalState = false;
|
||||||
|
|
||||||
const closeSse = subscribeParsedPdfDocumentEvents(documentId, {
|
const closeSse = subscribeParsedPdfDocumentEvents(documentId, {
|
||||||
opId: initialOpId?.trim() || null,
|
opId: initialOpId?.trim() || null,
|
||||||
}, {
|
}, {
|
||||||
onSnapshot: (snapshot) => {
|
onSnapshot: (snapshot) => {
|
||||||
if (controller.signal.aborted) return;
|
if (controller.signal.aborted) return;
|
||||||
|
if (isResolvingTerminalState) return;
|
||||||
if (typeof snapshot.opId === 'string' && snapshot.opId.trim()) {
|
if (typeof snapshot.opId === 'string' && snapshot.opId.trim()) {
|
||||||
setActiveParseOpId(snapshot.opId.trim());
|
setActiveParseOpId(snapshot.opId.trim());
|
||||||
}
|
}
|
||||||
setParseStatus(snapshot.parseStatus);
|
setParseStatus(snapshot.parseStatus);
|
||||||
setParseProgress(snapshot.parseProgress);
|
setParseProgress(snapshot.parseProgress);
|
||||||
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
|
if (snapshot.parseStatus === 'ready') {
|
||||||
if (snapshot.parseStatus === 'failed') {
|
isResolvingTerminalState = true;
|
||||||
setParsedDocument(null);
|
void (async () => {
|
||||||
setActiveParseOpId(null);
|
try {
|
||||||
} else {
|
await loadParsedDocumentOnce(documentId, controller.signal);
|
||||||
void fetchParsedDocument(
|
} catch (error) {
|
||||||
documentId,
|
if (error instanceof DOMException && error.name === 'AbortError') return;
|
||||||
'ready',
|
console.error('Failed to load parsed PDF after ready status:', error);
|
||||||
controller.signal,
|
resetParsedDocumentState();
|
||||||
typeof snapshot.opId === 'string' ? snapshot.opId : (initialOpId ?? null),
|
} finally {
|
||||||
);
|
if (parseSseCloseRef.current === closeSse) {
|
||||||
}
|
closeSse();
|
||||||
|
parseSseCloseRef.current = null;
|
||||||
|
}
|
||||||
|
if (parseStreamAbortRef.current === controller) {
|
||||||
|
parseStreamAbortRef.current = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (snapshot.parseStatus === 'failed') {
|
||||||
|
isResolvingTerminalState = true;
|
||||||
closeSse();
|
closeSse();
|
||||||
parseSseCloseRef.current = null;
|
parseSseCloseRef.current = null;
|
||||||
if (parsePollAbortRef.current === controller) {
|
if (parseStreamAbortRef.current === controller) {
|
||||||
parsePollAbortRef.current = null;
|
parseStreamAbortRef.current = null;
|
||||||
}
|
}
|
||||||
|
resetParsedDocumentState();
|
||||||
|
setActiveParseOpId(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -281,11 +275,11 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
if (parseSseCloseRef.current === closeSse) {
|
if (parseSseCloseRef.current === closeSse) {
|
||||||
parseSseCloseRef.current = null;
|
parseSseCloseRef.current = null;
|
||||||
}
|
}
|
||||||
if (parsePollAbortRef.current === controller) {
|
if (parseStreamAbortRef.current === controller) {
|
||||||
parsePollAbortRef.current = null;
|
parseStreamAbortRef.current = null;
|
||||||
}
|
}
|
||||||
}, { once: true });
|
}, { once: true });
|
||||||
}, [fetchParsedDocument, setActiveParseOpId]);
|
}, [loadParsedDocumentOnce, resetParsedDocumentState, setActiveParseOpId]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
pdfDocumentRef.current = pdfDocument;
|
pdfDocumentRef.current = pdfDocument;
|
||||||
|
|
@ -297,6 +291,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setCurrDocPage(currDocPageNumber);
|
setCurrDocPage(currDocPageNumber);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
}, [currDocPageNumber]);
|
}, [currDocPageNumber]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -324,11 +319,13 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
if (!currentPdf) return;
|
if (!currentPdf) return;
|
||||||
const seq = ++loadSeqRef.current;
|
const seq = ++loadSeqRef.current;
|
||||||
const pageNumber = currDocPageNumber;
|
const pageNumber = currDocPageNumber;
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
|
||||||
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
|
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
|
||||||
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
|
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
|
||||||
|
|
||||||
if (parseStatus !== 'ready' || !parsedDocument) {
|
if (parseStatus !== 'ready' || !parsedDocument) {
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
setTTSText('', { location: currDocPageNumber });
|
setTTSText('', { location: currDocPageNumber });
|
||||||
return;
|
return;
|
||||||
|
|
@ -401,7 +398,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (text !== currDocText || text === '') {
|
const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber;
|
||||||
|
if (shouldPreparePlayback) {
|
||||||
setCurrDocText(text);
|
setCurrDocText(text);
|
||||||
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
|
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
|
||||||
setTTSText(text, {
|
setTTSText(text, {
|
||||||
|
|
@ -414,6 +412,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
|
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
lastPreparedPlaybackPageRef.current = currDocPageNumber;
|
||||||
|
setIsPlaybackReady(true);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||||
return;
|
return;
|
||||||
|
|
@ -461,14 +461,16 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
// or fast refresh.
|
// or fast refresh.
|
||||||
pdfDocGenerationRef.current += 1;
|
pdfDocGenerationRef.current += 1;
|
||||||
loadSeqRef.current += 1;
|
loadSeqRef.current += 1;
|
||||||
parsePollAbortRef.current?.abort();
|
parseStreamAbortRef.current?.abort();
|
||||||
parsePollAbortRef.current = null;
|
parseStreamAbortRef.current = null;
|
||||||
parseSseCloseRef.current?.();
|
parseSseCloseRef.current?.();
|
||||||
parseSseCloseRef.current = null;
|
parseSseCloseRef.current = null;
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
setPdfDocument(undefined);
|
setPdfDocument(undefined);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setCurrDocId(id);
|
setCurrDocId(id);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
|
|
@ -489,7 +491,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setParseStatus(initialParseStatus);
|
setParseStatus(initialParseStatus);
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
setActiveParseOpId(null);
|
setActiveParseOpId(null);
|
||||||
startParsedPolling(id, null);
|
startParsedEventStream(id, null);
|
||||||
void fetchDocumentSettings(id, controller.signal);
|
void fetchDocumentSettings(id, controller.signal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -524,7 +526,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setCurrDocText,
|
setCurrDocText,
|
||||||
setPdfDocument,
|
setPdfDocument,
|
||||||
fetchDocumentSettings,
|
fetchDocumentSettings,
|
||||||
startParsedPolling,
|
startParsedEventStream,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise<void> => {
|
const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise<void> => {
|
||||||
|
|
@ -546,14 +548,16 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setParseStatus(forced.status);
|
setParseStatus(forced.status);
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
setActiveParseOpId(forced.opId ?? null);
|
setActiveParseOpId(forced.opId ?? null);
|
||||||
startParsedPolling(currDocId, forced.opId ?? null);
|
startParsedEventStream(currDocId, forced.opId ?? null);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to force PDF reparse:', error);
|
console.error('Failed to force PDF reparse:', error);
|
||||||
}
|
}
|
||||||
}, [currDocId, startParsedPolling]);
|
}, [currDocId, startParsedEventStream]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clears the current document state
|
* Clears the current document state
|
||||||
|
|
@ -567,14 +571,15 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
docLoadSeqRef.current += 1;
|
docLoadSeqRef.current += 1;
|
||||||
docLoadAbortRef.current?.abort();
|
docLoadAbortRef.current?.abort();
|
||||||
docLoadAbortRef.current = null;
|
docLoadAbortRef.current = null;
|
||||||
parsePollAbortRef.current?.abort();
|
parseStreamAbortRef.current?.abort();
|
||||||
parsePollAbortRef.current = null;
|
parseStreamAbortRef.current = null;
|
||||||
parseSseCloseRef.current?.();
|
parseSseCloseRef.current?.();
|
||||||
parseSseCloseRef.current = null;
|
parseSseCloseRef.current = null;
|
||||||
setCurrDocId(undefined);
|
setCurrDocId(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
setPdfDocument(undefined);
|
setPdfDocument(undefined);
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
|
|
@ -582,6 +587,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
setActiveParseOpId(null);
|
setActiveParseOpId(null);
|
||||||
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
stop();
|
stop();
|
||||||
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
|
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
|
||||||
|
|
@ -681,6 +687,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
parsedDocument,
|
parsedDocument,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
parseProgress,
|
parseProgress,
|
||||||
|
|
@ -708,6 +715,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
parsedDocument,
|
parsedDocument,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
parseProgress,
|
parseProgress,
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,15 @@ import { documents } from '@/db/schema';
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker';
|
import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker';
|
||||||
import { isAbortLikeError } from '@/lib/server/compute/abort-like-error';
|
import { isAbortLikeError } from '@/lib/server/compute/abort-like-error';
|
||||||
import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/server/compute/worker-parse-state';
|
import {
|
||||||
|
isWorkerOperationStateStale,
|
||||||
|
mergeNonReadyParseSnapshot,
|
||||||
|
snapshotFromWorkerState,
|
||||||
|
} from '@/lib/server/compute/worker-parse-state';
|
||||||
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
||||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
|
|
@ -67,7 +72,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
|
||||||
const state = await healStaleDocumentParseState({
|
const state = await healStaleDocumentParseState({
|
||||||
documentId: row.id,
|
documentId: row.id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
state: parseDocumentParseState(row.parseState),
|
state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)),
|
||||||
});
|
});
|
||||||
const parseStatus = normalizeParseStatus(state.status);
|
const parseStatus = normalizeParseStatus(state.status);
|
||||||
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
|
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
|
||||||
|
|
@ -82,13 +87,20 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
|
||||||
&& workerState.opId === opId
|
&& workerState.opId === opId
|
||||||
&& !isWorkerOperationStateStale(workerState, opStaleMs)
|
&& !isWorkerOperationStateStale(workerState, opStaleMs)
|
||||||
) {
|
) {
|
||||||
|
const merged = mergeNonReadyParseSnapshot({
|
||||||
|
parseStatus,
|
||||||
|
parseProgress: state.progress ?? null,
|
||||||
|
workerState,
|
||||||
|
});
|
||||||
|
const workerSnapshot = snapshotFromWorkerState(workerState);
|
||||||
|
const fromWorker = workerSnapshot.parseStatus === 'pending' || workerSnapshot.parseStatus === 'running';
|
||||||
return {
|
return {
|
||||||
snapshot: {
|
snapshot: {
|
||||||
...snapshotFromWorkerState(workerState),
|
...merged,
|
||||||
opId: workerState.opId,
|
opId: workerState.opId,
|
||||||
},
|
},
|
||||||
opId: workerState.opId,
|
opId: workerState.opId,
|
||||||
fromWorker: true,
|
fromWorker,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -199,7 +211,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
|
|
||||||
let initialState = await toSnapshotState(row, requestedOpId);
|
let initialState = await toSnapshotState(row, requestedOpId);
|
||||||
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
|
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
|
||||||
const state = parseDocumentParseState(row.parseState);
|
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
const created = await backfillPendingPdfParseOperation({
|
const created = await backfillPendingPdfParseOperation({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
|
|
@ -261,22 +273,19 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
let current = initialState.snapshot;
|
let current = initialState.snapshot;
|
||||||
let signature = JSON.stringify(current);
|
let signature = JSON.stringify(current);
|
||||||
let currentOpId = requestedOpId ?? initialState.opId;
|
let currentOpId = requestedOpId ?? initialState.opId;
|
||||||
let currentFromWorker = initialState.fromWorker;
|
|
||||||
let lastEventId: number | null = null;
|
let lastEventId: number | null = null;
|
||||||
let loggedMissingOpId = false;
|
let loggedMissingOpId = false;
|
||||||
const pinnedRequestedOp = requestedOpId ?? null;
|
const pinnedRequestedOp = requestedOpId ?? null;
|
||||||
const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot, fromWorker: boolean): boolean => {
|
const shouldCloseForTerminalSnapshot = (snapshot: ParsedSnapshot): boolean => {
|
||||||
const isTerminal = snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed';
|
const isTerminal = snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed';
|
||||||
if (!isTerminal) return false;
|
if (!isTerminal) return false;
|
||||||
// If caller pinned an opId, keep streaming until worker confirms that
|
if (pinnedRequestedOp && snapshot.opId !== pinnedRequestedOp) return false;
|
||||||
// op state. DB fallback can report stale terminal status for other rows.
|
|
||||||
if (pinnedRequestedOp && snapshot.opId === pinnedRequestedOp && !fromWorker) return false;
|
|
||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
|
|
||||||
writeSnapshot(current);
|
writeSnapshot(current);
|
||||||
|
|
||||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
if (shouldCloseForTerminalSnapshot(current)) {
|
||||||
closeStream();
|
closeStream();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -303,7 +312,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
}
|
}
|
||||||
current = next.snapshot;
|
current = next.snapshot;
|
||||||
signature = next.signature;
|
signature = next.signature;
|
||||||
currentFromWorker = next.fromWorker;
|
|
||||||
if (!requestedOpId && next.opId !== currentOpId) {
|
if (!requestedOpId && next.opId !== currentOpId) {
|
||||||
currentOpId = next.opId;
|
currentOpId = next.opId;
|
||||||
lastEventId = null;
|
lastEventId = null;
|
||||||
|
|
@ -312,7 +320,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
workerAbort = null;
|
workerAbort = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
if (shouldCloseForTerminalSnapshot(current)) {
|
||||||
closeStream();
|
closeStream();
|
||||||
}
|
}
|
||||||
}).catch((error) => {
|
}).catch((error) => {
|
||||||
|
|
@ -349,7 +357,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
}
|
}
|
||||||
current = next.snapshot;
|
current = next.snapshot;
|
||||||
signature = next.signature;
|
signature = next.signature;
|
||||||
currentFromWorker = next.fromWorker;
|
|
||||||
currentOpId = requestedOpId ?? next.opId;
|
currentOpId = requestedOpId ?? next.opId;
|
||||||
if (!currentOpId) {
|
if (!currentOpId) {
|
||||||
if (!loggedMissingOpId) {
|
if (!loggedMissingOpId) {
|
||||||
|
|
@ -357,7 +364,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
logger.warn({
|
logger.warn({
|
||||||
event: 'documents.parsed.events.missing_opid_non_terminal',
|
event: 'documents.parsed.events.missing_opid_non_terminal',
|
||||||
degraded: true,
|
degraded: true,
|
||||||
step: 'poll_without_worker_op',
|
step: 'missing_opid_fallback',
|
||||||
documentId: id,
|
documentId: id,
|
||||||
storageUserIdHash,
|
storageUserIdHash,
|
||||||
parseStatus: current.parseStatus,
|
parseStatus: current.parseStatus,
|
||||||
|
|
@ -367,7 +374,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
} else if (loggedMissingOpId) {
|
} else if (loggedMissingOpId) {
|
||||||
loggedMissingOpId = false;
|
loggedMissingOpId = false;
|
||||||
}
|
}
|
||||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
if (shouldCloseForTerminalSnapshot(current)) {
|
||||||
closeStream();
|
closeStream();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -471,19 +478,23 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
);
|
);
|
||||||
if (!workerSnapshot || workerSnapshot.opId !== currentOpId) continue;
|
if (!workerSnapshot || workerSnapshot.opId !== currentOpId) continue;
|
||||||
|
|
||||||
|
const mergedSnapshot = mergeNonReadyParseSnapshot({
|
||||||
|
parseStatus: current.parseStatus,
|
||||||
|
parseProgress: current.parseProgress,
|
||||||
|
workerState: workerSnapshot,
|
||||||
|
});
|
||||||
const nextSnapshot: ParsedSnapshot = {
|
const nextSnapshot: ParsedSnapshot = {
|
||||||
...snapshotFromWorkerState(workerSnapshot),
|
...mergedSnapshot,
|
||||||
opId: workerSnapshot.opId,
|
opId: workerSnapshot.opId,
|
||||||
};
|
};
|
||||||
const nextSignature = JSON.stringify(nextSnapshot);
|
const nextSignature = JSON.stringify(nextSnapshot);
|
||||||
if (nextSignature !== signature) {
|
if (nextSignature !== signature) {
|
||||||
current = nextSnapshot;
|
current = nextSnapshot;
|
||||||
signature = nextSignature;
|
signature = nextSignature;
|
||||||
currentFromWorker = true;
|
|
||||||
writeSnapshot(current);
|
writeSnapshot(current);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
if (shouldCloseForTerminalSnapshot(current)) {
|
||||||
closeStream();
|
closeStream();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -522,12 +533,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
}
|
}
|
||||||
current = next.snapshot;
|
current = next.snapshot;
|
||||||
signature = next.signature;
|
signature = next.signature;
|
||||||
currentFromWorker = next.fromWorker;
|
|
||||||
if (!requestedOpId && next.opId !== currentOpId) {
|
if (!requestedOpId && next.opId !== currentOpId) {
|
||||||
currentOpId = next.opId;
|
currentOpId = next.opId;
|
||||||
lastEventId = null;
|
lastEventId = null;
|
||||||
}
|
}
|
||||||
if (shouldCloseForTerminalSnapshot(current, currentFromWorker)) {
|
if (shouldCloseForTerminalSnapshot(current)) {
|
||||||
closeStream();
|
closeStream();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,39 +4,36 @@ import { and, eq, inArray } from 'drizzle-orm';
|
||||||
import { db } from '@/db';
|
import { db } from '@/db';
|
||||||
import { documents } from '@/db/schema';
|
import { documents } from '@/db/schema';
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
|
|
||||||
import {
|
import {
|
||||||
documentParseStateFromWorkerState,
|
|
||||||
isWorkerOperationStateStale,
|
isWorkerOperationStateStale,
|
||||||
snapshotFromWorkerState,
|
snapshotFromWorkerState,
|
||||||
} from '@/lib/server/compute/worker-parse-state';
|
} from '@/lib/server/compute/worker-parse-state';
|
||||||
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
||||||
import {
|
import {
|
||||||
documentKey,
|
|
||||||
getParsedDocumentBlob,
|
getParsedDocumentBlob,
|
||||||
getParsedDocumentBlobByKey,
|
getParsedDocumentBlobByKey,
|
||||||
isMissingBlobError,
|
isMissingBlobError,
|
||||||
isValidDocumentId,
|
isValidDocumentId,
|
||||||
putParsedDocumentBlob,
|
|
||||||
} from '@/lib/server/documents/blobstore';
|
} from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill';
|
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||||
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
||||||
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
import { checkJobRate, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||||
import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response';
|
import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response';
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger';
|
import { createRequestLogger, hashForLog } from '@/lib/server/logger';
|
||||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||||
import { logDegraded } from '@/lib/server/errors/logging';
|
|
||||||
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
||||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
||||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutJobResult } from '@openreader/compute-core/api-contracts';
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
|
@ -100,102 +97,12 @@ async function writeParseRowState(input: {
|
||||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function finalizeFromWorkerState(input: {
|
|
||||||
workerState: WorkerOperationState<PdfLayoutJobResult>;
|
|
||||||
row: ParseRow;
|
|
||||||
namespace: string | null;
|
|
||||||
logger: ServerLogger;
|
|
||||||
}): Promise<NextResponse> {
|
|
||||||
const snapshot = snapshotFromWorkerState(input.workerState);
|
|
||||||
const workerParseState = documentParseStateFromWorkerState(input.workerState);
|
|
||||||
|
|
||||||
if (snapshot.parseStatus === 'pending' || snapshot.parseStatus === 'running') {
|
|
||||||
await writeParseRowState({
|
|
||||||
documentId: input.row.id,
|
|
||||||
userId: input.row.userId,
|
|
||||||
parseState: stringifyDocumentParseState(workerParseState),
|
|
||||||
});
|
|
||||||
return NextResponse.json({
|
|
||||||
parseStatus: snapshot.parseStatus,
|
|
||||||
parseProgress: snapshot.parseProgress,
|
|
||||||
opId: input.workerState.opId,
|
|
||||||
}, { status: 202 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (snapshot.parseStatus === 'failed') {
|
|
||||||
await writeParseRowState({
|
|
||||||
documentId: input.row.id,
|
|
||||||
userId: input.row.userId,
|
|
||||||
parseState: stringifyDocumentParseState(workerParseState),
|
|
||||||
});
|
|
||||||
|
|
||||||
return NextResponse.json({
|
|
||||||
parseStatus: 'failed',
|
|
||||||
parseProgress: null,
|
|
||||||
opId: input.workerState.opId,
|
|
||||||
error: input.workerState.error?.message ?? 'Worker parse failed',
|
|
||||||
}, { status: 202 });
|
|
||||||
}
|
|
||||||
|
|
||||||
let parsedJsonKey: string | null = null;
|
|
||||||
if (input.workerState.result && 'parsedObjectKey' in input.workerState.result) {
|
|
||||||
parsedJsonKey = typeof input.workerState.result.parsedObjectKey === 'string'
|
|
||||||
? input.workerState.result.parsedObjectKey
|
|
||||||
: null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!parsedJsonKey && input.workerState.result && 'parsed' in input.workerState.result && input.workerState.result.parsed) {
|
|
||||||
const parsedJson = Buffer.from(JSON.stringify(input.workerState.result.parsed));
|
|
||||||
parsedJsonKey = await putParsedDocumentBlob(input.row.id, parsedJson, input.namespace);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!parsedJsonKey) {
|
|
||||||
return errorResponse(new Error('Worker completed without parsed output'), {
|
|
||||||
apiErrorMessage: 'Worker completed without parsed output',
|
|
||||||
normalize: { code: 'DOCUMENTS_PARSED_WORKER_OUTPUT_MISSING', errorClass: 'upstream' },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
await writeParseRowState({
|
|
||||||
documentId: input.row.id,
|
|
||||||
userId: input.row.userId,
|
|
||||||
parseState: stringifyDocumentParseState(workerParseState),
|
|
||||||
parsedJsonKey,
|
|
||||||
});
|
|
||||||
|
|
||||||
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
|
||||||
let parsedDoc: ParsedPdfDocument | null = null;
|
|
||||||
try {
|
|
||||||
parsedDoc = JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
|
|
||||||
} catch {
|
|
||||||
parsedDoc = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!hasAnyParsedBlocks(parsedDoc)) {
|
|
||||||
input.logger.warn({
|
|
||||||
event: 'documents.parsed.no_blocks_in_output',
|
|
||||||
documentId: input.row.id,
|
|
||||||
userIdHash: hashForLog(input.row.userId),
|
|
||||||
parsedJsonKey,
|
|
||||||
}, 'Worker output parsed successfully but contained no blocks');
|
|
||||||
}
|
|
||||||
|
|
||||||
return new NextResponse(new Uint8Array(json), {
|
|
||||||
status: 200,
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Cache-Control': 'no-store',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||||
const { logger } = createRequestLogger({
|
const { logger } = createRequestLogger({
|
||||||
route: '/api/documents/[id]/parsed',
|
route: '/api/documents/[id]/parsed',
|
||||||
request: req,
|
request: req,
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const opStaleMs = getComputeOpStaleMs();
|
|
||||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||||
|
|
||||||
const authCtxOrRes = await requireAuthContext(req);
|
const authCtxOrRes = await requireAuthContext(req);
|
||||||
|
|
@ -204,8 +111,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
|
|
||||||
const params = await ctx.params;
|
const params = await ctx.params;
|
||||||
const id = (params.id || '').trim().toLowerCase();
|
const id = (params.id || '').trim().toLowerCase();
|
||||||
const retryFailed = req.nextUrl.searchParams.get('retry') === '1';
|
|
||||||
const requestedOpId = normalizeOpId(req.nextUrl.searchParams.get('opId'));
|
|
||||||
if (!isValidDocumentId(id)) {
|
if (!isValidDocumentId(id)) {
|
||||||
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
@ -220,112 +125,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestedOpId) {
|
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
const workerState = await fetchWorkerOperationState<PdfLayoutJobResult>(requestedOpId);
|
|
||||||
if (workerState && workerState.opId === requestedOpId) {
|
|
||||||
return finalizeFromWorkerState({
|
|
||||||
workerState,
|
|
||||||
row,
|
|
||||||
namespace: testNamespace,
|
|
||||||
logger,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
logDegraded(logger, {
|
|
||||||
event: 'documents.parsed.requested_op_unavailable',
|
|
||||||
msg: 'Requested worker operation id was unavailable',
|
|
||||||
step: 'requested_op_lookup',
|
|
||||||
context: {
|
|
||||||
documentId: id,
|
|
||||||
userIdHash: hashForLog(row.userId),
|
|
||||||
opId: requestedOpId,
|
|
||||||
error: {
|
|
||||||
name: 'RequestedOperationUnavailable',
|
|
||||||
message: `Requested worker operation ${requestedOpId} was unavailable`,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let state = parseDocumentParseState(row.parseState);
|
|
||||||
state = await healStaleDocumentParseState({
|
|
||||||
documentId: id,
|
|
||||||
userId: row.userId,
|
|
||||||
state,
|
|
||||||
});
|
|
||||||
|
|
||||||
const effectiveStatus = normalizeParseStatus(state.status);
|
const effectiveStatus = normalizeParseStatus(state.status);
|
||||||
const effectiveProgress = state.progress ?? null;
|
const effectiveProgress = state.progress ?? null;
|
||||||
const effectiveOpId = normalizeOpId(state.opId);
|
const effectiveOpId = normalizeOpId(state.opId);
|
||||||
|
|
||||||
if (effectiveOpId && effectiveStatus !== 'ready') {
|
|
||||||
const workerState = await fetchWorkerOperationState<PdfLayoutJobResult>(effectiveOpId);
|
|
||||||
if (
|
|
||||||
workerState
|
|
||||||
&& workerState.opId === effectiveOpId
|
|
||||||
&& !isWorkerOperationStateStale(workerState, opStaleMs)
|
|
||||||
) {
|
|
||||||
return finalizeFromWorkerState({
|
|
||||||
workerState,
|
|
||||||
row,
|
|
||||||
namespace: testNamespace,
|
|
||||||
logger,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!effectiveOpId && (effectiveStatus === 'pending' || effectiveStatus === 'running')) {
|
|
||||||
const created = await backfillPendingPdfParseOperation({
|
|
||||||
documentId: id,
|
|
||||||
userId: row.userId,
|
|
||||||
namespace: testNamespace,
|
|
||||||
state,
|
|
||||||
});
|
|
||||||
if (created) {
|
|
||||||
const snapshot = snapshotFromWorkerState(created);
|
|
||||||
await writeParseRowState({
|
|
||||||
documentId: row.id,
|
|
||||||
userId: row.userId,
|
|
||||||
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
|
|
||||||
});
|
|
||||||
return NextResponse.json({
|
|
||||||
parseStatus: snapshot.parseStatus,
|
|
||||||
parseProgress: snapshot.parseProgress,
|
|
||||||
opId: created.opId,
|
|
||||||
}, { status: 202 });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (effectiveStatus === 'failed' && retryFailed) {
|
|
||||||
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
|
||||||
const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig);
|
|
||||||
if (!rateDecision.allowed) {
|
|
||||||
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
|
|
||||||
}
|
|
||||||
const created = await createOrReusePdfWorkerOperation({
|
|
||||||
documentId: id,
|
|
||||||
namespace: testNamespace,
|
|
||||||
documentObjectKey: documentKey(id, testNamespace),
|
|
||||||
});
|
|
||||||
await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig);
|
|
||||||
const snapshot = snapshotFromWorkerState(created);
|
|
||||||
await writeParseRowState({
|
|
||||||
documentId: row.id,
|
|
||||||
userId: row.userId,
|
|
||||||
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
|
|
||||||
});
|
|
||||||
return NextResponse.json({
|
|
||||||
parseStatus: snapshot.parseStatus,
|
|
||||||
parseProgress: snapshot.parseProgress,
|
|
||||||
opId: created.opId,
|
|
||||||
}, { status: 202 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (effectiveStatus !== 'ready') {
|
if (effectiveStatus !== 'ready') {
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
parseStatus: effectiveStatus,
|
parseStatus: effectiveStatus,
|
||||||
parseProgress: effectiveProgress,
|
parseProgress: effectiveProgress,
|
||||||
opId: effectiveOpId,
|
opId: effectiveOpId,
|
||||||
}, { status: 202 });
|
}, { status: 409 });
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -409,7 +219,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
let state = parseDocumentParseState(row.parseState);
|
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
state = await healStaleDocumentParseState({
|
state = await healStaleDocumentParseState({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
|
|
@ -441,25 +251,33 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
|
return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname });
|
||||||
}
|
}
|
||||||
|
|
||||||
const created = await createOrReusePdfWorkerOperation({
|
const forceToken = randomUUID();
|
||||||
|
const startedParse = await startPdfParseOperation({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
|
userId: authCtxOrRes.userId,
|
||||||
namespace: testNamespace,
|
namespace: testNamespace,
|
||||||
documentObjectKey: documentKey(id, testNamespace),
|
forceToken,
|
||||||
forceToken: randomUUID(),
|
|
||||||
});
|
});
|
||||||
await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig);
|
const snapshot = snapshotFromWorkerState(startedParse.workerState);
|
||||||
|
|
||||||
const snapshot = snapshotFromWorkerState(created);
|
|
||||||
await writeParseRowState({
|
await writeParseRowState({
|
||||||
documentId: row.id,
|
documentId: row.id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
parseState: stringifyDocumentParseState(documentParseStateFromWorkerState(created)),
|
parseState: stringifyDocumentParseState(startedParse.parseState),
|
||||||
|
});
|
||||||
|
enqueueParsePdfJob({
|
||||||
|
documentId: id,
|
||||||
|
userId: row.userId,
|
||||||
|
namespace: testNamespace,
|
||||||
|
forceToken,
|
||||||
|
initialOpId: startedParse.workerState.opId,
|
||||||
|
initialJobId: startedParse.workerState.jobId,
|
||||||
|
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
|
||||||
});
|
});
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
parseStatus: snapshot.parseStatus,
|
parseStatus: snapshot.parseStatus,
|
||||||
parseProgress: snapshot.parseProgress,
|
parseProgress: snapshot.parseProgress,
|
||||||
opId: created.opId,
|
opId: startedParse.workerState.opId,
|
||||||
}, { status: 202 });
|
}, { status: 202 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return errorResponse(error, {
|
return errorResponse(error, {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore';
|
import { isValidTempUploadToken, putTempDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
|
|
@ -26,11 +26,12 @@ export async function PUT(req: NextRequest) {
|
||||||
|
|
||||||
const ctxOrRes = await requireAuthContext(req);
|
const ctxOrRes = await requireAuthContext(req);
|
||||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||||
|
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||||
|
|
||||||
const url = new URL(req.url);
|
const url = new URL(req.url);
|
||||||
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
|
const token = (url.searchParams.get('token') || '').trim().toLowerCase();
|
||||||
if (!isValidDocumentId(id)) {
|
if (!isValidTempUploadToken(token)) {
|
||||||
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
return NextResponse.json({ error: 'Invalid upload token' }, { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream';
|
const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream';
|
||||||
|
|
@ -86,7 +87,7 @@ export async function PUT(req: NextRequest) {
|
||||||
const namespace = getOpenReaderTestNamespace(req.headers);
|
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await putDocumentBlob(id, body, contentType, namespace);
|
await putTempDocumentBlob(token, ctxOrRes.userId, body, contentType, namespace);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (!isPreconditionFailed(error)) {
|
if (!isPreconditionFailed(error)) {
|
||||||
throw error;
|
throw error;
|
||||||
|
|
@ -97,12 +98,12 @@ export async function PUT(req: NextRequest) {
|
||||||
event: 'documents.blob.upload.fallback.proxy_used',
|
event: 'documents.blob.upload.fallback.proxy_used',
|
||||||
degraded: true,
|
degraded: true,
|
||||||
fallbackPath: 'upload_proxy',
|
fallbackPath: 'upload_proxy',
|
||||||
documentId: id,
|
uploadToken: token,
|
||||||
contentType,
|
contentType,
|
||||||
bytes: body.byteLength,
|
bytes: body.byteLength,
|
||||||
}, 'Document upload fallback proxy used');
|
}, 'Document upload fallback proxy used');
|
||||||
|
|
||||||
return NextResponse.json({ success: true, id });
|
return NextResponse.json({ success: true, token });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
serverLogger.error({
|
serverLogger.error({
|
||||||
event: 'documents.blob.upload.fallback.failed',
|
event: 'documents.blob.upload.fallback.failed',
|
||||||
|
|
|
||||||
257
src/app/api/documents/blob/upload/finalize/route.ts
Normal file
257
src/app/api/documents/blob/upload/finalize/route.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
||||||
|
import path from 'path';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
|
import {
|
||||||
|
TEMP_DOCUMENT_UPLOAD_TTL_MS,
|
||||||
|
copyTempDocumentBlobToDocument,
|
||||||
|
deleteTempDocumentUpload,
|
||||||
|
getTempDocumentBlob,
|
||||||
|
getTempDocumentFinalizeReceipt,
|
||||||
|
headDocumentBlob,
|
||||||
|
headTempDocumentBlob,
|
||||||
|
isMissingBlobError,
|
||||||
|
isPreconditionFailed,
|
||||||
|
isValidTempUploadToken,
|
||||||
|
putDocumentBlob,
|
||||||
|
putTempDocumentFinalizeReceipt,
|
||||||
|
} from '@/lib/server/documents/blobstore';
|
||||||
|
import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert';
|
||||||
|
import { registerUploadedDocument } from '@/lib/server/documents/register-upload';
|
||||||
|
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
||||||
|
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||||
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
|
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
|
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
type FinalizeUpload = {
|
||||||
|
token: string;
|
||||||
|
name: string;
|
||||||
|
type: DocumentType;
|
||||||
|
lastModified: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type FinalizeReceipt = {
|
||||||
|
stored: BaseDocument;
|
||||||
|
};
|
||||||
|
|
||||||
|
function s3NotConfiguredResponse(): NextResponse {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType {
|
||||||
|
if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') {
|
||||||
|
return rawType;
|
||||||
|
}
|
||||||
|
return toDocumentTypeFromName(safeName);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeLastModified(value: unknown): number {
|
||||||
|
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFinalizePayload(body: unknown): FinalizeUpload[] {
|
||||||
|
if (!body || typeof body !== 'object') return [];
|
||||||
|
const rawUploads = (body as { uploads?: unknown }).uploads;
|
||||||
|
if (!Array.isArray(rawUploads)) return [];
|
||||||
|
|
||||||
|
const uploads: FinalizeUpload[] = [];
|
||||||
|
for (const rawUpload of rawUploads) {
|
||||||
|
if (!rawUpload || typeof rawUpload !== 'object') continue;
|
||||||
|
const rec = rawUpload as Record<string, unknown>;
|
||||||
|
const token = typeof rec.token === 'string' ? rec.token.trim().toLowerCase() : '';
|
||||||
|
if (!isValidTempUploadToken(token)) continue;
|
||||||
|
const fallbackName = `upload-${token}.txt`;
|
||||||
|
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
|
||||||
|
uploads.push({
|
||||||
|
token,
|
||||||
|
name,
|
||||||
|
type: normalizeDocumentType(rec.type, name),
|
||||||
|
lastModified: normalizeLastModified(rec.lastModified),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return uploads;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadTempUpload(input: {
|
||||||
|
token: string;
|
||||||
|
userId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
}): Promise<{ contentType: string; size: number; lastModified: number; body: Buffer }> {
|
||||||
|
const RETRIES = 3;
|
||||||
|
const RETRY_DELAY_MS = 500;
|
||||||
|
|
||||||
|
let lastError: unknown = null;
|
||||||
|
for (let attempt = 0; attempt < RETRIES; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const head = await headTempDocumentBlob(input.token, input.userId, input.namespace);
|
||||||
|
const body = await getTempDocumentBlob(input.token, input.userId, input.namespace);
|
||||||
|
return {
|
||||||
|
contentType: head.contentType ?? 'application/octet-stream',
|
||||||
|
size: head.contentLength > 0 ? head.contentLength : body.byteLength,
|
||||||
|
lastModified: head.lastModified ?? Date.now(),
|
||||||
|
body,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error;
|
||||||
|
if (isMissingBlobError(error) && attempt < RETRIES - 1) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError instanceof Error ? lastError : new Error('Temporary upload is unavailable');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finalizeOne(input: {
|
||||||
|
upload: FinalizeUpload;
|
||||||
|
userId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
}): Promise<BaseDocument> {
|
||||||
|
const existingReceipt = await getTempDocumentFinalizeReceipt<FinalizeReceipt>(
|
||||||
|
input.upload.token,
|
||||||
|
input.userId,
|
||||||
|
input.namespace,
|
||||||
|
);
|
||||||
|
if (existingReceipt?.stored) {
|
||||||
|
return existingReceipt.stored;
|
||||||
|
}
|
||||||
|
|
||||||
|
const temp = await loadTempUpload({
|
||||||
|
token: input.upload.token,
|
||||||
|
userId: input.userId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (Date.now() - temp.lastModified > TEMP_DOCUMENT_UPLOAD_TTL_MS) {
|
||||||
|
await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch(() => undefined);
|
||||||
|
throw new Error('Temporary upload expired before finalize');
|
||||||
|
}
|
||||||
|
|
||||||
|
const isDocxUpload = input.upload.type === 'docx';
|
||||||
|
const finalizedType: DocumentType = isDocxUpload ? 'pdf' : input.upload.type;
|
||||||
|
const finalizedBody = isDocxUpload
|
||||||
|
? await convertDocxBufferToPdfBuffer(temp.body)
|
||||||
|
: temp.body;
|
||||||
|
const finalizedContentType = finalizedType === 'pdf'
|
||||||
|
? 'application/pdf'
|
||||||
|
: temp.contentType;
|
||||||
|
const finalizedName = finalizedType === 'pdf' && input.upload.type === 'docx'
|
||||||
|
? safeDocumentName(`${path.parse(input.upload.name).name}.pdf`, 'upload.pdf')
|
||||||
|
: input.upload.name;
|
||||||
|
const documentId = createHash('sha256').update(finalizedBody).digest('hex');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await headDocumentBlob(documentId, input.namespace);
|
||||||
|
} catch (error) {
|
||||||
|
if (!isMissingBlobError(error)) throw error;
|
||||||
|
if (!isDocxUpload) {
|
||||||
|
try {
|
||||||
|
await copyTempDocumentBlobToDocument(
|
||||||
|
input.upload.token,
|
||||||
|
input.userId,
|
||||||
|
documentId,
|
||||||
|
input.namespace,
|
||||||
|
finalizedContentType,
|
||||||
|
{ ifNoneMatch: true },
|
||||||
|
);
|
||||||
|
} catch (copyError) {
|
||||||
|
if (!isPreconditionFailed(copyError)) throw copyError;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
await putDocumentBlob(
|
||||||
|
documentId,
|
||||||
|
finalizedBody,
|
||||||
|
finalizedContentType,
|
||||||
|
input.namespace,
|
||||||
|
{ ifNoneMatch: true },
|
||||||
|
);
|
||||||
|
} catch (putError) {
|
||||||
|
if (!isPreconditionFailed(putError)) throw putError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const canonicalHead = await headDocumentBlob(documentId, input.namespace);
|
||||||
|
const stored = await registerUploadedDocument({
|
||||||
|
documentId,
|
||||||
|
userId: input.userId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
name: finalizedName,
|
||||||
|
type: finalizedType,
|
||||||
|
size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength,
|
||||||
|
lastModified: input.upload.lastModified,
|
||||||
|
});
|
||||||
|
|
||||||
|
await putTempDocumentFinalizeReceipt(
|
||||||
|
input.upload.token,
|
||||||
|
input.userId,
|
||||||
|
input.namespace,
|
||||||
|
Buffer.from(JSON.stringify({ stored }), 'utf8'),
|
||||||
|
);
|
||||||
|
|
||||||
|
await deleteTempDocumentUpload(input.upload.token, input.userId, input.namespace).catch((error) => {
|
||||||
|
serverLogger.warn({
|
||||||
|
event: 'documents.blob.upload.finalize.temp_delete_failed',
|
||||||
|
degraded: true,
|
||||||
|
fallbackPath: 'leave_temp_upload',
|
||||||
|
documentId,
|
||||||
|
token: input.upload.token,
|
||||||
|
error: errorToLog(error),
|
||||||
|
}, 'Failed to delete temp upload after finalize');
|
||||||
|
});
|
||||||
|
|
||||||
|
return stored;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function POST(req: NextRequest) {
|
||||||
|
try {
|
||||||
|
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||||
|
|
||||||
|
const ctxOrRes = await requireAuthContext(req);
|
||||||
|
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||||
|
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||||
|
const userId = ctxOrRes.userId;
|
||||||
|
|
||||||
|
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||||
|
const uploads = parseFinalizePayload(await req.json().catch(() => null));
|
||||||
|
if (uploads.length === 0) {
|
||||||
|
return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const stored = await Promise.all(
|
||||||
|
uploads.map((upload) => finalizeOne({
|
||||||
|
upload,
|
||||||
|
userId,
|
||||||
|
namespace,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
|
||||||
|
return NextResponse.json({ stored });
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Error && error.message === 'Temporary upload expired before finalize') {
|
||||||
|
return NextResponse.json({ error: error.message }, { status: 410 });
|
||||||
|
}
|
||||||
|
if (isMissingBlobError(error)) {
|
||||||
|
return NextResponse.json({ error: 'Temporary upload missing. Upload bytes again and retry finalize.' }, { status: 409 });
|
||||||
|
}
|
||||||
|
|
||||||
|
serverLogger.error({
|
||||||
|
event: 'documents.blob.upload.finalize.failed',
|
||||||
|
error: errorToLog(error),
|
||||||
|
}, 'Failed to finalize uploaded documents');
|
||||||
|
return errorResponse(error, {
|
||||||
|
apiErrorMessage: 'Failed to finalize uploaded documents',
|
||||||
|
normalize: { code: 'DOCUMENTS_BLOB_UPLOAD_FINALIZE_FAILED', errorClass: 'storage' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,11 @@
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore';
|
import {
|
||||||
|
TEMP_DOCUMENT_UPLOAD_TTL_MS,
|
||||||
|
deleteExpiredTempDocumentUploads,
|
||||||
|
presignTempPut,
|
||||||
|
} from '@/lib/server/documents/blobstore';
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
|
|
@ -10,7 +15,6 @@ import { errorResponse } from '@/lib/server/errors/next-response';
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
type PresignUpload = {
|
type PresignUpload = {
|
||||||
id: string;
|
|
||||||
contentType: string;
|
contentType: string;
|
||||||
size: number;
|
size: number;
|
||||||
};
|
};
|
||||||
|
|
@ -24,14 +28,12 @@ function parseUploads(body: unknown): PresignUpload[] {
|
||||||
for (const raw of rawUploads) {
|
for (const raw of rawUploads) {
|
||||||
if (!raw || typeof raw !== 'object') continue;
|
if (!raw || typeof raw !== 'object') continue;
|
||||||
const rec = raw as Record<string, unknown>;
|
const rec = raw as Record<string, unknown>;
|
||||||
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
|
|
||||||
if (!isValidDocumentId(id)) continue;
|
|
||||||
const contentType =
|
const contentType =
|
||||||
typeof rec.contentType === 'string' && rec.contentType.trim()
|
typeof rec.contentType === 'string' && rec.contentType.trim()
|
||||||
? rec.contentType.trim()
|
? rec.contentType.trim()
|
||||||
: 'application/octet-stream';
|
: 'application/octet-stream';
|
||||||
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
||||||
uploads.push({ id, contentType, size });
|
uploads.push({ contentType, size });
|
||||||
}
|
}
|
||||||
return uploads;
|
return uploads;
|
||||||
}
|
}
|
||||||
|
|
@ -47,6 +49,8 @@ export async function POST(req: NextRequest) {
|
||||||
|
|
||||||
const ctxOrRes = await requireAuthContext(req);
|
const ctxOrRes = await requireAuthContext(req);
|
||||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||||
|
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||||
|
const userId = ctxOrRes.userId;
|
||||||
|
|
||||||
const body = await req.json().catch(() => null);
|
const body = await req.json().catch(() => null);
|
||||||
const uploads = parseUploads(body);
|
const uploads = parseUploads(body);
|
||||||
|
|
@ -68,13 +72,16 @@ export async function POST(req: NextRequest) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const namespace = getOpenReaderTestNamespace(req.headers);
|
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||||
|
await deleteExpiredTempDocumentUploads(userId, namespace, Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS)
|
||||||
|
.catch(() => undefined);
|
||||||
const signed = await Promise.all(
|
const signed = await Promise.all(
|
||||||
uploads.map(async (upload) => {
|
uploads.map(async (upload) => {
|
||||||
const res = await presignPut(upload.id, upload.contentType, namespace, {
|
const token = randomUUID();
|
||||||
|
const res = await presignTempPut(token, userId, upload.contentType, namespace, {
|
||||||
contentLength: upload.size,
|
contentLength: upload.size,
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
id: upload.id,
|
token,
|
||||||
url: res.url,
|
url: res.url,
|
||||||
headers: res.headers,
|
headers: res.headers,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,209 +0,0 @@
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
|
||||||
import { writeFile, mkdir, readFile, readdir, rm, stat } from 'fs/promises';
|
|
||||||
import { spawn } from 'child_process';
|
|
||||||
import path from 'path';
|
|
||||||
import { existsSync } from 'fs';
|
|
||||||
import { randomUUID, createHash } from 'crypto';
|
|
||||||
import { pathToFileURL } from 'url';
|
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
|
||||||
import { db } from '@/db';
|
|
||||||
import { documents } from '@/db/schema';
|
|
||||||
import { safeDocumentName } from '@/lib/server/documents/utils';
|
|
||||||
import { enqueueDocumentPreview } from '@/lib/server/documents/previews';
|
|
||||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
|
||||||
import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
|
||||||
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
|
||||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
|
||||||
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
|
|
||||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
|
||||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
|
||||||
|
|
||||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
|
||||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
|
||||||
|
|
||||||
async function ensureTempDir() {
|
|
||||||
if (!existsSync(DOCSTORE_DIR)) {
|
|
||||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
|
||||||
}
|
|
||||||
if (!existsSync(TEMP_DIR)) {
|
|
||||||
await mkdir(TEMP_DIR, { recursive: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const args: string[] = [];
|
|
||||||
if (profileDir) {
|
|
||||||
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
|
|
||||||
}
|
|
||||||
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
|
|
||||||
const proc = spawn('soffice', args);
|
|
||||||
|
|
||||||
proc.on('error', (error) => reject(error));
|
|
||||||
proc.on('close', (code) => {
|
|
||||||
if (code === 0) resolve();
|
|
||||||
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100): Promise<string> {
|
|
||||||
const end = Date.now() + timeoutMs;
|
|
||||||
while (Date.now() < end) {
|
|
||||||
const files = await readdir(dir);
|
|
||||||
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
|
|
||||||
if (pdf) {
|
|
||||||
const pdfPath = path.join(dir, pdf);
|
|
||||||
try {
|
|
||||||
const first = await stat(pdfPath);
|
|
||||||
await new Promise((res) => setTimeout(res, intervalMs));
|
|
||||||
const second = await stat(pdfPath);
|
|
||||||
if (second.size > 0 && second.size === first.size) {
|
|
||||||
return pdfPath;
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// ignore transient errors
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((res) => setTimeout(res, intervalMs));
|
|
||||||
}
|
|
||||||
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
|
||||||
try {
|
|
||||||
if (!isS3Configured()) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
|
||||||
{ status: 503 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
|
||||||
|
|
||||||
const ctxOrRes = await requireAuthContext(req);
|
|
||||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
|
||||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
|
||||||
const storageUserId = ctxOrRes.userId;
|
|
||||||
|
|
||||||
await ensureTempDir();
|
|
||||||
|
|
||||||
const formData = await req.formData();
|
|
||||||
const file = formData.get('file');
|
|
||||||
if (!(file instanceof File)) {
|
|
||||||
return NextResponse.json({ error: 'No file provided' }, { status: 400 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!file.name.toLowerCase().endsWith('.docx')) {
|
|
||||||
return NextResponse.json({ error: 'File must be a .docx document' }, { status: 400 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const docxBytes = Buffer.from(await file.arrayBuffer());
|
|
||||||
// Keep stable IDs tied to source bytes.
|
|
||||||
const id = createHash('sha256').update(docxBytes).digest('hex');
|
|
||||||
|
|
||||||
const tempId = randomUUID();
|
|
||||||
const jobDir = path.join(TEMP_DIR, tempId);
|
|
||||||
await mkdir(jobDir, { recursive: true });
|
|
||||||
const profileDir = path.join(jobDir, 'lo-profile');
|
|
||||||
await mkdir(profileDir, { recursive: true });
|
|
||||||
const inputPath = path.join(jobDir, 'input.docx');
|
|
||||||
|
|
||||||
await writeFile(inputPath, docxBytes);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await convertDocxToPdf(inputPath, jobDir, profileDir);
|
|
||||||
const pdfPath = await waitForPdfReady(jobDir);
|
|
||||||
const pdfContent = await readFile(pdfPath);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace);
|
|
||||||
} catch (error) {
|
|
||||||
// Idempotent behavior: if blob already exists for this sha, continue.
|
|
||||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined;
|
|
||||||
const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed';
|
|
||||||
if (!isPreconditionFailed) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
|
|
||||||
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
|
|
||||||
|
|
||||||
await db
|
|
||||||
.insert(documents)
|
|
||||||
.values({
|
|
||||||
id,
|
|
||||||
userId: storageUserId,
|
|
||||||
name: derivedName,
|
|
||||||
type: 'pdf',
|
|
||||||
size: pdfContent.length,
|
|
||||||
lastModified,
|
|
||||||
filePath: id,
|
|
||||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
|
||||||
parsedJsonKey: null,
|
|
||||||
})
|
|
||||||
.onConflictDoUpdate({
|
|
||||||
target: [documents.id, documents.userId],
|
|
||||||
set: {
|
|
||||||
name: derivedName,
|
|
||||||
type: 'pdf',
|
|
||||||
size: pdfContent.length,
|
|
||||||
lastModified,
|
|
||||||
filePath: id,
|
|
||||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
|
||||||
parsedJsonKey: null,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
await enqueueDocumentPreview(
|
|
||||||
{
|
|
||||||
id,
|
|
||||||
type: 'pdf',
|
|
||||||
lastModified,
|
|
||||||
},
|
|
||||||
testNamespace,
|
|
||||||
).catch((error) => {
|
|
||||||
serverLogger.warn({
|
|
||||||
event: 'documents.docx.preview_enqueue.failed',
|
|
||||||
degraded: true,
|
|
||||||
fallbackPath: 'skip_preview_enqueue',
|
|
||||||
documentId: id,
|
|
||||||
error: errorToLog(error),
|
|
||||||
}, 'Failed to enqueue preview for converted DOCX');
|
|
||||||
});
|
|
||||||
|
|
||||||
// Record upload-driven parse load (see register route for rationale).
|
|
||||||
const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
|
||||||
await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig);
|
|
||||||
enqueueParsePdfJob({
|
|
||||||
documentId: id,
|
|
||||||
userId: storageUserId,
|
|
||||||
namespace: testNamespace,
|
|
||||||
});
|
|
||||||
|
|
||||||
return NextResponse.json({
|
|
||||||
stored: {
|
|
||||||
id,
|
|
||||||
name: derivedName,
|
|
||||||
type: 'pdf',
|
|
||||||
size: pdfContent.length,
|
|
||||||
lastModified,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
serverLogger.error({
|
|
||||||
event: 'documents.docx.convert_upload.failed',
|
|
||||||
error: errorToLog(error),
|
|
||||||
}, 'Failed converting/uploading DOCX');
|
|
||||||
return errorResponse(error, {
|
|
||||||
apiErrorMessage: 'Failed to convert document',
|
|
||||||
normalize: { code: 'DOCUMENTS_DOCX_CONVERT_UPLOAD_FAILED', errorClass: 'upstream' },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,25 +1,20 @@
|
||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import { and, count, eq, inArray } from 'drizzle-orm';
|
import { and, count, eq, inArray } from 'drizzle-orm';
|
||||||
import { db } from '@/db';
|
import { db } from '@/db';
|
||||||
import { documents } from '@/db/schema';
|
import { documents } from '@/db/schema';
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
import { toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
||||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||||
import {
|
import {
|
||||||
cleanupDocumentPreviewArtifacts,
|
cleanupDocumentPreviewArtifacts,
|
||||||
deleteDocumentPreviewRows,
|
deleteDocumentPreviewRows,
|
||||||
enqueueDocumentPreview,
|
|
||||||
} from '@/lib/server/documents/previews';
|
} from '@/lib/server/documents/previews';
|
||||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter';
|
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
|
||||||
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
stringifyDocumentParseState,
|
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
|
|
@ -27,14 +22,6 @@ import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
type RegisterDocument = {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
type: DocumentType;
|
|
||||||
size: number;
|
|
||||||
lastModified: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
function s3NotConfiguredResponse(): NextResponse {
|
function s3NotConfiguredResponse(): NextResponse {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||||
|
|
@ -49,175 +36,6 @@ function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType
|
||||||
return toDocumentTypeFromName(safeName);
|
return toDocumentTypeFromName(safeName);
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeLastModified(value: unknown): number {
|
|
||||||
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseDocumentPayload(body: unknown): RegisterDocument[] {
|
|
||||||
if (!body || typeof body !== 'object') return [];
|
|
||||||
const rawDocs = (body as { documents?: unknown }).documents;
|
|
||||||
if (!Array.isArray(rawDocs)) return [];
|
|
||||||
|
|
||||||
const docs: RegisterDocument[] = [];
|
|
||||||
for (const rawDoc of rawDocs) {
|
|
||||||
if (!rawDoc || typeof rawDoc !== 'object') continue;
|
|
||||||
const rec = rawDoc as Record<string, unknown>;
|
|
||||||
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
|
|
||||||
if (!isValidDocumentId(id)) continue;
|
|
||||||
const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`;
|
|
||||||
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
|
|
||||||
const type = normalizeDocumentType(rec.type, name);
|
|
||||||
const lastModified = normalizeLastModified(rec.lastModified);
|
|
||||||
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
|
||||||
docs.push({ id, name, type, size, lastModified });
|
|
||||||
}
|
|
||||||
return docs;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
|
||||||
try {
|
|
||||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
|
||||||
|
|
||||||
const ctxOrRes = await requireAuthContext(req);
|
|
||||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
|
||||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
|
||||||
|
|
||||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
|
||||||
const storageUserId = ctxOrRes.userId;
|
|
||||||
|
|
||||||
const body = await req.json().catch(() => null);
|
|
||||||
const documentsData = parseDocumentPayload(body);
|
|
||||||
if (documentsData.length === 0) {
|
|
||||||
return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const stored: BaseDocument[] = [];
|
|
||||||
|
|
||||||
// Resolve the parse rate-limit config once (only when a PDF is present).
|
|
||||||
const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf')
|
|
||||||
? getPdfLayoutRateConfig(await getResolvedRuntimeConfig())
|
|
||||||
: null;
|
|
||||||
|
|
||||||
for (const doc of documentsData) {
|
|
||||||
let headSize = doc.size;
|
|
||||||
|
|
||||||
// Retry HEAD check to handle S3 read-after-write propagation delays.
|
|
||||||
// The client uploads bytes directly to S3 via presigned URL, then
|
|
||||||
// immediately calls this endpoint. On serverless platforms the HEAD
|
|
||||||
// request may reach S3 before the PUT is visible.
|
|
||||||
const HEAD_RETRIES = 3;
|
|
||||||
const HEAD_RETRY_DELAY_MS = 500;
|
|
||||||
let headError: unknown = null;
|
|
||||||
for (let attempt = 0; attempt < HEAD_RETRIES; attempt++) {
|
|
||||||
headError = null;
|
|
||||||
try {
|
|
||||||
const head = await headDocumentBlob(doc.id, testNamespace);
|
|
||||||
if (head.contentLength > 0) headSize = head.contentLength;
|
|
||||||
break;
|
|
||||||
} catch (error) {
|
|
||||||
if (isMissingBlobError(error)) {
|
|
||||||
headError = error;
|
|
||||||
if (attempt < HEAD_RETRIES - 1) {
|
|
||||||
await new Promise((r) => setTimeout(r, HEAD_RETRY_DELAY_MS));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (headError && isMissingBlobError(headError)) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{
|
|
||||||
error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`,
|
|
||||||
},
|
|
||||||
{ status: 409 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await db
|
|
||||||
.insert(documents)
|
|
||||||
.values({
|
|
||||||
id: doc.id,
|
|
||||||
userId: storageUserId,
|
|
||||||
name: doc.name,
|
|
||||||
type: doc.type,
|
|
||||||
size: headSize,
|
|
||||||
lastModified: doc.lastModified,
|
|
||||||
filePath: doc.id,
|
|
||||||
parseState: doc.type === 'pdf'
|
|
||||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
|
||||||
: null,
|
|
||||||
parsedJsonKey: null,
|
|
||||||
})
|
|
||||||
.onConflictDoUpdate({
|
|
||||||
target: [documents.id, documents.userId],
|
|
||||||
set: {
|
|
||||||
name: doc.name,
|
|
||||||
type: doc.type,
|
|
||||||
size: headSize,
|
|
||||||
lastModified: doc.lastModified,
|
|
||||||
filePath: doc.id,
|
|
||||||
parseState: doc.type === 'pdf'
|
|
||||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
|
||||||
: null,
|
|
||||||
parsedJsonKey: null,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
stored.push({
|
|
||||||
id: doc.id,
|
|
||||||
name: doc.name,
|
|
||||||
type: doc.type,
|
|
||||||
size: headSize,
|
|
||||||
lastModified: doc.lastModified,
|
|
||||||
scope: 'user',
|
|
||||||
});
|
|
||||||
|
|
||||||
await enqueueDocumentPreview(
|
|
||||||
{
|
|
||||||
id: doc.id,
|
|
||||||
type: doc.type,
|
|
||||||
lastModified: doc.lastModified,
|
|
||||||
},
|
|
||||||
testNamespace,
|
|
||||||
).catch((error) => {
|
|
||||||
serverLogger.warn({
|
|
||||||
event: 'documents.preview.enqueue.failed',
|
|
||||||
degraded: true,
|
|
||||||
fallbackPath: 'skip_preview_enqueue',
|
|
||||||
documentId: doc.id,
|
|
||||||
error: errorToLog(error),
|
|
||||||
}, 'Failed to enqueue document preview');
|
|
||||||
});
|
|
||||||
|
|
||||||
if (doc.type === 'pdf') {
|
|
||||||
// Account for upload-driven parse load in the same ledger the explicit
|
|
||||||
// re-parse limiter reads. We record (not reject) here so a legitimate
|
|
||||||
// bulk upload always parses; the recorded load still throttles
|
|
||||||
// subsequent loopable re-parse spam via /parsed.
|
|
||||||
await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] });
|
|
||||||
enqueueParsePdfJob({
|
|
||||||
documentId: doc.id,
|
|
||||||
userId: storageUserId,
|
|
||||||
namespace: testNamespace,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return NextResponse.json({ success: true, stored });
|
|
||||||
} catch (error) {
|
|
||||||
serverLogger.error({
|
|
||||||
event: 'documents.register.failed',
|
|
||||||
error: errorToLog(error),
|
|
||||||
}, 'Failed to register documents');
|
|
||||||
return errorResponse(error, {
|
|
||||||
apiErrorMessage: 'Failed to register documents',
|
|
||||||
normalize: { code: 'DOCUMENTS_REGISTER_FAILED', errorClass: 'db' },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(req: NextRequest) {
|
export async function GET(req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||||
|
|
@ -260,13 +78,16 @@ export async function GET(req: NextRequest) {
|
||||||
|
|
||||||
const results: BaseDocument[] = rows.map((doc) => {
|
const results: BaseDocument[] = rows.map((doc) => {
|
||||||
const type = normalizeDocumentType(doc.type, doc.name);
|
const type = normalizeDocumentType(doc.type, doc.name);
|
||||||
|
const parseState = type === 'pdf'
|
||||||
|
? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState))
|
||||||
|
: null;
|
||||||
return {
|
return {
|
||||||
id: doc.id,
|
id: doc.id,
|
||||||
name: doc.name,
|
name: doc.name,
|
||||||
size: Number(doc.size),
|
size: Number(doc.size),
|
||||||
lastModified: Number(doc.lastModified),
|
lastModified: Number(doc.lastModified),
|
||||||
type,
|
type,
|
||||||
parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null,
|
parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null,
|
||||||
parsedJsonKey: doc.parsedJsonKey,
|
parsedJsonKey: doc.parsedJsonKey,
|
||||||
scope: 'user',
|
scope: 'user',
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -122,16 +122,14 @@ interface DocumentListInnerProps {
|
||||||
function SidebarUploadLoader({
|
function SidebarUploadLoader({
|
||||||
totalFiles,
|
totalFiles,
|
||||||
completedFiles,
|
completedFiles,
|
||||||
phase,
|
|
||||||
currentFileName,
|
currentFileName,
|
||||||
}: {
|
}: {
|
||||||
totalFiles: number;
|
totalFiles: number;
|
||||||
completedFiles: number;
|
completedFiles: number;
|
||||||
phase: 'uploading' | 'converting';
|
phase: 'uploading';
|
||||||
currentFileName: string | null;
|
currentFileName: string | null;
|
||||||
}) {
|
}) {
|
||||||
const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0;
|
const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0;
|
||||||
const label = phase === 'converting' ? 'Converting' : 'Uploading';
|
|
||||||
const radius = 7;
|
const radius = 7;
|
||||||
const stroke = 2;
|
const stroke = 2;
|
||||||
const size = 18;
|
const size = 18;
|
||||||
|
|
@ -143,12 +141,12 @@ function SidebarUploadLoader({
|
||||||
<div className="rounded-md border border-line bg-surface-sunken px-2 py-1.5">
|
<div className="rounded-md border border-line bg-surface-sunken px-2 py-1.5">
|
||||||
<div className="flex items-center justify-between gap-2">
|
<div className="flex items-center justify-between gap-2">
|
||||||
<div className="min-w-0 flex items-center gap-1.5 text-[11px] leading-tight">
|
<div className="min-w-0 flex items-center gap-1.5 text-[11px] leading-tight">
|
||||||
<span className="font-medium text-foreground">{label}</span>
|
<span className="font-medium text-foreground">Uploading</span>
|
||||||
<span className="shrink-0 tabular-nums text-soft">{completedFiles}/{totalFiles}</span>
|
<span className="shrink-0 tabular-nums text-soft">{completedFiles}/{totalFiles}</span>
|
||||||
</div>
|
</div>
|
||||||
<div className="shrink-0 flex items-center gap-1 text-accent" aria-label={`Upload progress ${progress}%`}>
|
<div className="shrink-0 flex items-center gap-1 text-accent" aria-label={`Upload progress ${progress}%`}>
|
||||||
<span className="text-[10px] tabular-nums text-soft">{progress}%</span>
|
<span className="text-[10px] tabular-nums text-soft">{progress}%</span>
|
||||||
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`} className={phase === 'converting' ? 'animate-spin' : ''}>
|
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`}>
|
||||||
<circle
|
<circle
|
||||||
cx={size / 2}
|
cx={size / 2}
|
||||||
cy={size / 2}
|
cy={size / 2}
|
||||||
|
|
@ -228,13 +226,11 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
|
|
||||||
const {
|
const {
|
||||||
pdfDocs,
|
pdfDocs,
|
||||||
removePDFDocument,
|
|
||||||
isPDFLoading,
|
isPDFLoading,
|
||||||
epubDocs,
|
epubDocs,
|
||||||
removeEPUBDocument,
|
|
||||||
isEPUBLoading,
|
isEPUBLoading,
|
||||||
htmlDocs,
|
htmlDocs,
|
||||||
removeHTMLDocument,
|
deleteDocument,
|
||||||
isHTMLLoading,
|
isHTMLLoading,
|
||||||
} = useDocuments();
|
} = useDocuments();
|
||||||
|
|
||||||
|
|
@ -436,9 +432,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
const handleDelete = useCallback(async () => {
|
const handleDelete = useCallback(async () => {
|
||||||
if (!documentToDelete) return;
|
if (!documentToDelete) return;
|
||||||
try {
|
try {
|
||||||
if (documentToDelete.type === 'pdf') await removePDFDocument(documentToDelete.id);
|
await deleteDocument(documentToDelete.id);
|
||||||
else if (documentToDelete.type === 'epub') await removeEPUBDocument(documentToDelete.id);
|
|
||||||
else if (documentToDelete.type === 'html') await removeHTMLDocument(documentToDelete.id);
|
|
||||||
setFolders((prev) =>
|
setFolders((prev) =>
|
||||||
prev.map((f) => ({
|
prev.map((f) => ({
|
||||||
...f,
|
...f,
|
||||||
|
|
@ -451,7 +445,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to remove document:', err);
|
console.error('Failed to remove document:', err);
|
||||||
}
|
}
|
||||||
}, [documentToDelete, removePDFDocument, removeEPUBDocument, removeHTMLDocument]);
|
}, [deleteDocument, documentToDelete]);
|
||||||
|
|
||||||
const handleDeleteDoc = useCallback((doc: DocumentListDocument) => {
|
const handleDeleteDoc = useCallback((doc: DocumentListDocument) => {
|
||||||
setDocumentToDelete({ id: doc.id, name: doc.name, type: doc.type });
|
setDocumentToDelete({ id: doc.id, name: doc.name, type: doc.type });
|
||||||
|
|
@ -577,10 +571,8 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
if (batches.length === 0) return null;
|
if (batches.length === 0) return null;
|
||||||
const totalFiles = batches.reduce((sum, batch) => sum + batch.totalFiles, 0);
|
const totalFiles = batches.reduce((sum, batch) => sum + batch.totalFiles, 0);
|
||||||
const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0);
|
const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0);
|
||||||
const convertingBatch = batches.find((batch) => batch.phase === 'converting');
|
const currentFileName = batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
|
||||||
const phase: 'uploading' | 'converting' = convertingBatch ? 'converting' : 'uploading';
|
return { totalFiles, completedFiles, phase: 'uploading' as const, currentFileName };
|
||||||
const currentFileName = convertingBatch?.currentFileName ?? batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
|
|
||||||
return { totalFiles, completedFiles, phase, currentFileName };
|
|
||||||
}, [activeUploadBatches]);
|
}, [activeUploadBatches]);
|
||||||
|
|
||||||
const fallbackViewMode: ViewMode = viewMode;
|
const fallbackViewMode: ViewMode = viewMode;
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@ import { useState, useCallback, useId, type ReactNode } from 'react';
|
||||||
import { useDropzone } from 'react-dropzone';
|
import { useDropzone } from 'react-dropzone';
|
||||||
import { UploadIcon } from '@/components/icons/Icons';
|
import { UploadIcon } from '@/components/icons/Icons';
|
||||||
import { useDocuments } from '@/contexts/DocumentContext';
|
import { useDocuments } from '@/contexts/DocumentContext';
|
||||||
import { uploadDocxAsPdf } from '@/lib/client/api/documents';
|
|
||||||
import { useFeatureFlag } from '@/contexts/RuntimeConfigContext';
|
import { useFeatureFlag } from '@/contexts/RuntimeConfigContext';
|
||||||
import { dropzoneSurfaceClass } from '@/components/ui';
|
import { dropzoneSurfaceClass } from '@/components/ui';
|
||||||
|
|
||||||
|
|
@ -20,7 +19,7 @@ export interface UploadBatchState {
|
||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
totalFiles: number;
|
totalFiles: number;
|
||||||
completedFiles: number;
|
completedFiles: number;
|
||||||
phase: 'uploading' | 'converting';
|
phase: 'uploading';
|
||||||
currentFileName: string | null;
|
currentFileName: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -33,13 +32,9 @@ export function DocumentUploader({
|
||||||
const uploaderId = useId();
|
const uploaderId = useId();
|
||||||
const enableDocx = useFeatureFlag('enableDocxConversion');
|
const enableDocx = useFeatureFlag('enableDocxConversion');
|
||||||
const {
|
const {
|
||||||
addPDFDocument: addPDF,
|
uploadDocuments,
|
||||||
addEPUBDocument: addEPUB,
|
|
||||||
addHTMLDocument: addHTML,
|
|
||||||
refreshDocuments,
|
|
||||||
} = useDocuments();
|
} = useDocuments();
|
||||||
const [isUploading, setIsUploading] = useState(false);
|
const [isUploading, setIsUploading] = useState(false);
|
||||||
const [isConverting, setIsConverting] = useState(false);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
const emitBatchState = useCallback((state: Omit<UploadBatchState, 'uploaderId'>) => {
|
const emitBatchState = useCallback((state: Omit<UploadBatchState, 'uploaderId'>) => {
|
||||||
|
|
@ -63,72 +58,13 @@ export function DocumentUploader({
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (const file of acceptedFiles) {
|
await uploadDocuments(acceptedFiles);
|
||||||
if (file.type === 'application/pdf') {
|
completedFiles = acceptedFiles.length;
|
||||||
emitBatchState({
|
|
||||||
isActive: true,
|
|
||||||
totalFiles,
|
|
||||||
completedFiles,
|
|
||||||
phase: 'uploading',
|
|
||||||
currentFileName: file.name,
|
|
||||||
});
|
|
||||||
await addPDF(file);
|
|
||||||
completedFiles += 1;
|
|
||||||
} else if (file.type === 'application/epub+zip') {
|
|
||||||
emitBatchState({
|
|
||||||
isActive: true,
|
|
||||||
totalFiles,
|
|
||||||
completedFiles,
|
|
||||||
phase: 'uploading',
|
|
||||||
currentFileName: file.name,
|
|
||||||
});
|
|
||||||
await addEPUB(file);
|
|
||||||
completedFiles += 1;
|
|
||||||
} else if (file.type === 'text/plain' || file.type === 'text/markdown' || file.name.endsWith('.md')) {
|
|
||||||
emitBatchState({
|
|
||||||
isActive: true,
|
|
||||||
totalFiles,
|
|
||||||
completedFiles,
|
|
||||||
phase: 'uploading',
|
|
||||||
currentFileName: file.name,
|
|
||||||
});
|
|
||||||
await addHTML(file);
|
|
||||||
completedFiles += 1;
|
|
||||||
} else if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
|
|
||||||
// Preserve prior UX: show "Converting DOCX..." state rather than generic uploading.
|
|
||||||
setIsUploading(false);
|
|
||||||
setIsConverting(true);
|
|
||||||
emitBatchState({
|
|
||||||
isActive: true,
|
|
||||||
totalFiles,
|
|
||||||
completedFiles,
|
|
||||||
phase: 'converting',
|
|
||||||
currentFileName: file.name,
|
|
||||||
});
|
|
||||||
// Convert+upload directly on the server. Use sha(docx) as stable ID to avoid duplicates.
|
|
||||||
await uploadDocxAsPdf(file);
|
|
||||||
await refreshDocuments();
|
|
||||||
setIsConverting(false);
|
|
||||||
setIsUploading(true);
|
|
||||||
completedFiles += 1;
|
|
||||||
} else {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
emitBatchState({
|
|
||||||
isActive: true,
|
|
||||||
totalFiles,
|
|
||||||
completedFiles,
|
|
||||||
phase: 'uploading',
|
|
||||||
currentFileName: null,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError('Failed to upload file. Please try again.');
|
setError('Failed to upload file. Please try again.');
|
||||||
console.error('Upload error:', err);
|
console.error('Upload error:', err);
|
||||||
} finally {
|
} finally {
|
||||||
setIsUploading(false);
|
setIsUploading(false);
|
||||||
setIsConverting(false);
|
|
||||||
emitBatchState({
|
emitBatchState({
|
||||||
isActive: false,
|
isActive: false,
|
||||||
totalFiles,
|
totalFiles,
|
||||||
|
|
@ -137,7 +73,7 @@ export function DocumentUploader({
|
||||||
currentFileName: null,
|
currentFileName: null,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}, [addHTML, addPDF, addEPUB, refreshDocuments, enableDocx, emitBatchState]);
|
}, [uploadDocuments, emitBatchState]);
|
||||||
|
|
||||||
const { getRootProps, getInputProps, isDragActive } = useDropzone({
|
const { getRootProps, getInputProps, isDragActive } = useDropzone({
|
||||||
onDrop,
|
onDrop,
|
||||||
|
|
@ -151,12 +87,12 @@ export function DocumentUploader({
|
||||||
} : {})
|
} : {})
|
||||||
},
|
},
|
||||||
multiple: true,
|
multiple: true,
|
||||||
disabled: isUploading || isConverting,
|
disabled: isUploading,
|
||||||
noClick: variant === 'overlay',
|
noClick: variant === 'overlay',
|
||||||
noKeyboard: variant === 'overlay'
|
noKeyboard: variant === 'overlay'
|
||||||
});
|
});
|
||||||
|
|
||||||
const isDisabled = isUploading || isConverting;
|
const isDisabled = isUploading;
|
||||||
|
|
||||||
if (variant === 'overlay') {
|
if (variant === 'overlay') {
|
||||||
const rootProps = getRootProps();
|
const rootProps = getRootProps();
|
||||||
|
|
@ -209,8 +145,6 @@ export function DocumentUploader({
|
||||||
<UploadIcon className="w-3.5 h-3.5 text-soft group-hover:text-accent shrink-0 transition-colors duration-base" />
|
<UploadIcon className="w-3.5 h-3.5 text-soft group-hover:text-accent shrink-0 transition-colors duration-base" />
|
||||||
{isUploading ? (
|
{isUploading ? (
|
||||||
<p className="text-[12px] font-medium truncate flex-1">Uploading…</p>
|
<p className="text-[12px] font-medium truncate flex-1">Uploading…</p>
|
||||||
) : isConverting ? (
|
|
||||||
<p className="text-[12px] font-medium truncate flex-1">Converting DOCX…</p>
|
|
||||||
) : (
|
) : (
|
||||||
<div className="flex items-center gap-2 min-w-0 flex-1">
|
<div className="flex items-center gap-2 min-w-0 flex-1">
|
||||||
<p className="text-[12px] truncate flex-1">
|
<p className="text-[12px] truncate flex-1">
|
||||||
|
|
@ -225,8 +159,6 @@ export function DocumentUploader({
|
||||||
<UploadIcon className="w-7 h-7 sm:w-10 sm:h-10 mb-2 text-soft" />
|
<UploadIcon className="w-7 h-7 sm:w-10 sm:h-10 mb-2 text-soft" />
|
||||||
{isUploading ? (
|
{isUploading ? (
|
||||||
<p className="text-sm sm:text-lg font-semibold text-foreground">Uploading file...</p>
|
<p className="text-sm sm:text-lg font-semibold text-foreground">Uploading file...</p>
|
||||||
) : isConverting ? (
|
|
||||||
<p className="text-sm sm:text-lg font-semibold text-foreground">Converting DOCX to PDF...</p>
|
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
<p className="mb-2 text-sm sm:text-lg font-semibold text-foreground">
|
<p className="mb-2 text-sm sm:text-lg font-semibold text-foreground">
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl';
|
||||||
import { Navigator } from '@/components/player/Navigator';
|
import { Navigator } from '@/components/player/Navigator';
|
||||||
import { IconButton } from '@/components/ui';
|
import { IconButton } from '@/components/ui';
|
||||||
|
|
||||||
export default function TTSPlayer({ currentPage, numPages }: {
|
export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: {
|
||||||
currentPage?: number;
|
currentPage?: number;
|
||||||
numPages?: number | undefined;
|
numPages?: number | undefined;
|
||||||
|
isPlaybackReady?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const {
|
const {
|
||||||
isPlaying,
|
isPlaying,
|
||||||
|
|
@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={skipBackward}
|
onClick={skipBackward}
|
||||||
aria-label="Skip backward"
|
aria-label="Skip backward"
|
||||||
disabled={isProcessing}
|
disabled={isProcessing || !isPlaybackReady}
|
||||||
className="relative"
|
className="relative"
|
||||||
>
|
>
|
||||||
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
|
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
|
||||||
|
|
@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={togglePlay}
|
onClick={togglePlay}
|
||||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||||
disabled={isProcessing && !isPlaying}
|
disabled={!isPlaying && (!isPlaybackReady || isProcessing)}
|
||||||
className="relative"
|
className="relative"
|
||||||
>
|
>
|
||||||
{isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />}
|
{!isPlaying && !isPlaybackReady
|
||||||
|
? <LoadingSpinner />
|
||||||
|
: (isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />)}
|
||||||
</IconButton>
|
</IconButton>
|
||||||
|
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={skipForward}
|
onClick={skipForward}
|
||||||
aria-label="Skip forward"
|
aria-label="Skip forward"
|
||||||
disabled={isProcessing}
|
disabled={isProcessing || !isPlaybackReady}
|
||||||
className="relative"
|
className="relative"
|
||||||
>
|
>
|
||||||
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}
|
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}
|
||||||
|
|
|
||||||
|
|
@ -3,32 +3,53 @@
|
||||||
import { createContext, useCallback, useContext, useEffect, useMemo, ReactNode } from 'react';
|
import { createContext, useCallback, useContext, useEffect, useMemo, ReactNode } from 'react';
|
||||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||||
import type { BaseDocument } from '@/types/documents';
|
import type { BaseDocument } from '@/types/documents';
|
||||||
import { listDocuments, uploadDocuments, deleteDocuments } from '@/lib/client/api/documents';
|
import {
|
||||||
import { putCachedEpub, putCachedHtml, putCachedPdf, evictCachedEpub, evictCachedHtml, evictCachedPdf } from '@/lib/client/cache/documents';
|
deleteDocuments as deleteServerDocuments,
|
||||||
|
listDocuments,
|
||||||
|
uploadDocuments as uploadServerDocuments,
|
||||||
|
} from '@/lib/client/api/documents';
|
||||||
|
import { cacheStoredDocumentFromBytes, evictCachedDocument } from '@/lib/client/cache/documents';
|
||||||
import { useAuthSession } from '@/hooks/useAuthSession';
|
import { useAuthSession } from '@/hooks/useAuthSession';
|
||||||
|
|
||||||
interface DocumentContextType {
|
interface DocumentContextType {
|
||||||
pdfDocs: Array<BaseDocument & { type: 'pdf' }>;
|
pdfDocs: Array<BaseDocument & { type: 'pdf' }>;
|
||||||
addPDFDocument: (file: File) => Promise<string>;
|
|
||||||
removePDFDocument: (id: string) => Promise<void>;
|
|
||||||
isPDFLoading: boolean;
|
isPDFLoading: boolean;
|
||||||
|
|
||||||
epubDocs: Array<BaseDocument & { type: 'epub' }>;
|
epubDocs: Array<BaseDocument & { type: 'epub' }>;
|
||||||
addEPUBDocument: (file: File) => Promise<string>;
|
|
||||||
removeEPUBDocument: (id: string) => Promise<void>;
|
|
||||||
isEPUBLoading: boolean;
|
isEPUBLoading: boolean;
|
||||||
|
|
||||||
htmlDocs: Array<BaseDocument & { type: 'html' }>;
|
htmlDocs: Array<BaseDocument & { type: 'html' }>;
|
||||||
addHTMLDocument: (file: File) => Promise<string>;
|
|
||||||
removeHTMLDocument: (id: string) => Promise<void>;
|
|
||||||
isHTMLLoading: boolean;
|
isHTMLLoading: boolean;
|
||||||
|
|
||||||
|
uploadDocuments: (files: File[]) => Promise<BaseDocument[]>;
|
||||||
|
deleteDocument: (id: string) => Promise<void>;
|
||||||
refreshDocuments: () => Promise<void>;
|
refreshDocuments: () => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DocumentContext = createContext<DocumentContextType | undefined>(undefined);
|
const DocumentContext = createContext<DocumentContextType | undefined>(undefined);
|
||||||
const DOCUMENTS_QUERY_KEY = 'documents';
|
const DOCUMENTS_QUERY_KEY = 'documents';
|
||||||
|
|
||||||
|
type SupportedDocument = BaseDocument & { type: 'pdf' | 'epub' | 'html' };
|
||||||
|
|
||||||
|
function mergeStoredDocuments(
|
||||||
|
previous: SupportedDocument[] | undefined,
|
||||||
|
uploaded: BaseDocument[],
|
||||||
|
): SupportedDocument[] {
|
||||||
|
const next = [...(previous ?? [])];
|
||||||
|
|
||||||
|
for (let index = uploaded.length - 1; index >= 0; index -= 1) {
|
||||||
|
const stored = uploaded[index];
|
||||||
|
if (stored.type !== 'pdf' && stored.type !== 'epub' && stored.type !== 'html') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const supportedStored = stored as SupportedDocument;
|
||||||
|
const withoutExisting = next.filter((document) => document.id !== supportedStored.id);
|
||||||
|
next.splice(0, next.length, supportedStored, ...withoutExisting);
|
||||||
|
}
|
||||||
|
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
export function DocumentProvider({ children }: { children: ReactNode }) {
|
export function DocumentProvider({ children }: { children: ReactNode }) {
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
const { data: sessionData, isPending: isSessionPending } = useAuthSession();
|
const { data: sessionData, isPending: isSessionPending } = useAuthSession();
|
||||||
|
|
@ -81,67 +102,63 @@ export function DocumentProvider({ children }: { children: ReactNode }) {
|
||||||
return { pdfDocs, epubDocs, htmlDocs };
|
return { pdfDocs, epubDocs, htmlDocs };
|
||||||
}, [docs]);
|
}, [docs]);
|
||||||
|
|
||||||
const cacheUploaded = useCallback(async (stored: BaseDocument, file: File) => {
|
const uploadDocuments = useCallback(async (files: File[]): Promise<BaseDocument[]> => {
|
||||||
try {
|
if (files.length === 0) return [];
|
||||||
if (stored.type === 'pdf') {
|
|
||||||
await putCachedPdf(stored, await file.arrayBuffer());
|
|
||||||
} else if (stored.type === 'epub') {
|
|
||||||
await putCachedEpub(stored, await file.arrayBuffer());
|
|
||||||
} else if (stored.type === 'html') {
|
|
||||||
const buf = await file.arrayBuffer();
|
|
||||||
const decoded = new TextDecoder().decode(new Uint8Array(buf));
|
|
||||||
await putCachedHtml(stored, decoded);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
// Cache failures should not block uploads.
|
|
||||||
console.warn('Failed to cache uploaded document:', stored.id, err);
|
|
||||||
}
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
const addDocument = useCallback(async (file: File): Promise<string> => {
|
const stored = await uploadServerDocuments(files);
|
||||||
const [stored] = await uploadDocuments([file]);
|
await Promise.allSettled(
|
||||||
if (!stored) throw new Error('Upload succeeded but returned no document');
|
stored.map(async (document, index) => {
|
||||||
await cacheUploaded(stored, file);
|
const file = files[index];
|
||||||
const isSupported = stored.type === 'pdf' || stored.type === 'epub' || stored.type === 'html';
|
if (!file) return;
|
||||||
if (!isSupported) return stored.id;
|
const sourceType = file.name
|
||||||
const supportedStored = stored as BaseDocument & { type: 'pdf' | 'epub' | 'html' };
|
? (
|
||||||
queryClient.setQueryData<Array<BaseDocument & { type: 'pdf' | 'epub' | 'html' }>>(documentsQueryKey, (prev = []) => {
|
file.name.toLowerCase().endsWith('.pdf')
|
||||||
const next = prev.filter((d) => d.id !== supportedStored.id);
|
? 'pdf'
|
||||||
return [supportedStored, ...next];
|
: file.name.toLowerCase().endsWith('.epub')
|
||||||
});
|
? 'epub'
|
||||||
return stored.id;
|
: file.name.toLowerCase().endsWith('.docx')
|
||||||
}, [cacheUploaded, queryClient, documentsQueryKey]);
|
? 'docx'
|
||||||
|
: 'html'
|
||||||
const addPDFDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
|
)
|
||||||
const addEPUBDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
|
: (
|
||||||
const addHTMLDocument = useCallback(async (file: File) => addDocument(file), [addDocument]);
|
file.type === 'application/pdf'
|
||||||
|
? 'pdf'
|
||||||
const removeById = useCallback(async (id: string) => {
|
: file.type === 'application/epub+zip'
|
||||||
await deleteDocuments({ ids: [id] });
|
? 'epub'
|
||||||
await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]);
|
: file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||||
queryClient.setQueryData<Array<BaseDocument & { type: 'pdf' | 'epub' | 'html' }>>(documentsQueryKey, (prev = []) =>
|
? 'docx'
|
||||||
prev.filter((d) => d.id !== id),
|
: 'html'
|
||||||
|
);
|
||||||
|
if (document.type !== sourceType) return;
|
||||||
|
await cacheStoredDocumentFromBytes(document, await file.arrayBuffer());
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
}, [queryClient, documentsQueryKey]);
|
|
||||||
|
|
||||||
const removePDFDocument = useCallback(async (id: string) => removeById(id), [removeById]);
|
queryClient.setQueryData<SupportedDocument[]>(documentsQueryKey, (previous) =>
|
||||||
const removeEPUBDocument = useCallback(async (id: string) => removeById(id), [removeById]);
|
mergeStoredDocuments(previous, stored),
|
||||||
const removeHTMLDocument = useCallback(async (id: string) => removeById(id), [removeById]);
|
);
|
||||||
|
|
||||||
|
return stored;
|
||||||
|
}, [documentsQueryKey, queryClient]);
|
||||||
|
|
||||||
|
const deleteDocument = useCallback(async (id: string) => {
|
||||||
|
await deleteServerDocuments({ ids: [id] });
|
||||||
|
await evictCachedDocument(id);
|
||||||
|
queryClient.setQueryData<SupportedDocument[]>(documentsQueryKey, (previous = []) =>
|
||||||
|
previous.filter((document) => document.id !== id),
|
||||||
|
);
|
||||||
|
}, [documentsQueryKey, queryClient]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<DocumentContext.Provider value={{
|
<DocumentContext.Provider value={{
|
||||||
pdfDocs: docsByType.pdfDocs,
|
pdfDocs: docsByType.pdfDocs,
|
||||||
addPDFDocument,
|
|
||||||
removePDFDocument,
|
|
||||||
isPDFLoading: isLoading,
|
isPDFLoading: isLoading,
|
||||||
epubDocs: docsByType.epubDocs,
|
epubDocs: docsByType.epubDocs,
|
||||||
addEPUBDocument,
|
|
||||||
removeEPUBDocument,
|
|
||||||
isEPUBLoading: isLoading,
|
isEPUBLoading: isLoading,
|
||||||
htmlDocs: docsByType.htmlDocs,
|
htmlDocs: docsByType.htmlDocs,
|
||||||
addHTMLDocument,
|
|
||||||
removeHTMLDocument,
|
|
||||||
isHTMLLoading: isLoading,
|
isHTMLLoading: isLoading,
|
||||||
|
uploadDocuments,
|
||||||
|
deleteDocument,
|
||||||
refreshDocuments,
|
refreshDocuments,
|
||||||
|
|
||||||
}}>
|
}}>
|
||||||
|
|
|
||||||
|
|
@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
||||||
const isPlayingRef = useRef(false);
|
const isPlayingRef = useRef(false);
|
||||||
const pauseEpochRef = useRef(0);
|
const pauseEpochRef = useRef(0);
|
||||||
|
const pendingPlaybackStartRef = useRef(false);
|
||||||
const sentencesRef = useRef<string[]>([]);
|
const sentencesRef = useRef<string[]>([]);
|
||||||
const currentIndexRef = useRef(0);
|
const currentIndexRef = useRef(0);
|
||||||
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
|
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
|
||||||
|
|
@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const recordManualPause = useCallback(() => {
|
const recordManualPause = useCallback(() => {
|
||||||
// Cancel any queued auto-resume intent and mark an explicit user pause.
|
// Cancel any queued auto-resume intent and mark an explicit user pause.
|
||||||
resumeAfterLocationChangeRef.current = false;
|
resumeAfterLocationChangeRef.current = false;
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
pauseEpochRef.current += 1;
|
pauseEpochRef.current += 1;
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
|
@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
* Used for external control of playback state
|
* Used for external control of playback state
|
||||||
*/
|
*/
|
||||||
const pause = useCallback(() => {
|
const pause = useCallback(() => {
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
recordManualPause();
|
recordManualPause();
|
||||||
clearPendingEpubJump();
|
clearPendingEpubJump();
|
||||||
abortPendingTtsRequests(true);
|
abortPendingTtsRequests(true);
|
||||||
|
|
@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const shouldPause = normalizedOptions.shouldPause ?? false;
|
const shouldPause = normalizedOptions.shouldPause ?? false;
|
||||||
const pauseEpochAtStart = pauseEpochRef.current;
|
const pauseEpochAtStart = pauseEpochRef.current;
|
||||||
const pendingAutoResume = resumeAfterLocationChangeRef.current;
|
const pendingAutoResume = resumeAfterLocationChangeRef.current;
|
||||||
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume);
|
const pendingPlaybackStart = pendingPlaybackStartRef.current;
|
||||||
if (shouldPause || pendingAutoResume) {
|
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart);
|
||||||
resumeAfterLocationChangeRef.current = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keep track of previous state and pause playback
|
// Keep track of previous state and pause playback
|
||||||
invalidatePlaybackRun();
|
invalidatePlaybackRun();
|
||||||
|
|
@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
try {
|
try {
|
||||||
if (newSentences.length === 0) {
|
if (newSentences.length === 0) {
|
||||||
console.warn('No sentences found in text');
|
console.warn('No sentences found in text');
|
||||||
|
if (shouldPause || pendingAutoResume) {
|
||||||
|
resumeAfterLocationChangeRef.current = false;
|
||||||
|
}
|
||||||
|
if (pendingPlaybackStart) {
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
|
}
|
||||||
setIsProcessing(false);
|
setIsProcessing(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (shouldPause || pendingAutoResume) {
|
||||||
|
resumeAfterLocationChangeRef.current = false;
|
||||||
|
}
|
||||||
|
if (pendingPlaybackStart) {
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
|
}
|
||||||
|
|
||||||
if (!isEPUB && typeof resolvedLocation === 'number') {
|
if (!isEPUB && typeof resolvedLocation === 'number') {
|
||||||
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
|
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
|
||||||
if (firstFingerprint) {
|
if (firstFingerprint) {
|
||||||
|
|
@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
// Ensure audio is unlocked while we're still in the click/tap handler.
|
// Ensure audio is unlocked while we're still in the click/tap handler.
|
||||||
unlockPlaybackOnUserGesture();
|
unlockPlaybackOnUserGesture();
|
||||||
|
|
||||||
|
const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]);
|
||||||
|
if (!activeHowl && !hasPreparedSentence) {
|
||||||
|
pendingPlaybackStartRef.current = true;
|
||||||
|
setIsProcessing(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Resume current sentence if we already have a paused Howl.
|
// Resume current sentence if we already have a paused Howl.
|
||||||
if (activeHowl) {
|
if (activeHowl) {
|
||||||
applyPlaybackRateToHowl(activeHowl);
|
applyPlaybackRateToHowl(activeHowl);
|
||||||
|
|
@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
setIsPlaying(true);
|
setIsPlaying(true);
|
||||||
}, [
|
}, [
|
||||||
activeHowl,
|
activeHowl,
|
||||||
|
currentIndex,
|
||||||
applyPlaybackRateToHowl,
|
applyPlaybackRateToHowl,
|
||||||
isPlaying,
|
isPlaying,
|
||||||
pauseActiveHowl,
|
pauseActiveHowl,
|
||||||
|
playbackSegments,
|
||||||
recordManualPause,
|
recordManualPause,
|
||||||
clearPendingEpubJump,
|
clearPendingEpubJump,
|
||||||
abortPendingTtsRequests,
|
abortPendingTtsRequests,
|
||||||
|
sentences,
|
||||||
unlockPlaybackOnUserGesture,
|
unlockPlaybackOnUserGesture,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
abortAudio();
|
abortAudio();
|
||||||
clearWarmAudioCache();
|
clearWarmAudioCache();
|
||||||
playbackInFlightRef.current = false;
|
playbackInFlightRef.current = false;
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
pendingJumpTargetRef.current = null;
|
pendingJumpTargetRef.current = null;
|
||||||
clearPendingEpubJump();
|
clearPendingEpubJump();
|
||||||
bumpEpubPreloadGeneration();
|
bumpEpubPreloadGeneration();
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,8 @@
|
||||||
import { sha256HexFromArrayBuffer } from '@/lib/client/sha256';
|
|
||||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
import type { DocumentSettings } from '@/types/document-settings';
|
import type { DocumentSettings } from '@/types/document-settings';
|
||||||
|
|
||||||
export type UploadSource = {
|
export type UploadSource = {
|
||||||
id: string;
|
|
||||||
name: string;
|
name: string;
|
||||||
type: DocumentType;
|
type: DocumentType;
|
||||||
size: number;
|
size: number;
|
||||||
|
|
@ -23,8 +21,12 @@ function toUploadBody(body: UploadSource['body']): BodyInit {
|
||||||
return body as unknown as BodyInit;
|
return body as unknown as BodyInit;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise<void> {
|
async function uploadDocumentSourceViaProxy(
|
||||||
const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, {
|
source: UploadSource,
|
||||||
|
token: string,
|
||||||
|
options?: UploadOptions,
|
||||||
|
): Promise<void> {
|
||||||
|
const res = await fetch(`/api/documents/blob/upload/fallback?token=${encodeURIComponent(token)}`, {
|
||||||
method: 'PUT',
|
method: 'PUT',
|
||||||
headers: { 'Content-Type': source.contentType || 'application/octet-stream' },
|
headers: { 'Content-Type': source.contentType || 'application/octet-stream' },
|
||||||
body: toUploadBody(source.body),
|
body: toUploadBody(source.body),
|
||||||
|
|
@ -45,6 +47,14 @@ function documentTypeForName(name: string): DocumentType {
|
||||||
return 'html';
|
return 'html';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function documentTypeForMime(contentType: string): DocumentType | null {
|
||||||
|
const normalized = contentType.trim().toLowerCase();
|
||||||
|
if (normalized === 'application/pdf') return 'pdf';
|
||||||
|
if (normalized === 'application/epub+zip') return 'epub';
|
||||||
|
if (normalized === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') return 'docx';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string {
|
export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string {
|
||||||
if (doc.type === 'pdf') return 'application/pdf';
|
if (doc.type === 'pdf') return 'application/pdf';
|
||||||
if (doc.type === 'epub') return 'application/epub+zip';
|
if (doc.type === 'epub') return 'application/epub+zip';
|
||||||
|
|
@ -80,31 +90,20 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
|
||||||
|
|
||||||
export async function getParsedPdfDocument(
|
export async function getParsedPdfDocument(
|
||||||
id: string,
|
id: string,
|
||||||
options?: { signal?: AbortSignal; retryFailed?: boolean; opId?: string },
|
options?: { signal?: AbortSignal },
|
||||||
): Promise<
|
): Promise<ParsedPdfDocument> {
|
||||||
| { status: 'ready'; parsed: ParsedPdfDocument }
|
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed`, {
|
||||||
| { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null; opId?: string | null }
|
|
||||||
> {
|
|
||||||
const params = new URLSearchParams();
|
|
||||||
if (options?.retryFailed) params.set('retry', '1');
|
|
||||||
if (options?.opId) params.set('opId', options.opId);
|
|
||||||
const query = params.size > 0 ? `?${params.toString()}` : '';
|
|
||||||
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, {
|
|
||||||
signal: options?.signal,
|
signal: options?.signal,
|
||||||
cache: 'no-store',
|
cache: 'no-store',
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 202) {
|
if (res.status === 409) {
|
||||||
const data = (await res.json().catch(() => null)) as {
|
const data = (await res.json().catch(() => null)) as {
|
||||||
parseStatus?: string;
|
parseStatus?: string;
|
||||||
parseProgress?: PdfParseProgress | null;
|
parseProgress?: PdfParseProgress | null;
|
||||||
opId?: string | null;
|
opId?: string | null;
|
||||||
} | null;
|
} | null;
|
||||||
const parseStatus = data?.parseStatus;
|
throw new Error(data?.parseStatus ? `Parsed PDF is not ready (${data.parseStatus})` : 'Parsed PDF is not ready');
|
||||||
if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') {
|
|
||||||
return { status: parseStatus, parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null };
|
|
||||||
}
|
|
||||||
return { status: 'pending', parseProgress: data?.parseProgress ?? null, opId: data?.opId ?? null };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
|
|
@ -112,8 +111,7 @@ export async function getParsedPdfDocument(
|
||||||
throw new Error(data?.error || 'Failed to load parsed PDF');
|
throw new Error(data?.error || 'Failed to load parsed PDF');
|
||||||
}
|
}
|
||||||
|
|
||||||
const parsed = (await res.json()) as ParsedPdfDocument;
|
return (await res.json()) as ParsedPdfDocument;
|
||||||
return { status: 'ready', parsed };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function subscribeParsedPdfDocumentEvents(
|
export function subscribeParsedPdfDocumentEvents(
|
||||||
|
|
@ -241,7 +239,6 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
uploads: sources.map((source) => ({
|
uploads: sources.map((source) => ({
|
||||||
id: source.id,
|
|
||||||
contentType: source.contentType,
|
contentType: source.contentType,
|
||||||
size: source.size,
|
size: source.size,
|
||||||
})),
|
})),
|
||||||
|
|
@ -255,14 +252,18 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
||||||
}
|
}
|
||||||
|
|
||||||
const presigned = (await presignRes.json()) as {
|
const presigned = (await presignRes.json()) as {
|
||||||
uploads?: Array<{ id: string; url: string; headers?: Record<string, string> }>;
|
uploads?: Array<{ token: string; url: string; headers?: Record<string, string> }>;
|
||||||
};
|
};
|
||||||
const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload]));
|
const uploads = presigned.uploads || [];
|
||||||
|
if (uploads.length !== sources.length) {
|
||||||
|
throw new Error('Upload preparation returned an unexpected number of temp uploads');
|
||||||
|
}
|
||||||
|
|
||||||
for (const source of sources) {
|
for (let index = 0; index < sources.length; index += 1) {
|
||||||
const upload = byId.get(source.id);
|
const source = sources[index];
|
||||||
if (!upload?.url) {
|
const upload = uploads[index];
|
||||||
throw new Error(`Missing presigned upload for document ${source.id}`);
|
if (!upload?.url || !upload.token) {
|
||||||
|
throw new Error(`Missing presigned upload for document ${source.name}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
let putError: unknown = null;
|
let putError: unknown = null;
|
||||||
|
|
@ -285,7 +286,7 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await uploadDocumentSourceViaProxy(source, options);
|
await uploadDocumentSourceViaProxy(source, upload.token, options);
|
||||||
} catch (proxyError) {
|
} catch (proxyError) {
|
||||||
const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error';
|
const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error';
|
||||||
const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error';
|
const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error';
|
||||||
|
|
@ -293,27 +294,26 @@ export async function uploadDocumentSources(sources: UploadSource[], options?: U
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const registerRes = await fetch('/api/documents', {
|
const finalizeRes = await fetch('/api/documents/blob/upload/finalize', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
documents: sources.map((source) => ({
|
uploads: sources.map((source, index) => ({
|
||||||
id: source.id,
|
token: uploads[index]?.token,
|
||||||
name: source.name,
|
name: source.name,
|
||||||
type: source.type,
|
type: source.type,
|
||||||
size: source.size,
|
|
||||||
lastModified: source.lastModified,
|
lastModified: source.lastModified,
|
||||||
})),
|
})),
|
||||||
}),
|
}),
|
||||||
signal: options?.signal,
|
signal: options?.signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!registerRes.ok) {
|
if (!finalizeRes.ok) {
|
||||||
const data = (await registerRes.json().catch(() => null)) as { error?: string } | null;
|
const data = (await finalizeRes.json().catch(() => null)) as { error?: string } | null;
|
||||||
throw new Error(data?.error || 'Failed to register uploaded documents');
|
throw new Error(data?.error || 'Failed to finalize uploaded documents');
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = (await registerRes.json()) as { stored: BaseDocument[] };
|
const data = (await finalizeRes.json()) as { stored: BaseDocument[] };
|
||||||
return data.stored || [];
|
return data.stored || [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -322,14 +322,14 @@ export async function uploadDocuments(files: File[], options?: UploadOptions): P
|
||||||
|
|
||||||
const sources: UploadSource[] = [];
|
const sources: UploadSource[] = [];
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const bytes = await file.arrayBuffer();
|
const name = file.name || '';
|
||||||
const id = await sha256HexFromArrayBuffer(bytes);
|
const type = name
|
||||||
const type = documentTypeForName(file.name);
|
? documentTypeForName(name)
|
||||||
const name = file.name || `${id}.${type}`;
|
: (documentTypeForMime(file.type) ?? 'html');
|
||||||
const contentType = file.type || mimeTypeForDoc({ name, type });
|
const resolvedName = name || `upload.${type}`;
|
||||||
|
const contentType = file.type || mimeTypeForDoc({ name: resolvedName, type });
|
||||||
sources.push({
|
sources.push({
|
||||||
id,
|
name: resolvedName,
|
||||||
name,
|
|
||||||
type,
|
type,
|
||||||
size: file.size,
|
size: file.size,
|
||||||
lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(),
|
lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(),
|
||||||
|
|
@ -504,23 +504,3 @@ export async function getDocumentPreviewStatus(
|
||||||
|
|
||||||
throw new Error(`Failed to load preview status (status ${res.status})`);
|
throw new Error(`Failed to load preview status (status ${res.status})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise<BaseDocument> {
|
|
||||||
const form = new FormData();
|
|
||||||
form.append('file', file);
|
|
||||||
|
|
||||||
const res = await fetch('/api/documents/docx-to-pdf/upload', {
|
|
||||||
method: 'POST',
|
|
||||||
body: form,
|
|
||||||
signal: options?.signal,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
|
||||||
throw new Error(data?.error || 'Failed to convert DOCX');
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await res.json()) as { stored: BaseDocument };
|
|
||||||
if (!data?.stored) throw new Error('DOCX conversion succeeded but returned no document');
|
|
||||||
return data.stored;
|
|
||||||
}
|
|
||||||
|
|
|
||||||
4
src/lib/client/cache/documents.ts
vendored
4
src/lib/client/cache/documents.ts
vendored
|
|
@ -107,6 +107,10 @@ export async function evictCachedHtml(id: string): Promise<void> {
|
||||||
await removeHtmlDocument(id);
|
await removeHtmlDocument(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function evictCachedDocument(id: string): Promise<void> {
|
||||||
|
await Promise.allSettled([evictCachedPdf(id), evictCachedEpub(id), evictCachedHtml(id)]);
|
||||||
|
}
|
||||||
|
|
||||||
export async function clearDocumentCache(): Promise<void> {
|
export async function clearDocumentCache(): Promise<void> {
|
||||||
const { clearPdfDocuments, clearEpubDocuments, clearHtmlDocuments } = await import('@/lib/client/dexie');
|
const { clearPdfDocuments, clearEpubDocuments, clearHtmlDocuments } = await import('@/lib/client/dexie');
|
||||||
await Promise.all([clearPdfDocuments(), clearEpubDocuments(), clearHtmlDocuments()]);
|
await Promise.all([clearPdfDocuments(), clearEpubDocuments(), clearHtmlDocuments()]);
|
||||||
|
|
|
||||||
|
|
@ -971,11 +971,9 @@ export async function syncDocumentsToServer(
|
||||||
|
|
||||||
for (const doc of pdfDocs) {
|
for (const doc of pdfDocs) {
|
||||||
const bytes = new Uint8Array(doc.data);
|
const bytes = new Uint8Array(doc.data);
|
||||||
const id = await sha256HexFromBytes(bytes);
|
|
||||||
uploads.push({
|
uploads.push({
|
||||||
oldId: doc.id,
|
oldId: doc.id,
|
||||||
source: {
|
source: {
|
||||||
id,
|
|
||||||
name: doc.name,
|
name: doc.name,
|
||||||
type: 'pdf',
|
type: 'pdf',
|
||||||
size: bytes.byteLength,
|
size: bytes.byteLength,
|
||||||
|
|
@ -992,11 +990,9 @@ export async function syncDocumentsToServer(
|
||||||
|
|
||||||
for (const doc of epubDocs) {
|
for (const doc of epubDocs) {
|
||||||
const bytes = new Uint8Array(doc.data);
|
const bytes = new Uint8Array(doc.data);
|
||||||
const id = await sha256HexFromBytes(bytes);
|
|
||||||
uploads.push({
|
uploads.push({
|
||||||
oldId: doc.id,
|
oldId: doc.id,
|
||||||
source: {
|
source: {
|
||||||
id,
|
|
||||||
name: doc.name,
|
name: doc.name,
|
||||||
type: 'epub',
|
type: 'epub',
|
||||||
size: bytes.byteLength,
|
size: bytes.byteLength,
|
||||||
|
|
@ -1013,11 +1009,9 @@ export async function syncDocumentsToServer(
|
||||||
|
|
||||||
for (const doc of htmlDocs) {
|
for (const doc of htmlDocs) {
|
||||||
const encoded = textEncoder.encode(doc.data);
|
const encoded = textEncoder.encode(doc.data);
|
||||||
const id = await sha256HexFromBytes(encoded);
|
|
||||||
uploads.push({
|
uploads.push({
|
||||||
oldId: doc.id,
|
oldId: doc.id,
|
||||||
source: {
|
source: {
|
||||||
id,
|
|
||||||
name: doc.name,
|
name: doc.name,
|
||||||
type: 'html',
|
type: 'html',
|
||||||
size: encoded.byteLength,
|
size: encoded.byteLength,
|
||||||
|
|
@ -1036,11 +1030,13 @@ export async function syncDocumentsToServer(
|
||||||
onProgress(50, 'Uploading to server...');
|
onProgress(50, 'Uploading to server...');
|
||||||
}
|
}
|
||||||
|
|
||||||
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||||
|
|
||||||
for (const entry of uploads) {
|
for (let index = 0; index < uploads.length; index += 1) {
|
||||||
if (entry.oldId === entry.source.id) continue;
|
const entry = uploads[index];
|
||||||
await applyDocumentIdMapping(entry.oldId, entry.source.id);
|
const nextId = stored[index]?.id;
|
||||||
|
if (!nextId || entry.oldId === nextId) continue;
|
||||||
|
await applyDocumentIdMapping(entry.oldId, nextId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (onProgress) {
|
if (onProgress) {
|
||||||
|
|
@ -1064,11 +1060,9 @@ export async function syncSelectedDocumentsToServer(
|
||||||
const data = await getPdfDocument(doc.id);
|
const data = await getPdfDocument(doc.id);
|
||||||
if (data) {
|
if (data) {
|
||||||
const bytes = new Uint8Array(data.data);
|
const bytes = new Uint8Array(data.data);
|
||||||
const id = await sha256HexFromBytes(bytes);
|
|
||||||
uploads.push({
|
uploads.push({
|
||||||
oldId: data.id,
|
oldId: data.id,
|
||||||
source: {
|
source: {
|
||||||
id,
|
|
||||||
name: data.name,
|
name: data.name,
|
||||||
type: 'pdf',
|
type: 'pdf',
|
||||||
size: bytes.byteLength,
|
size: bytes.byteLength,
|
||||||
|
|
@ -1082,11 +1076,9 @@ export async function syncSelectedDocumentsToServer(
|
||||||
const data = await getEpubDocument(doc.id);
|
const data = await getEpubDocument(doc.id);
|
||||||
if (data) {
|
if (data) {
|
||||||
const bytes = new Uint8Array(data.data);
|
const bytes = new Uint8Array(data.data);
|
||||||
const id = await sha256HexFromBytes(bytes);
|
|
||||||
uploads.push({
|
uploads.push({
|
||||||
oldId: data.id,
|
oldId: data.id,
|
||||||
source: {
|
source: {
|
||||||
id,
|
|
||||||
name: data.name,
|
name: data.name,
|
||||||
type: 'epub',
|
type: 'epub',
|
||||||
size: bytes.byteLength,
|
size: bytes.byteLength,
|
||||||
|
|
@ -1100,11 +1092,9 @@ export async function syncSelectedDocumentsToServer(
|
||||||
const data = await getHtmlDocument(doc.id);
|
const data = await getHtmlDocument(doc.id);
|
||||||
if (data) {
|
if (data) {
|
||||||
const bytes = textEncoder.encode(data.data);
|
const bytes = textEncoder.encode(data.data);
|
||||||
const id = await sha256HexFromBytes(bytes);
|
|
||||||
uploads.push({
|
uploads.push({
|
||||||
oldId: data.id,
|
oldId: data.id,
|
||||||
source: {
|
source: {
|
||||||
id,
|
|
||||||
name: data.name,
|
name: data.name,
|
||||||
type: 'html',
|
type: 'html',
|
||||||
size: bytes.byteLength,
|
size: bytes.byteLength,
|
||||||
|
|
@ -1121,11 +1111,13 @@ export async function syncSelectedDocumentsToServer(
|
||||||
}
|
}
|
||||||
|
|
||||||
if (onProgress) onProgress(50, 'Uploading to server...');
|
if (onProgress) onProgress(50, 'Uploading to server...');
|
||||||
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
const stored = await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||||
|
|
||||||
for (const entry of uploads) {
|
for (let index = 0; index < uploads.length; index += 1) {
|
||||||
if (entry.oldId === entry.source.id) continue;
|
const entry = uploads[index];
|
||||||
await applyDocumentIdMapping(entry.oldId, entry.source.id);
|
const nextId = stored[index]?.id;
|
||||||
|
if (!nextId || entry.oldId === nextId) continue;
|
||||||
|
await applyDocumentIdMapping(entry.oldId, nextId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (onProgress) {
|
if (onProgress) {
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
|
|
@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState(
|
||||||
? parseProgress
|
? parseProgress
|
||||||
: null,
|
: null,
|
||||||
updatedAt: nowMs,
|
updatedAt: nowMs,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
|
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
|
||||||
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
|
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
|
||||||
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),
|
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { createHash, randomUUID } from 'node:crypto';
|
import { createHash, randomUUID } from 'node:crypto';
|
||||||
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
|
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
|
||||||
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
|
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
|
||||||
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core';
|
import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||||
import type {
|
import type {
|
||||||
|
|
@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string {
|
||||||
return [
|
return [
|
||||||
'pdf_layout',
|
'pdf_layout',
|
||||||
'v1',
|
'v1',
|
||||||
|
PDF_PARSER_VERSION,
|
||||||
input.documentId,
|
input.documentId,
|
||||||
input.namespace ?? '',
|
input.namespace ?? '',
|
||||||
input.documentObjectKey ?? '',
|
input.documentObjectKey ?? '',
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import {
|
import {
|
||||||
|
CopyObjectCommand,
|
||||||
DeleteObjectCommand,
|
DeleteObjectCommand,
|
||||||
DeleteObjectsCommand,
|
DeleteObjectsCommand,
|
||||||
GetObjectCommand,
|
GetObjectCommand,
|
||||||
|
|
@ -11,6 +12,9 @@ import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage
|
||||||
|
|
||||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||||
|
const TEMP_UPLOAD_TOKEN_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||||
|
const TEMP_UPLOAD_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/;
|
||||||
|
export const TEMP_DOCUMENT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
function sanitizeNamespace(namespace: string | null): string | null {
|
function sanitizeNamespace(namespace: string | null): string | null {
|
||||||
if (!namespace) return null;
|
if (!namespace) return null;
|
||||||
|
|
@ -69,6 +73,23 @@ export function isValidDocumentId(id: string): boolean {
|
||||||
return DOCUMENT_ID_REGEX.test(id);
|
return DOCUMENT_ID_REGEX.test(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isPreconditionFailed(error: unknown): boolean {
|
||||||
|
if (!error || typeof error !== 'object') return false;
|
||||||
|
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||||
|
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isValidTempUploadToken(token: string): boolean {
|
||||||
|
return TEMP_UPLOAD_TOKEN_REGEX.test(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitizeTempUploadUserId(userId: string): string {
|
||||||
|
if (!TEMP_UPLOAD_USER_ID_REGEX.test(userId)) {
|
||||||
|
throw new Error(`Invalid temp upload user id: ${userId}`);
|
||||||
|
}
|
||||||
|
return encodeURIComponent(userId);
|
||||||
|
}
|
||||||
|
|
||||||
export function documentKey(id: string, namespace: string | null): string {
|
export function documentKey(id: string, namespace: string | null): string {
|
||||||
if (!isValidDocumentId(id)) {
|
if (!isValidDocumentId(id)) {
|
||||||
throw new Error(`Invalid document id: ${id}`);
|
throw new Error(`Invalid document id: ${id}`);
|
||||||
|
|
@ -90,6 +111,27 @@ export function documentParsedKey(id: string, namespace: string | null): string
|
||||||
return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`;
|
return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const ns = sanitizeNamespace(namespace);
|
||||||
|
const nsSegment = ns ? `ns/${ns}/` : '';
|
||||||
|
return `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}users/${sanitizeTempUploadUserId(userId)}/`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function tempDocumentUploadKey(token: string, userId: string, namespace: string | null): string {
|
||||||
|
if (!isValidTempUploadToken(token)) {
|
||||||
|
throw new Error(`Invalid temp upload token: ${token}`);
|
||||||
|
}
|
||||||
|
return `${tempDocumentUploadPrefix(userId, namespace)}${token}.bin`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function tempDocumentUploadReceiptKey(token: string, userId: string, namespace: string | null): string {
|
||||||
|
if (!isValidTempUploadToken(token)) {
|
||||||
|
throw new Error(`Invalid temp upload token: ${token}`);
|
||||||
|
}
|
||||||
|
return `${tempDocumentUploadPrefix(userId, namespace)}${token}.receipt.json`;
|
||||||
|
}
|
||||||
|
|
||||||
function legacyDocumentParsedKey(id: string, namespace: string | null): string {
|
function legacyDocumentParsedKey(id: string, namespace: string | null): string {
|
||||||
if (!isValidDocumentId(id)) {
|
if (!isValidDocumentId(id)) {
|
||||||
throw new Error(`Invalid document id: ${id}`);
|
throw new Error(`Invalid document id: ${id}`);
|
||||||
|
|
@ -138,6 +180,40 @@ export async function presignPut(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function presignTempPut(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
contentType: string,
|
||||||
|
namespace: string | null,
|
||||||
|
options?: { contentLength?: number },
|
||||||
|
): Promise<{ url: string; headers: Record<string, string> }> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3Client();
|
||||||
|
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||||
|
const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream';
|
||||||
|
const contentLength =
|
||||||
|
typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0
|
||||||
|
? Math.floor(options.contentLength)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
const command = new PutObjectCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Key: key,
|
||||||
|
ContentType: normalizedType,
|
||||||
|
ServerSideEncryption: 'AES256',
|
||||||
|
...(contentLength !== undefined ? { ContentLength: contentLength } : {}),
|
||||||
|
});
|
||||||
|
const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 });
|
||||||
|
|
||||||
|
return {
|
||||||
|
url,
|
||||||
|
headers: {
|
||||||
|
'Content-Type': normalizedType,
|
||||||
|
'x-amz-server-side-encryption': 'AES256',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function headDocumentBlob(
|
export async function headDocumentBlob(
|
||||||
id: string,
|
id: string,
|
||||||
namespace: string | null,
|
namespace: string | null,
|
||||||
|
|
@ -153,6 +229,23 @@ export async function headDocumentBlob(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function headTempDocumentBlob(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
namespace: string | null,
|
||||||
|
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null; lastModified: number | null }> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||||
|
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||||
|
return {
|
||||||
|
contentLength: Number(res.ContentLength ?? 0),
|
||||||
|
contentType: res.ContentType ?? null,
|
||||||
|
eTag: res.ETag ?? null,
|
||||||
|
lastModified: res.LastModified?.getTime() ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function getDocumentRange(
|
export async function getDocumentRange(
|
||||||
id: string,
|
id: string,
|
||||||
start: number,
|
start: number,
|
||||||
|
|
@ -185,6 +278,23 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro
|
||||||
return bodyToBuffer(res.Body);
|
return bodyToBuffer(res.Body);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getTempDocumentBlob(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
namespace: string | null,
|
||||||
|
): Promise<Buffer> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||||
|
const res = await client.send(
|
||||||
|
new GetObjectCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Key: key,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return bodyToBuffer(res.Body);
|
||||||
|
}
|
||||||
|
|
||||||
export async function getDocumentBlobStream(id: string, namespace: string | null): Promise<DocumentBlobBody> {
|
export async function getDocumentBlobStream(id: string, namespace: string | null): Promise<DocumentBlobBody> {
|
||||||
const cfg = getS3Config();
|
const cfg = getS3Config();
|
||||||
const client = getS3ProxyClient();
|
const client = getS3ProxyClient();
|
||||||
|
|
@ -198,6 +308,29 @@ export async function getDocumentBlobStream(id: string, namespace: string | null
|
||||||
return res.Body as DocumentBlobBody;
|
return res.Body as DocumentBlobBody;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getTempDocumentFinalizeReceipt<T>(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
namespace: string | null,
|
||||||
|
): Promise<T | null> {
|
||||||
|
try {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
const key = tempDocumentUploadReceiptKey(token, userId, namespace);
|
||||||
|
const res = await client.send(
|
||||||
|
new GetObjectCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Key: key,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const body = await bodyToBuffer(res.Body);
|
||||||
|
return JSON.parse(body.toString('utf8')) as T;
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingBlobError(error)) return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
|
export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
|
||||||
const cfg = getS3Config();
|
const cfg = getS3Config();
|
||||||
const client = getS3ProxyClient();
|
const client = getS3ProxyClient();
|
||||||
|
|
@ -241,6 +374,26 @@ export async function putParsedDocumentBlob(id: string, body: Buffer, namespace:
|
||||||
return key;
|
return key;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function putTempDocumentFinalizeReceipt(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
namespace: string | null,
|
||||||
|
body: Buffer,
|
||||||
|
): Promise<void> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
const key = tempDocumentUploadReceiptKey(token, userId, namespace);
|
||||||
|
await client.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Key: key,
|
||||||
|
Body: body,
|
||||||
|
ContentType: 'application/json',
|
||||||
|
ServerSideEncryption: 'AES256',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function presignGet(
|
export async function presignGet(
|
||||||
id: string,
|
id: string,
|
||||||
namespace: string | null,
|
namespace: string | null,
|
||||||
|
|
@ -264,6 +417,7 @@ export async function putDocumentBlob(
|
||||||
body: Buffer,
|
body: Buffer,
|
||||||
contentType: string,
|
contentType: string,
|
||||||
namespace: string | null,
|
namespace: string | null,
|
||||||
|
options?: { ifNoneMatch?: boolean },
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const cfg = getS3Config();
|
const cfg = getS3Config();
|
||||||
const client = getS3ProxyClient();
|
const client = getS3ProxyClient();
|
||||||
|
|
@ -275,6 +429,51 @@ export async function putDocumentBlob(
|
||||||
Body: body,
|
Body: body,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
ServerSideEncryption: 'AES256',
|
ServerSideEncryption: 'AES256',
|
||||||
|
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function putTempDocumentBlob(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
body: Buffer,
|
||||||
|
contentType: string,
|
||||||
|
namespace: string | null,
|
||||||
|
): Promise<void> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
const key = tempDocumentUploadKey(token, userId, namespace);
|
||||||
|
await client.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Key: key,
|
||||||
|
Body: body,
|
||||||
|
ContentType: contentType,
|
||||||
|
ServerSideEncryption: 'AES256',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function copyTempDocumentBlobToDocument(
|
||||||
|
token: string,
|
||||||
|
userId: string,
|
||||||
|
documentId: string,
|
||||||
|
namespace: string | null,
|
||||||
|
contentType: string,
|
||||||
|
options?: { ifNoneMatch?: boolean },
|
||||||
|
): Promise<void> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
await client.send(
|
||||||
|
new CopyObjectCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Key: documentKey(documentId, namespace),
|
||||||
|
CopySource: `${cfg.bucket}/${tempDocumentUploadKey(token, userId, namespace)}`,
|
||||||
|
ContentType: contentType,
|
||||||
|
MetadataDirective: 'REPLACE',
|
||||||
|
ServerSideEncryption: 'AES256',
|
||||||
|
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -292,6 +491,18 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
|
||||||
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
|
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadKey(token, userId, namespace) }));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteTempDocumentFinalizeReceipt(token: string, userId: string, namespace: string | null): Promise<void> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadReceiptKey(token, userId, namespace) }));
|
||||||
|
}
|
||||||
|
|
||||||
export function isMissingBlobError(error: unknown): boolean {
|
export function isMissingBlobError(error: unknown): boolean {
|
||||||
if (!error || typeof error !== 'object') return false;
|
if (!error || typeof error !== 'object') return false;
|
||||||
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
|
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
|
||||||
|
|
@ -339,3 +550,53 @@ export async function deleteDocumentPrefix(prefix: string): Promise<number> {
|
||||||
|
|
||||||
return deleted;
|
return deleted;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function deleteExpiredTempDocumentUploads(
|
||||||
|
userId: string,
|
||||||
|
namespace: string | null,
|
||||||
|
olderThanMs: number,
|
||||||
|
): Promise<number> {
|
||||||
|
const cfg = getS3Config();
|
||||||
|
const client = getS3ProxyClient();
|
||||||
|
const prefix = tempDocumentUploadPrefix(userId, namespace);
|
||||||
|
let continuationToken: string | undefined;
|
||||||
|
const keys: string[] = [];
|
||||||
|
|
||||||
|
do {
|
||||||
|
const listRes = await client.send(
|
||||||
|
new ListObjectsV2Command({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Prefix: prefix,
|
||||||
|
ContinuationToken: continuationToken,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const item of listRes.Contents ?? []) {
|
||||||
|
const key = item.Key;
|
||||||
|
const lastModified = item.LastModified?.getTime() ?? 0;
|
||||||
|
if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue;
|
||||||
|
keys.push(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||||
|
} while (continuationToken);
|
||||||
|
|
||||||
|
if (keys.length === 0) return 0;
|
||||||
|
|
||||||
|
let deleted = 0;
|
||||||
|
for (let i = 0; i < keys.length; i += 1000) {
|
||||||
|
const batch = keys.slice(i, i + 1000);
|
||||||
|
const deleteRes = await client.send(
|
||||||
|
new DeleteObjectsCommand({
|
||||||
|
Bucket: cfg.bucket,
|
||||||
|
Delete: {
|
||||||
|
Objects: batch.map((Key) => ({ Key })),
|
||||||
|
Quiet: true,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
deleted += deleteRes.Deleted?.length ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
|
||||||
94
src/lib/server/documents/docx-convert.ts
Normal file
94
src/lib/server/documents/docx-convert.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
import { spawn } from 'child_process';
|
||||||
|
import { existsSync } from 'fs';
|
||||||
|
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises';
|
||||||
|
import path from 'path';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
import { pathToFileURL } from 'url';
|
||||||
|
|
||||||
|
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||||
|
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||||
|
const DEFAULT_DOCX_CONVERSION_TIMEOUT_MS = 60_000;
|
||||||
|
|
||||||
|
async function ensureTempDir(): Promise<void> {
|
||||||
|
if (!existsSync(DOCSTORE_DIR)) {
|
||||||
|
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||||
|
}
|
||||||
|
if (!existsSync(TEMP_DIR)) {
|
||||||
|
await mkdir(TEMP_DIR, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const args: string[] = [];
|
||||||
|
if (profileDir) {
|
||||||
|
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
|
||||||
|
}
|
||||||
|
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
|
||||||
|
const proc = spawn('soffice', args);
|
||||||
|
let settled = false;
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
proc.kill();
|
||||||
|
reject(new Error(`LibreOffice conversion timed out after ${DEFAULT_DOCX_CONVERSION_TIMEOUT_MS}ms`));
|
||||||
|
}, DEFAULT_DOCX_CONVERSION_TIMEOUT_MS);
|
||||||
|
|
||||||
|
proc.on('error', (error) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
reject(error);
|
||||||
|
});
|
||||||
|
proc.on('close', (code) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
if (code === 0) resolve();
|
||||||
|
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForPdfReady(dir: string, timeoutMs = 20_000, intervalMs = 100): Promise<string> {
|
||||||
|
const end = Date.now() + timeoutMs;
|
||||||
|
while (Date.now() < end) {
|
||||||
|
const files = await readdir(dir);
|
||||||
|
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
|
||||||
|
if (pdf) {
|
||||||
|
const pdfPath = path.join(dir, pdf);
|
||||||
|
try {
|
||||||
|
const first = await stat(pdfPath);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||||
|
const second = await stat(pdfPath);
|
||||||
|
if (second.size > 0 && second.size === first.size) {
|
||||||
|
return pdfPath;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore transient filesystem races while LibreOffice is still writing.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||||
|
}
|
||||||
|
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function convertDocxBufferToPdfBuffer(docxBytes: Buffer): Promise<Buffer> {
|
||||||
|
await ensureTempDir();
|
||||||
|
|
||||||
|
const tempId = randomUUID();
|
||||||
|
const jobDir = path.join(TEMP_DIR, tempId);
|
||||||
|
await mkdir(jobDir, { recursive: true });
|
||||||
|
const profileDir = path.join(jobDir, 'lo-profile');
|
||||||
|
await mkdir(profileDir, { recursive: true });
|
||||||
|
const inputPath = path.join(jobDir, 'input.docx');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeFile(inputPath, docxBytes);
|
||||||
|
await convertDocxToPdf(inputPath, jobDir, profileDir);
|
||||||
|
const pdfPath = await waitForPdfReady(jobDir);
|
||||||
|
return await readFile(pdfPath);
|
||||||
|
} finally {
|
||||||
|
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,7 +1,5 @@
|
||||||
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
|
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||||
import { documentKey } from '@/lib/server/documents/blobstore';
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
|
||||||
import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter';
|
|
||||||
import { normalizeParseStatus, type DocumentParseState } from '@/lib/server/documents/parse-state';
|
import { normalizeParseStatus, type DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||||
|
|
||||||
|
|
@ -20,13 +18,20 @@ export async function backfillPendingPdfParseOperation(input: {
|
||||||
if (parseStatus === 'ready' || parseStatus === 'failed') return null;
|
if (parseStatus === 'ready' || parseStatus === 'failed') return null;
|
||||||
if (normalizeOpId(input.state.opId)) return null;
|
if (normalizeOpId(input.state.opId)) return null;
|
||||||
|
|
||||||
const created = await createOrReusePdfWorkerOperation({
|
const startedParse = await startPdfParseOperation({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
userId: input.userId,
|
||||||
namespace: input.namespace,
|
namespace: input.namespace,
|
||||||
documentObjectKey: documentKey(input.documentId, input.namespace),
|
|
||||||
});
|
});
|
||||||
|
if (startedParse.parseState.status === 'pending' || startedParse.parseState.status === 'running') {
|
||||||
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
enqueueParsePdfJob({
|
||||||
await recordJobEvent(input.userId, 'pdf_layout', created.opId, rateConfig);
|
documentId: input.documentId,
|
||||||
return created;
|
userId: input.userId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
initialOpId: startedParse.workerState.opId,
|
||||||
|
initialJobId: startedParse.workerState.jobId,
|
||||||
|
initialStatus: startedParse.parseState.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return startedParse.workerState;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: {
|
||||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||||
...(input.state.opId ? { opId: input.state.opId } : {}),
|
...(input.state.opId ? { opId: input.state.opId } : {}),
|
||||||
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
|
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
|
||||||
|
...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await db
|
await db
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
|
|
||||||
export interface DocumentParseState {
|
export interface DocumentParseState {
|
||||||
status: PdfParseStatus;
|
status: PdfParseStatus;
|
||||||
|
|
@ -7,6 +8,7 @@ export interface DocumentParseState {
|
||||||
error?: string | null;
|
error?: string | null;
|
||||||
opId?: string;
|
opId?: string;
|
||||||
jobId?: string;
|
jobId?: string;
|
||||||
|
parserVersion?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
|
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
|
||||||
|
|
@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars
|
||||||
return 'pending';
|
return 'pending';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean {
|
||||||
|
return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeDocumentParseStateForCurrentParserVersion(
|
||||||
|
state: DocumentParseState,
|
||||||
|
nowMs = Date.now(),
|
||||||
|
): DocumentParseState {
|
||||||
|
const status = normalizeParseStatus(state.status);
|
||||||
|
if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) {
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: nowMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string {
|
||||||
|
const parserVersion = parsed?.parserVersion?.trim();
|
||||||
|
return parserVersion || PDF_PARSER_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
||||||
if (!progress || typeof progress !== 'object') return null;
|
if (!progress || typeof progress !== 'object') return null;
|
||||||
const rec = progress as Record<string, unknown>;
|
const rec = progress as Record<string, unknown>;
|
||||||
|
|
@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
||||||
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
||||||
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
||||||
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
||||||
|
const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined;
|
||||||
return {
|
return {
|
||||||
status,
|
status,
|
||||||
progress,
|
progress,
|
||||||
|
|
@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
||||||
...(error ? { error } : {}),
|
...(error ? { error } : {}),
|
||||||
...(opId ? { opId } : {}),
|
...(opId ? { opId } : {}),
|
||||||
...(jobId ? { jobId } : {}),
|
...(jobId ? { jobId } : {}),
|
||||||
|
...(parserVersion ? { parserVersion } : {}),
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return { status: 'pending', progress: null };
|
return { status: 'pending', progress: null };
|
||||||
|
|
@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string {
|
||||||
...(state.error ? { error: state.error } : {}),
|
...(state.error ? { error: state.error } : {}),
|
||||||
...(state.opId ? { opId: state.opId } : {}),
|
...(state.opId ? { opId: state.opId } : {}),
|
||||||
...(state.jobId ? { jobId: state.jobId } : {}),
|
...(state.jobId ? { jobId: state.jobId } : {}),
|
||||||
|
...(state.parserVersion ? { parserVersion: state.parserVersion } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
import { eq } from 'drizzle-orm';
|
||||||
|
import { db } from '@/db';
|
||||||
|
import { documents } from '@/db/schema';
|
||||||
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
|
parseDocumentParseState,
|
||||||
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
|
||||||
|
type ReadyParsedPdfResult = {
|
||||||
|
parsedJsonKey: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function findReusableParsedPdfResult(
|
||||||
|
documentId: string,
|
||||||
|
): Promise<ReadyParsedPdfResult | null> {
|
||||||
|
const rows = await db
|
||||||
|
.select({
|
||||||
|
parseState: documents.parseState,
|
||||||
|
parsedJsonKey: documents.parsedJsonKey,
|
||||||
|
})
|
||||||
|
.from(documents)
|
||||||
|
.where(eq(documents.id, documentId));
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const parsedJsonKey = row.parsedJsonKey?.trim();
|
||||||
|
if (!parsedJsonKey) continue;
|
||||||
|
const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
|
if (parseState.status !== 'ready') continue;
|
||||||
|
return { parsedJsonKey };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
30
src/lib/server/documents/pdf-parse-operation.ts
Normal file
30
src/lib/server/documents/pdf-parse-operation.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create';
|
||||||
|
import { documentParseStateFromWorkerState } from '@/lib/server/compute/worker-parse-state';
|
||||||
|
import { documentKey } from '@/lib/server/documents/blobstore';
|
||||||
|
import { getPdfLayoutRateConfig, recordJobEvent } from '@/lib/server/rate-limit/job-rate-limiter';
|
||||||
|
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||||
|
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
|
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||||
|
|
||||||
|
export async function startPdfParseOperation(input: {
|
||||||
|
documentId: string;
|
||||||
|
userId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
forceToken?: string;
|
||||||
|
}): Promise<{
|
||||||
|
workerState: WorkerOperationState<PdfLayoutJobResult>;
|
||||||
|
parseState: DocumentParseState;
|
||||||
|
}> {
|
||||||
|
const workerState = await createOrReusePdfWorkerOperation({
|
||||||
|
documentId: input.documentId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
documentObjectKey: documentKey(input.documentId, input.namespace),
|
||||||
|
...(input.forceToken ? { forceToken: input.forceToken } : {}),
|
||||||
|
});
|
||||||
|
const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig());
|
||||||
|
await recordJobEvent(input.userId, 'pdf_layout', workerState.opId, rateConfig);
|
||||||
|
return {
|
||||||
|
workerState,
|
||||||
|
parseState: documentParseStateFromWorkerState(workerState),
|
||||||
|
};
|
||||||
|
}
|
||||||
106
src/lib/server/documents/register-upload.ts
Normal file
106
src/lib/server/documents/register-upload.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
||||||
|
import { db } from '@/db';
|
||||||
|
import { documents } from '@/db/schema';
|
||||||
|
import {
|
||||||
|
enqueueDocumentPreview,
|
||||||
|
} from '@/lib/server/documents/previews';
|
||||||
|
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||||
|
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
|
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||||
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||||
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
|
|
||||||
|
type RegisterUploadedDocumentInput = {
|
||||||
|
documentId: string;
|
||||||
|
userId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
name: string;
|
||||||
|
type: DocumentType;
|
||||||
|
size: number;
|
||||||
|
lastModified: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> {
|
||||||
|
const reusableParsedPdf = input.type === 'pdf'
|
||||||
|
? await findReusableParsedPdfResult(input.documentId)
|
||||||
|
: null;
|
||||||
|
const startedParse = input.type === 'pdf' && !reusableParsedPdf
|
||||||
|
? await startPdfParseOperation({
|
||||||
|
documentId: input.documentId,
|
||||||
|
userId: input.userId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
})
|
||||||
|
: null;
|
||||||
|
const parsedJsonKey = reusableParsedPdf?.parsedJsonKey ?? null;
|
||||||
|
const parseState = input.type === 'pdf'
|
||||||
|
? stringifyDocumentParseState(
|
||||||
|
reusableParsedPdf
|
||||||
|
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||||
|
: (startedParse?.parseState ?? { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }),
|
||||||
|
)
|
||||||
|
: null;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.insert(documents)
|
||||||
|
.values({
|
||||||
|
id: input.documentId,
|
||||||
|
userId: input.userId,
|
||||||
|
name: input.name,
|
||||||
|
type: input.type,
|
||||||
|
size: input.size,
|
||||||
|
lastModified: input.lastModified,
|
||||||
|
filePath: input.documentId,
|
||||||
|
parseState,
|
||||||
|
parsedJsonKey,
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: [documents.id, documents.userId],
|
||||||
|
set: {
|
||||||
|
name: input.name,
|
||||||
|
type: input.type,
|
||||||
|
size: input.size,
|
||||||
|
lastModified: input.lastModified,
|
||||||
|
filePath: input.documentId,
|
||||||
|
parseState,
|
||||||
|
parsedJsonKey,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await enqueueDocumentPreview(
|
||||||
|
{
|
||||||
|
id: input.documentId,
|
||||||
|
type: input.type,
|
||||||
|
lastModified: input.lastModified,
|
||||||
|
},
|
||||||
|
input.namespace,
|
||||||
|
).catch((error) => {
|
||||||
|
serverLogger.warn({
|
||||||
|
event: 'documents.preview.enqueue.failed',
|
||||||
|
degraded: true,
|
||||||
|
fallbackPath: 'skip_preview_enqueue',
|
||||||
|
documentId: input.documentId,
|
||||||
|
error: errorToLog(error),
|
||||||
|
}, 'Failed to enqueue document preview');
|
||||||
|
});
|
||||||
|
|
||||||
|
if (startedParse) {
|
||||||
|
enqueueParsePdfJob({
|
||||||
|
documentId: input.documentId,
|
||||||
|
userId: input.userId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
initialOpId: startedParse.workerState.opId,
|
||||||
|
initialJobId: startedParse.workerState.jobId,
|
||||||
|
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: input.documentId,
|
||||||
|
name: input.name,
|
||||||
|
type: input.type,
|
||||||
|
size: input.size,
|
||||||
|
lastModified: input.lastModified,
|
||||||
|
scope: 'user',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import { and, eq, inArray, isNull } from 'drizzle-orm';
|
import { and, eq, inArray, isNull } from 'drizzle-orm';
|
||||||
import { db } from '@/db';
|
import { db } from '@/db';
|
||||||
import { documents } from '@/db/schema';
|
import { documents } from '@/db/schema';
|
||||||
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||||
|
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||||
import { serverLogger } from '@/lib/server/logger';
|
import { serverLogger } from '@/lib/server/logger';
|
||||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
|
resolveParsedPdfParserVersion,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
type DocumentParseState,
|
type DocumentParseState,
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
|
@ -13,15 +16,20 @@ import { getCompute } from '@/lib/server/compute';
|
||||||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||||
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
|
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
|
||||||
import type {
|
import type {
|
||||||
|
ParsedPdfDocument,
|
||||||
PdfLayoutJobBase,
|
PdfLayoutJobBase,
|
||||||
PdfLayoutJobResult,
|
PdfLayoutJobResult,
|
||||||
PdfLayoutProgress,
|
PdfLayoutProgress,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
} from '@openreader/compute-core/api-contracts';
|
} from '@openreader/compute-core/api-contracts';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
|
||||||
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
|
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
|
||||||
userId: string;
|
userId: string;
|
||||||
forceToken?: string;
|
forceToken?: string;
|
||||||
|
initialOpId?: string;
|
||||||
|
initialJobId?: string;
|
||||||
|
initialStatus?: 'pending' | 'running';
|
||||||
};
|
};
|
||||||
|
|
||||||
const running = new Set<string>();
|
const running = new Set<string>();
|
||||||
|
|
@ -41,7 +49,12 @@ function keyFor(input: UserPdfLayoutJobRequest): string {
|
||||||
if (forceToken) {
|
if (forceToken) {
|
||||||
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
|
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
|
||||||
}
|
}
|
||||||
return `shared:${input.documentId}:${input.namespace || ''}`;
|
return `shared:${input.documentId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOpId(value: string | undefined): string | undefined {
|
||||||
|
const normalized = value?.trim();
|
||||||
|
return normalized ? normalized : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sleep(ms: number): Promise<void> {
|
function sleep(ms: number): Promise<void> {
|
||||||
|
|
@ -84,7 +97,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise<ParseRow[
|
||||||
|
|
||||||
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
|
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
|
||||||
if (!row.parsedJsonKey) return false;
|
if (!row.parsedJsonKey) return false;
|
||||||
return parseDocumentParseState(row.parseState).status === 'ready';
|
return normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)).status === 'ready';
|
||||||
}
|
}
|
||||||
|
|
||||||
function userIdsFromRows(rows: ParseRow[]): string[] {
|
function userIdsFromRows(rows: ParseRow[]): string[] {
|
||||||
|
|
@ -95,6 +108,20 @@ function parseStateMatchCondition(expected: string | null) {
|
||||||
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
|
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadParsedDocumentForResult(
|
||||||
|
parsed: ParsedPdfDocument | undefined,
|
||||||
|
parsedJsonKey: string,
|
||||||
|
): Promise<ParsedPdfDocument | null> {
|
||||||
|
if (parsed) return parsed;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||||
|
return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function updateParseStateForUsers(input: {
|
async function updateParseStateForUsers(input: {
|
||||||
documentId: string;
|
documentId: string;
|
||||||
userIds: string[];
|
userIds: string[];
|
||||||
|
|
@ -125,6 +152,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
|
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
|
||||||
|
|
||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
|
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||||
|
if (reusableParsed) {
|
||||||
|
const readyState: DocumentParseState = {
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
};
|
||||||
|
await updateParseStateForUsers({
|
||||||
|
documentId: input.documentId,
|
||||||
|
userIds: [input.userId],
|
||||||
|
parseState: stringifyDocumentParseState(readyState),
|
||||||
|
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const rows = await loadScopedRows(input);
|
const rows = await loadScopedRows(input);
|
||||||
const ready = rows.find(isReadyRow);
|
const ready = rows.find(isReadyRow);
|
||||||
if (ready) {
|
if (ready) {
|
||||||
|
|
@ -132,6 +176,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
};
|
};
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
|
@ -142,7 +187,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const statuses = rows.map((row) => parseDocumentParseState(row.parseState));
|
const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)));
|
||||||
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
|
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
|
||||||
const failedState = statuses.find((state) => state.status === 'failed');
|
const failedState = statuses.find((state) => state.status === 'failed');
|
||||||
if (failedState && !hasInFlight) {
|
if (failedState && !hasInFlight) {
|
||||||
|
|
@ -151,6 +196,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
...(failedState.error ? { error: failedState.error } : {}),
|
...(failedState.error ? { error: failedState.error } : {}),
|
||||||
|
...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}),
|
||||||
};
|
};
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
|
@ -174,6 +220,8 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
}
|
}
|
||||||
running.add(key);
|
running.add(key);
|
||||||
|
|
||||||
|
let activeOpId = normalizeOpId(input.initialOpId);
|
||||||
|
let activeJobId = normalizeOpId(input.initialJobId);
|
||||||
try {
|
try {
|
||||||
const scopedRows = await loadScopedRows(input);
|
const scopedRows = await loadScopedRows(input);
|
||||||
const scopedUserIds = userIdsFromRows(scopedRows);
|
const scopedUserIds = userIdsFromRows(scopedRows);
|
||||||
|
|
@ -181,12 +229,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
// Non-force jobs can short-circuit by reusing existing ready output from
|
// Non-force jobs can short-circuit by reusing existing ready output from
|
||||||
// any row in the same document scope.
|
// any row in the same document scope.
|
||||||
if (!input.forceToken?.trim()) {
|
if (!input.forceToken?.trim()) {
|
||||||
|
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||||
|
if (reusableParsed) {
|
||||||
|
const readyState: DocumentParseState = {
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
};
|
||||||
|
await updateParseStateForUsers({
|
||||||
|
documentId: input.documentId,
|
||||||
|
userIds: [input.userId],
|
||||||
|
parseState: stringifyDocumentParseState(readyState),
|
||||||
|
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const ready = scopedRows.find(isReadyRow);
|
const ready = scopedRows.find(isReadyRow);
|
||||||
if (ready) {
|
if (ready) {
|
||||||
const readyState: DocumentParseState = {
|
const readyState: DocumentParseState = {
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
};
|
};
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
|
@ -201,17 +267,21 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
const coordinator = [...scopedRows].sort((a, b) => a.userId.localeCompare(b.userId))[0];
|
const coordinator = [...scopedRows].sort((a, b) => a.userId.localeCompare(b.userId))[0];
|
||||||
if (!coordinator) return;
|
if (!coordinator) return;
|
||||||
|
|
||||||
const runningStateData: DocumentParseState = {
|
const initialInflightStatus = input.initialStatus === 'running' ? 'running' : 'pending';
|
||||||
status: 'running',
|
const inflightStateData: DocumentParseState = {
|
||||||
|
status: initialInflightStatus,
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
const runningState = stringifyDocumentParseState(runningStateData);
|
const inflightState = stringifyDocumentParseState(inflightStateData);
|
||||||
|
|
||||||
const claimRows = (await db
|
const claimRows = (await db
|
||||||
.update(documents)
|
.update(documents)
|
||||||
.set({
|
.set({
|
||||||
parseState: runningState,
|
parseState: inflightState,
|
||||||
})
|
})
|
||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
|
|
@ -234,12 +304,10 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
userIds: cohortUserIds,
|
userIds: cohortUserIds,
|
||||||
parseState: runningState,
|
parseState: inflightState,
|
||||||
});
|
});
|
||||||
|
|
||||||
const compute = await getCompute();
|
const compute = await getCompute();
|
||||||
let activeOpId: string | undefined;
|
|
||||||
let activeJobId: string | undefined;
|
|
||||||
let lastProgressWriteAt = 0;
|
let lastProgressWriteAt = 0;
|
||||||
let lastSnapshotWriteAt = 0;
|
let lastSnapshotWriteAt = 0;
|
||||||
let lastSnapshotStatus: 'pending' | 'running' | null = null;
|
let lastSnapshotStatus: 'pending' | 'running' | null = null;
|
||||||
|
|
@ -279,6 +347,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
status: mappedStatus,
|
status: mappedStatus,
|
||||||
progress: snapshot.progress ?? null,
|
progress: snapshot.progress ?? null,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
...(activeOpId ? { opId: activeOpId } : {}),
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
|
|
@ -305,6 +374,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
phase: progress.phase,
|
phase: progress.phase,
|
||||||
},
|
},
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
...(activeOpId ? { opId: activeOpId } : {}),
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
|
|
@ -327,12 +397,17 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
|
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey);
|
||||||
|
|
||||||
const finalScopedRows = await loadScopedRows(input);
|
const finalScopedRows = await loadScopedRows(input);
|
||||||
const finalUserIds = userIdsFromRows(finalScopedRows);
|
const finalUserIds = userIdsFromRows(finalScopedRows);
|
||||||
const readyState: DocumentParseState = {
|
const readyState: DocumentParseState = {
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
|
||||||
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
|
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
|
|
@ -385,6 +460,9 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
error: message,
|
error: message,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
|
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ export interface BaseDocument {
|
||||||
parsedJsonKey?: string | null;
|
parsedJsonKey?: string | null;
|
||||||
scope?: 'user';
|
scope?: 'user';
|
||||||
folderId?: string;
|
folderId?: string;
|
||||||
isConverting?: boolean;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PDFDocument extends BaseDocument {
|
export interface PDFDocument extends BaseDocument {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,4 @@
|
||||||
import { Page, expect, type TestInfo, type Locator } from '@playwright/test';
|
import { Page, expect, type TestInfo, type Locator } from '@playwright/test';
|
||||||
import fs from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
|
|
||||||
const DIR = './tests/files/';
|
const DIR = './tests/files/';
|
||||||
|
|
@ -10,14 +8,6 @@ function escapeRegExp(input: string) {
|
||||||
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||||
}
|
}
|
||||||
|
|
||||||
function fixturePath(fileName: string) {
|
|
||||||
return path.join(__dirname, 'files', fileName);
|
|
||||||
}
|
|
||||||
|
|
||||||
function sha256HexOfFile(filePath: string) {
|
|
||||||
return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
||||||
await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) });
|
await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) });
|
||||||
const loader = page.getByTestId('pdf-status-loader').first();
|
const loader = page.getByTestId('pdf-status-loader').first();
|
||||||
|
|
@ -40,7 +30,7 @@ async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Upload a sample epub or pdf
|
* Upload a sample document fixture
|
||||||
*/
|
*/
|
||||||
export async function uploadFile(page: Page, filePath: string) {
|
export async function uploadFile(page: Page, filePath: string) {
|
||||||
const input = page.locator('input[type=file]').first();
|
const input = page.locator('input[type=file]').first();
|
||||||
|
|
@ -50,7 +40,7 @@ export async function uploadFile(page: Page, filePath: string) {
|
||||||
await input.setInputFiles(`${DIR}${filePath}`);
|
await input.setInputFiles(`${DIR}${filePath}`);
|
||||||
|
|
||||||
// Wait for the uploader to finish processing. The input is disabled while
|
// Wait for the uploader to finish processing. The input is disabled while
|
||||||
// uploading/converting via react-dropzone's `disabled` prop.
|
// uploading via react-dropzone's `disabled` prop.
|
||||||
// Tolerate extremely fast operations where the disabled state may be missed.
|
// Tolerate extremely fast operations where the disabled state may be missed.
|
||||||
try {
|
try {
|
||||||
await expect(input).toBeDisabled({ timeout: 2000 });
|
await expect(input).toBeDisabled({ timeout: 2000 });
|
||||||
|
|
@ -69,14 +59,8 @@ export async function uploadAndDisplay(page: Page, fileName: string) {
|
||||||
const lower = fileName.toLowerCase();
|
const lower = fileName.toLowerCase();
|
||||||
|
|
||||||
if (lower.endsWith('.docx')) {
|
if (lower.endsWith('.docx')) {
|
||||||
// Best-effort: conversion can complete before we observe this UI state.
|
const convertedName = `${fileName.replace(/\.[^.]+$/, '')}.pdf`;
|
||||||
try {
|
const targetLink = page.getByRole('link', { name: new RegExp(escapeRegExp(convertedName), 'i') }).first();
|
||||||
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
const expectedId = sha256HexOfFile(fixturePath(fileName));
|
|
||||||
const targetLink = page.locator(`a[href$="/pdf/${expectedId}"]`).first();
|
|
||||||
await expect(targetLink).toBeVisible({ timeout: 15000 });
|
await expect(targetLink).toBeVisible({ timeout: 15000 });
|
||||||
await dismissOnboardingModals(page);
|
await dismissOnboardingModals(page);
|
||||||
await targetLink.click();
|
await targetLink.click();
|
||||||
|
|
@ -188,10 +172,11 @@ async function dismissOnboardingModals(page: Page): Promise<void> {
|
||||||
* Wait for the play button to be clickable and click it
|
* Wait for the play button to be clickable and click it
|
||||||
*/
|
*/
|
||||||
export async function waitAndClickPlay(page: Page) {
|
export async function waitAndClickPlay(page: Page) {
|
||||||
// Wait for play button selector without disabled attribute
|
const playButton = page.getByRole('button', { name: 'Play' });
|
||||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();
|
await expect(playButton).toBeVisible();
|
||||||
|
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||||
// Play the TTS by clicking the button
|
// Play the TTS by clicking the button
|
||||||
await page.getByRole('button', { name: 'Play' }).click();
|
await playButton.click();
|
||||||
// Use resilient processing transition helper (tolerates fast completion)
|
// Use resilient processing transition helper (tolerates fast completion)
|
||||||
await expectProcessingTransition(page);
|
await expectProcessingTransition(page);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => {
|
||||||
await page.waitForTimeout(750);
|
await page.waitForTimeout(750);
|
||||||
|
|
||||||
// After resize, playback should have paused (Play button visible)
|
// After resize, playback should have paused (Play button visible)
|
||||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });
|
const playButton = page.getByRole('button', { name: 'Play' });
|
||||||
|
await expect(playButton).toBeVisible({ timeout: 15000 });
|
||||||
|
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||||
|
|
||||||
// Resume playback and ensure processing -> playing
|
// Resume playback and ensure processing -> playing
|
||||||
await page.getByRole('button', { name: 'Play' }).click();
|
await playButton.click();
|
||||||
await expectProcessingTransition(page);
|
await expectProcessingTransition(page);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
150
tests/unit/blob-upload-finalize-docx.vitest.spec.ts
Normal file
150
tests/unit/blob-upload-finalize-docx.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
|
||||||
|
const hoisted = vi.hoisted(() => ({
|
||||||
|
requireAuthContext: vi.fn(),
|
||||||
|
registerUploadedDocument: vi.fn(),
|
||||||
|
convertDocxBufferToPdfBuffer: vi.fn(),
|
||||||
|
headTempDocumentBlob: vi.fn(),
|
||||||
|
getTempDocumentBlob: vi.fn(),
|
||||||
|
getTempDocumentFinalizeReceipt: vi.fn(),
|
||||||
|
putTempDocumentFinalizeReceipt: vi.fn(),
|
||||||
|
deleteTempDocumentUpload: vi.fn(),
|
||||||
|
headDocumentBlob: vi.fn(),
|
||||||
|
copyTempDocumentBlobToDocument: vi.fn(),
|
||||||
|
putDocumentBlob: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/auth/auth', () => ({
|
||||||
|
requireAuthContext: hoisted.requireAuthContext,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/register-upload', () => ({
|
||||||
|
registerUploadedDocument: hoisted.registerUploadedDocument,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/docx-convert', () => ({
|
||||||
|
convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||||
|
TEMP_DOCUMENT_UPLOAD_TTL_MS: 24 * 60 * 60 * 1000,
|
||||||
|
copyTempDocumentBlobToDocument: hoisted.copyTempDocumentBlobToDocument,
|
||||||
|
deleteTempDocumentUpload: hoisted.deleteTempDocumentUpload,
|
||||||
|
getTempDocumentBlob: hoisted.getTempDocumentBlob,
|
||||||
|
getTempDocumentFinalizeReceipt: hoisted.getTempDocumentFinalizeReceipt,
|
||||||
|
headDocumentBlob: hoisted.headDocumentBlob,
|
||||||
|
headTempDocumentBlob: hoisted.headTempDocumentBlob,
|
||||||
|
isMissingBlobError: vi.fn((error: unknown) => {
|
||||||
|
const maybe = error as { code?: string } | undefined;
|
||||||
|
return maybe?.code === 'NoSuchKey';
|
||||||
|
}),
|
||||||
|
isPreconditionFailed: vi.fn(() => false),
|
||||||
|
isValidTempUploadToken: vi.fn(() => true),
|
||||||
|
putDocumentBlob: hoisted.putDocumentBlob,
|
||||||
|
putTempDocumentFinalizeReceipt: hoisted.putTempDocumentFinalizeReceipt,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/testing/test-namespace', () => ({
|
||||||
|
getOpenReaderTestNamespace: vi.fn(() => null),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/storage/s3', () => ({
|
||||||
|
isS3Configured: vi.fn(() => true),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/logger', () => ({
|
||||||
|
errorToLog: vi.fn((error: unknown) => error),
|
||||||
|
serverLogger: {
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('POST /api/documents/blob/upload/finalize DOCX flow', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
hoisted.requireAuthContext.mockReset();
|
||||||
|
hoisted.registerUploadedDocument.mockReset();
|
||||||
|
hoisted.convertDocxBufferToPdfBuffer.mockReset();
|
||||||
|
hoisted.headTempDocumentBlob.mockReset();
|
||||||
|
hoisted.getTempDocumentBlob.mockReset();
|
||||||
|
hoisted.getTempDocumentFinalizeReceipt.mockReset();
|
||||||
|
hoisted.putTempDocumentFinalizeReceipt.mockReset();
|
||||||
|
hoisted.deleteTempDocumentUpload.mockReset();
|
||||||
|
hoisted.headDocumentBlob.mockReset();
|
||||||
|
hoisted.copyTempDocumentBlobToDocument.mockReset();
|
||||||
|
hoisted.putDocumentBlob.mockReset();
|
||||||
|
|
||||||
|
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||||
|
hoisted.getTempDocumentFinalizeReceipt.mockResolvedValue(null);
|
||||||
|
hoisted.headTempDocumentBlob.mockResolvedValue({
|
||||||
|
contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||||
|
contentLength: 12,
|
||||||
|
lastModified: Date.now(),
|
||||||
|
});
|
||||||
|
hoisted.getTempDocumentBlob.mockResolvedValue(Buffer.from('docx-bytes'));
|
||||||
|
hoisted.convertDocxBufferToPdfBuffer.mockResolvedValue(Buffer.from('pdf-bytes'));
|
||||||
|
hoisted.headDocumentBlob
|
||||||
|
.mockRejectedValueOnce({ code: 'NoSuchKey' })
|
||||||
|
.mockResolvedValue({
|
||||||
|
contentLength: Buffer.byteLength('pdf-bytes'),
|
||||||
|
contentType: 'application/pdf',
|
||||||
|
eTag: 'etag-1',
|
||||||
|
});
|
||||||
|
hoisted.registerUploadedDocument.mockImplementation(async (input: { documentId: string; name: string; type: string; size: number; lastModified: number }) => ({
|
||||||
|
id: input.documentId,
|
||||||
|
name: input.name,
|
||||||
|
type: input.type,
|
||||||
|
size: input.size,
|
||||||
|
lastModified: input.lastModified,
|
||||||
|
scope: 'user',
|
||||||
|
}));
|
||||||
|
hoisted.putTempDocumentFinalizeReceipt.mockResolvedValue(undefined);
|
||||||
|
hoisted.deleteTempDocumentUpload.mockResolvedValue(undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('converts raw DOCX during finalize and registers a PDF', async () => {
|
||||||
|
const { POST } = await import('../../src/app/api/documents/blob/upload/finalize/route');
|
||||||
|
const request = new NextRequest('http://localhost/api/documents/blob/upload/finalize', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({
|
||||||
|
uploads: [{
|
||||||
|
token: '123e4567-e89b-12d3-a456-426614174000',
|
||||||
|
name: 'Report.docx',
|
||||||
|
type: 'docx',
|
||||||
|
lastModified: 1700000000000,
|
||||||
|
}],
|
||||||
|
}),
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await POST(request);
|
||||||
|
const expectedId = createHash('sha256').update(Buffer.from('pdf-bytes')).digest('hex');
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
await expect(response.json()).resolves.toMatchObject({
|
||||||
|
stored: [{
|
||||||
|
id: expectedId,
|
||||||
|
name: 'Report.pdf',
|
||||||
|
type: 'pdf',
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
expect(hoisted.convertDocxBufferToPdfBuffer).toHaveBeenCalledTimes(1);
|
||||||
|
expect(hoisted.putDocumentBlob).toHaveBeenCalledWith(
|
||||||
|
expectedId,
|
||||||
|
Buffer.from('pdf-bytes'),
|
||||||
|
'application/pdf',
|
||||||
|
null,
|
||||||
|
{ ifNoneMatch: true },
|
||||||
|
);
|
||||||
|
expect(hoisted.copyTempDocumentBlobToDocument).not.toHaveBeenCalled();
|
||||||
|
expect(hoisted.registerUploadedDocument).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
documentId: expectedId,
|
||||||
|
name: 'Report.pdf',
|
||||||
|
type: 'pdf',
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
});
|
||||||
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
import { describe, expect, test } from 'vitest';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import { buildPdfOpKey } from '../../src/lib/server/compute/worker';
|
||||||
|
|
||||||
|
describe('pdf worker op key', () => {
|
||||||
|
test('includes parser version in the reuse boundary', () => {
|
||||||
|
expect(buildPdfOpKey({
|
||||||
|
documentId: 'doc-1',
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: 'documents/doc-1.pdf',
|
||||||
|
})).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`);
|
||||||
|
});
|
||||||
|
});
|
||||||
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
import { describe, expect, test } from 'vitest';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
|
parseDocumentParseState,
|
||||||
|
stringifyDocumentParseState,
|
||||||
|
} from '../../src/lib/server/documents/parse-state';
|
||||||
|
|
||||||
|
describe('document parse state parser-version handling', () => {
|
||||||
|
test('preserves parserVersion through stringify/parse', () => {
|
||||||
|
const state = parseDocumentParseState(stringifyDocumentParseState({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1234,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
}));
|
||||||
|
|
||||||
|
expect(state).toEqual({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1234,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('invalidates ready and inflight states from older parser versions', () => {
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
opId: 'op-old',
|
||||||
|
}, 2222)).toEqual({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 2222,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'running',
|
||||||
|
progress: {
|
||||||
|
totalPages: 10,
|
||||||
|
pagesParsed: 3,
|
||||||
|
currentPage: 4,
|
||||||
|
phase: 'infer',
|
||||||
|
},
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
opId: 'op-old',
|
||||||
|
}, 3333)).toEqual({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 3333,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('keeps failed states and current-version ready states intact', () => {
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'failed',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
error: 'no text layer',
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
})).toEqual({
|
||||||
|
status: 'failed',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
error: 'no text layer',
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
})).toEqual({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,156 @@
|
||||||
|
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
|
||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
|
||||||
|
const hoisted = vi.hoisted(() => ({
|
||||||
|
db: null as {
|
||||||
|
select: ReturnType<typeof vi.fn>;
|
||||||
|
update: ReturnType<typeof vi.fn>;
|
||||||
|
} | null,
|
||||||
|
row: {
|
||||||
|
id: 'doc-1',
|
||||||
|
userId: 'user-1',
|
||||||
|
parseState: null as string | null,
|
||||||
|
},
|
||||||
|
requireAuthContext: vi.fn(),
|
||||||
|
backfillPendingPdfParseOperation: vi.fn(),
|
||||||
|
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/db', () => ({
|
||||||
|
get db() {
|
||||||
|
return hoisted.db;
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/auth/auth', () => ({
|
||||||
|
requireAuthContext: hoisted.requireAuthContext,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/parse-state-backfill', () => ({
|
||||||
|
backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
|
||||||
|
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||||
|
isValidDocumentId: vi.fn(() => true),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/storage/s3', () => ({
|
||||||
|
isS3Configured: vi.fn(() => true),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/testing/test-namespace', () => ({
|
||||||
|
getOpenReaderTestNamespace: vi.fn(() => null),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/logger', () => ({
|
||||||
|
createRequestLogger: vi.fn(() => ({
|
||||||
|
logger: {
|
||||||
|
warn: vi.fn(),
|
||||||
|
info: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
},
|
||||||
|
requestId: 'req-test',
|
||||||
|
})),
|
||||||
|
hashForLog: vi.fn(() => 'user-hash'),
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('GET /api/documents/[id]/parsed/events legacy backfill', () => {
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
process.env.COMPUTE_WORKER_URL = 'http://localhost:4010';
|
||||||
|
process.env.COMPUTE_WORKER_TOKEN = 'worker-test-token';
|
||||||
|
|
||||||
|
hoisted.row = {
|
||||||
|
id: 'doc-1',
|
||||||
|
userId: 'user-1',
|
||||||
|
parseState: null,
|
||||||
|
};
|
||||||
|
hoisted.db = {
|
||||||
|
select: vi.fn(() => ({
|
||||||
|
from: vi.fn(() => ({
|
||||||
|
where: vi.fn(async () => ([{ ...hoisted.row }])),
|
||||||
|
})),
|
||||||
|
})),
|
||||||
|
update: vi.fn(() => ({
|
||||||
|
set: vi.fn((values: { parseState?: string | null }) => ({
|
||||||
|
where: vi.fn(async () => {
|
||||||
|
if (typeof values.parseState !== 'undefined') {
|
||||||
|
hoisted.row.parseState = values.parseState;
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}),
|
||||||
|
})),
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
hoisted.requireAuthContext.mockReset();
|
||||||
|
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||||
|
hoisted.backfillPendingPdfParseOperation.mockReset();
|
||||||
|
hoisted.healStaleDocumentParseState.mockClear();
|
||||||
|
|
||||||
|
global.fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_, reject) => {
|
||||||
|
const signal = init?.signal;
|
||||||
|
if (signal) {
|
||||||
|
signal.addEventListener('abort', () => {
|
||||||
|
reject(new DOMException('Aborted', 'AbortError'));
|
||||||
|
}, { once: true });
|
||||||
|
}
|
||||||
|
})) as typeof fetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
test('creates a worker op for legacy pending PDFs without opId', async () => {
|
||||||
|
hoisted.backfillPendingPdfParseOperation.mockResolvedValue({
|
||||||
|
opId: 'op-legacy-1',
|
||||||
|
opKey: 'pdf_layout|v1|doc-1||doc-1|',
|
||||||
|
jobId: 'job-legacy-1',
|
||||||
|
kind: 'pdf_layout',
|
||||||
|
status: 'queued',
|
||||||
|
queuedAt: Date.now(),
|
||||||
|
progress: null,
|
||||||
|
result: undefined,
|
||||||
|
error: undefined,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const { GET } = await import('../../src/app/api/documents/[id]/parsed/events/route');
|
||||||
|
const controller = new AbortController();
|
||||||
|
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed/events', {
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
const response = await GET(request, {
|
||||||
|
params: Promise.resolve({ id: 'doc-1' }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
const reader = response.body?.getReader();
|
||||||
|
expect(reader).toBeDefined();
|
||||||
|
|
||||||
|
const first = await reader!.read();
|
||||||
|
const chunk = new TextDecoder().decode(first.value);
|
||||||
|
expect(chunk).toContain('event: snapshot');
|
||||||
|
expect(chunk).toContain('"parseStatus":"pending"');
|
||||||
|
expect(chunk).toContain('"opId":"op-legacy-1"');
|
||||||
|
|
||||||
|
controller.abort();
|
||||||
|
await reader!.cancel().catch(() => {});
|
||||||
|
|
||||||
|
expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
documentId: 'doc-1',
|
||||||
|
userId: 'user-1',
|
||||||
|
namespace: null,
|
||||||
|
state: expect.objectContaining({ status: 'pending' }),
|
||||||
|
}));
|
||||||
|
const parseState = String(hoisted.row.parseState ?? '');
|
||||||
|
expect(parseState).toContain('"status":"pending"');
|
||||||
|
expect(parseState).toContain('"opId":"op-legacy-1"');
|
||||||
|
expect(parseState).toContain('"jobId":"job-legacy-1"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -14,8 +14,9 @@ const hoisted = vi.hoisted(() => ({
|
||||||
},
|
},
|
||||||
requireAuthContext: vi.fn(),
|
requireAuthContext: vi.fn(),
|
||||||
fetchWorkerOperationState: vi.fn(),
|
fetchWorkerOperationState: vi.fn(),
|
||||||
backfillPendingPdfParseOperation: vi.fn(),
|
|
||||||
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
|
healStaleDocumentParseState: vi.fn(async ({ state }) => state),
|
||||||
|
startPdfParseOperation: vi.fn(),
|
||||||
|
enqueueParsePdfJob: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('@/db', () => ({
|
vi.mock('@/db', () => ({
|
||||||
|
|
@ -32,14 +33,18 @@ vi.mock('@/lib/server/compute/worker-op-state', () => ({
|
||||||
fetchWorkerOperationState: hoisted.fetchWorkerOperationState,
|
fetchWorkerOperationState: hoisted.fetchWorkerOperationState,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('@/lib/server/documents/parse-state-backfill', () => ({
|
|
||||||
backfillPendingPdfParseOperation: hoisted.backfillPendingPdfParseOperation,
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
|
vi.mock('@/lib/server/documents/parse-state-healing', () => ({
|
||||||
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
|
healStaleDocumentParseState: hoisted.healStaleDocumentParseState,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/documents/pdf-parse-operation', () => ({
|
||||||
|
startPdfParseOperation: hoisted.startPdfParseOperation,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@/lib/server/jobs/user-pdf-layout-job', () => ({
|
||||||
|
enqueueParsePdfJob: hoisted.enqueueParsePdfJob,
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock('@/lib/server/documents/blobstore', () => ({
|
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||||
documentKey: vi.fn(),
|
documentKey: vi.fn(),
|
||||||
getParsedDocumentBlob: vi.fn(),
|
getParsedDocumentBlob: vi.fn(),
|
||||||
|
|
@ -69,7 +74,7 @@ vi.mock('@/lib/server/logger', () => ({
|
||||||
hashForLog: vi.fn(() => 'user-hash'),
|
hashForLog: vi.fn(() => 'user-hash'),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
describe('GET /api/documents/[id]/parsed legacy backfill', () => {
|
describe('GET /api/documents/[id]/parsed pure data fetch', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
process.env.BASE_URL = 'http://localhost:3003';
|
process.env.BASE_URL = 'http://localhost:3003';
|
||||||
process.env.AUTH_SECRET = 'test-secret';
|
process.env.AUTH_SECRET = 'test-secret';
|
||||||
|
|
@ -104,44 +109,25 @@ describe('GET /api/documents/[id]/parsed legacy backfill', () => {
|
||||||
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||||
hoisted.fetchWorkerOperationState.mockReset();
|
hoisted.fetchWorkerOperationState.mockReset();
|
||||||
hoisted.fetchWorkerOperationState.mockResolvedValue(null);
|
hoisted.fetchWorkerOperationState.mockResolvedValue(null);
|
||||||
hoisted.backfillPendingPdfParseOperation.mockReset();
|
|
||||||
hoisted.healStaleDocumentParseState.mockClear();
|
hoisted.healStaleDocumentParseState.mockClear();
|
||||||
|
hoisted.startPdfParseOperation.mockReset();
|
||||||
|
hoisted.enqueueParsePdfJob.mockReset();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('creates a worker op for legacy pending PDFs without opId', async () => {
|
test('returns non-ready status without creating a worker op for legacy pending PDFs without opId', async () => {
|
||||||
hoisted.backfillPendingPdfParseOperation.mockResolvedValue({
|
|
||||||
opId: 'op-legacy-1',
|
|
||||||
opKey: 'pdf_layout|v1|doc-1||doc-1|',
|
|
||||||
jobId: 'job-legacy-1',
|
|
||||||
kind: 'pdf_layout',
|
|
||||||
status: 'queued',
|
|
||||||
queuedAt: Date.now(),
|
|
||||||
progress: null,
|
|
||||||
result: undefined,
|
|
||||||
error: undefined,
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const { GET } = await import('../../src/app/api/documents/[id]/parsed/route');
|
const { GET } = await import('../../src/app/api/documents/[id]/parsed/route');
|
||||||
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed');
|
const request = new NextRequest('http://localhost/api/documents/doc-1/parsed');
|
||||||
const response = await GET(request, {
|
const response = await GET(request, {
|
||||||
params: Promise.resolve({ id: 'doc-1' }),
|
params: Promise.resolve({ id: 'doc-1' }),
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(response.status).toBe(202);
|
expect(response.status).toBe(409);
|
||||||
await expect(response.json()).resolves.toMatchObject({
|
await expect(response.json()).resolves.toMatchObject({
|
||||||
parseStatus: 'pending',
|
parseStatus: 'pending',
|
||||||
opId: 'op-legacy-1',
|
opId: null,
|
||||||
});
|
});
|
||||||
expect(hoisted.backfillPendingPdfParseOperation).toHaveBeenCalledWith(expect.objectContaining({
|
expect(hoisted.startPdfParseOperation).not.toHaveBeenCalled();
|
||||||
documentId: 'doc-1',
|
expect(hoisted.enqueueParsePdfJob).not.toHaveBeenCalled();
|
||||||
userId: 'user-1',
|
expect(hoisted.row.parseState).toBeNull();
|
||||||
namespace: null,
|
|
||||||
state: expect.objectContaining({ status: 'pending' }),
|
|
||||||
}));
|
|
||||||
const parseState = String(hoisted.row.parseState ?? '');
|
|
||||||
expect(parseState).toContain('"status":"pending"');
|
|
||||||
expect(parseState).toContain('"opId":"op-legacy-1"');
|
|
||||||
expect(parseState).toContain('"jobId":"job-legacy-1"');
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||||
import {
|
import {
|
||||||
documentParseStateFromWorkerState,
|
documentParseStateFromWorkerState,
|
||||||
|
|
@ -11,7 +12,7 @@ function makeWorkerState(
|
||||||
): WorkerOperationState<PdfLayoutJobResult> {
|
): WorkerOperationState<PdfLayoutJobResult> {
|
||||||
return {
|
return {
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
opKey: 'pdf_layout|v1|doc-1',
|
opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`,
|
||||||
kind: 'pdf_layout',
|
kind: 'pdf_layout',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
status: 'queued',
|
status: 'queued',
|
||||||
|
|
@ -41,6 +42,7 @@ describe('worker parse state mapping', () => {
|
||||||
status: 'pending',
|
status: 'pending',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: 1234,
|
updatedAt: 1234,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
});
|
});
|
||||||
|
|
@ -66,6 +68,7 @@ describe('worker parse state mapping', () => {
|
||||||
phase: 'infer',
|
phase: 'infer',
|
||||||
},
|
},
|
||||||
updatedAt: 5678,
|
updatedAt: 5678,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
});
|
});
|
||||||
|
|
@ -84,6 +87,7 @@ describe('worker parse state mapping', () => {
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: 9999,
|
updatedAt: 9999,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
error: 'layout model crashed',
|
error: 'layout model crashed',
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,29 @@ test.describe('Document Upload Tests', () => {
|
||||||
await expectDocumentListed(page, 'sample.txt');
|
await expectDocumentListed(page, 'sample.txt');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('reuses the same canonical id for identical uploads', async ({ page }) => {
|
||||||
|
await uploadFile(page, 'sample.pdf');
|
||||||
|
await uploadFile(page, 'sample.pdf');
|
||||||
|
|
||||||
|
const result = await page.evaluate(async () => {
|
||||||
|
const res = await fetch('/api/documents', { cache: 'no-store' });
|
||||||
|
if (!res.ok) {
|
||||||
|
return { ok: false as const, reason: `status:${res.status}` };
|
||||||
|
}
|
||||||
|
const data = await res.json() as { documents?: Array<{ id: string; name: string }> };
|
||||||
|
const matching = (data.documents || []).filter((doc) => doc.name === 'sample.pdf');
|
||||||
|
const uniqueIds = Array.from(new Set(matching.map((doc) => doc.id)));
|
||||||
|
return { ok: true as const, matchingCount: matching.length, uniqueIds };
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.ok).toBeTruthy();
|
||||||
|
if (!result.ok) {
|
||||||
|
throw new Error(`Failed to inspect uploaded documents: ${result.reason}`);
|
||||||
|
}
|
||||||
|
expect(result.matchingCount).toBe(1);
|
||||||
|
expect(result.uniqueIds).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
test('hashes text/HTML docs using UTF-8 encoded stored string', async ({ page }) => {
|
test('hashes text/HTML docs using UTF-8 encoded stored string', async ({ page }) => {
|
||||||
await uploadFile(page, 'sample.txt');
|
await uploadFile(page, 'sample.txt');
|
||||||
await expectDocumentListed(page, 'sample.txt');
|
await expectDocumentListed(page, 'sample.txt');
|
||||||
|
|
@ -77,13 +100,7 @@ test.describe('Document Upload Tests', () => {
|
||||||
|
|
||||||
test('uploads and converts a DOCX document', async ({ page }) => {
|
test('uploads and converts a DOCX document', async ({ page }) => {
|
||||||
await uploadFile(page, 'sample.docx');
|
await uploadFile(page, 'sample.docx');
|
||||||
// Should see the converting message (best-effort; conversion may complete extremely fast)
|
// DOCX uploads are normalized into stored PDFs with the same basename.
|
||||||
try {
|
|
||||||
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
// After conversion, should see the PDF with the same name
|
|
||||||
await expectDocumentListed(page, 'sample.pdf');
|
await expectDocumentListed(page, 'sample.pdf');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue