feat(pdf): introduce parser versioning and playback readiness state
Add PDF_PARSER_VERSION constant and propagate parser versioning throughout the PDF parsing, job, and API layers. Implement normalization of parse state to ensure compatibility with the current parser version, and enable reuse of parsed PDF results when possible. Add isPlaybackReady state to document hooks and TTS player, improving playback UX by disabling controls until content is ready. Update tests to reflect new playback readiness logic. BREAKING CHANGE: PDF parse state and job logic now require explicit parserVersion; older parse states may be treated as pending until reprocessed.
This commit is contained in:
parent
17c69f074f
commit
2e4f36f5c5
27 changed files with 398 additions and 48 deletions
|
|
@ -16,6 +16,7 @@ export {
|
|||
} from './config/timeout';
|
||||
export { renderPage } from './pdf/render';
|
||||
export { mergeTextWithRegions } from './pdf/merge';
|
||||
export { PDF_PARSER_VERSION } from './pdf/parser-version';
|
||||
export { stitchCrossPageBlocks } from './pdf/stitch';
|
||||
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { ensureModel } from './model';
|
|||
import { runLayoutModel } from './runLayoutModel';
|
||||
import { mergeTextWithRegions } from './merge';
|
||||
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||
import { PDF_PARSER_VERSION } from './parser-version';
|
||||
import { stitchCrossPageBlocks } from './stitch';
|
||||
import { renderPage } from './render';
|
||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||
|
|
@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
|
|||
const doc: ParsedPdfDocument = {
|
||||
schemaVersion: 1,
|
||||
documentId: input.documentId,
|
||||
parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69',
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
parsedAt: Date.now(),
|
||||
pages,
|
||||
};
|
||||
|
|
|
|||
1
compute/core/src/pdf/parser-version.ts
Normal file
1
compute/core/src/pdf/parser-version.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||
|
|
@ -1,12 +1,15 @@
|
|||
import fs from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import path from 'node:path';
|
||||
|
||||
let cachedStandardFontDataUrl: string | null = null;
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
export function resolvePdfjsStandardFontDataUrl(): string {
|
||||
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
||||
|
||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
||||
const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json'));
|
||||
const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts');
|
||||
|
||||
if (!fs.existsSync(standardFontDir)) {
|
||||
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export default function EPUBPage() {
|
|||
const {
|
||||
setCurrentDocument,
|
||||
currDocName,
|
||||
isPlaybackReady,
|
||||
clearCurrDoc,
|
||||
createFullAudioBook: createEPUBAudioBook,
|
||||
regenerateChapter: regenerateEPUBChapter,
|
||||
|
|
@ -242,7 +243,7 @@ export default function EPUBPage() {
|
|||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<TTSPlayer />
|
||||
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||
)}
|
||||
<DocumentSettings
|
||||
epub
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ export interface EpubDocumentState {
|
|||
currDocPages: number | undefined;
|
||||
currDocPage: number | string;
|
||||
currDocText: string | undefined;
|
||||
isPlaybackReady: boolean;
|
||||
setCurrentDocument: (id: string) => Promise<void>;
|
||||
clearCurrDoc: () => void;
|
||||
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
|
||||
|
|
@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [currDocText, setCurrDocText] = useState<string>();
|
||||
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||
// Mirror state into a ref so resolveEpubLocator (registered once with
|
||||
// TTSContext via a stable callback) can always read the latest page text
|
||||
// without forcing re-registration on every page turn.
|
||||
|
|
@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
setCurrDocData(undefined);
|
||||
setCurrDocName(undefined);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
setCurrDocPages(undefined);
|
||||
isEPUBSetOnce.current = false;
|
||||
shouldPauseRef.current = true;
|
||||
|
|
@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
*/
|
||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||
try {
|
||||
setIsPlaybackReady(false);
|
||||
const meta = await getDocumentMetadata(id);
|
||||
if (!meta) {
|
||||
clearCurrDoc();
|
||||
|
|
@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
*/
|
||||
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
|
||||
try {
|
||||
setIsPlaybackReady(false);
|
||||
const location = rendition?.location;
|
||||
if (!location) return '';
|
||||
const { start, end } = location;
|
||||
|
|
@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
nextText: continuationPreview
|
||||
});
|
||||
setCurrDocText(textContent);
|
||||
setIsPlaybackReady(true);
|
||||
|
||||
return textContent;
|
||||
} catch (error) {
|
||||
|
|
@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
clearCurrDoc,
|
||||
extractPageText,
|
||||
walkUpcomingRenderedLocations,
|
||||
|
|
@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
clearCurrDoc,
|
||||
extractPageText,
|
||||
walkUpcomingRenderedLocations,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export default function HTMLPage() {
|
|||
setCurrentDocument,
|
||||
currDocData,
|
||||
currDocName,
|
||||
isPlaybackReady,
|
||||
blocks,
|
||||
isTxt,
|
||||
clearCurrDoc,
|
||||
|
|
@ -233,7 +234,7 @@ export default function HTMLPage() {
|
|||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<TTSPlayer />
|
||||
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||
)}
|
||||
<DocumentSettings
|
||||
html
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ export interface HtmlDocumentState {
|
|||
currDocData: string | undefined;
|
||||
currDocName: string | undefined;
|
||||
currDocText: string | undefined;
|
||||
isPlaybackReady: boolean;
|
||||
blocks: HtmlBlock[];
|
||||
isTxt: boolean;
|
||||
setCurrentDocument: (id: string) => Promise<void>;
|
||||
|
|
@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
|
||||
const [currDocData, setCurrDocData] = useState<string>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||
|
||||
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
|
||||
const blocks = useMemo(
|
||||
|
|
@ -91,22 +93,36 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
// sentence splitting + sequential advancement from there.
|
||||
const lastFedDocRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (!currDocText) return;
|
||||
if (currDocData === undefined) {
|
||||
setIsPlaybackReady(false);
|
||||
return;
|
||||
}
|
||||
if (!currDocText) {
|
||||
setIsPlaybackReady(true);
|
||||
return;
|
||||
}
|
||||
const key = `${currDocData ?? ''}::${currDocText.length}`;
|
||||
if (lastFedDocRef.current === key) return;
|
||||
if (lastFedDocRef.current === key) {
|
||||
setIsPlaybackReady(true);
|
||||
return;
|
||||
}
|
||||
setIsPlaybackReady(false);
|
||||
lastFedDocRef.current = key;
|
||||
setTTSText(currDocText);
|
||||
setIsPlaybackReady(true);
|
||||
}, [currDocText, currDocData, setTTSText]);
|
||||
|
||||
const clearCurrDoc = useCallback(() => {
|
||||
setCurrDocData(undefined);
|
||||
setCurrDocName(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastFedDocRef.current = null;
|
||||
stop();
|
||||
}, [stop]);
|
||||
|
||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||
try {
|
||||
setIsPlaybackReady(false);
|
||||
const meta = await getDocumentMetadata(id);
|
||||
if (!meta) {
|
||||
console.error('Document not found on server');
|
||||
|
|
@ -207,6 +223,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
currDocData,
|
||||
currDocName,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
blocks,
|
||||
isTxt,
|
||||
setCurrentDocument,
|
||||
|
|
@ -218,6 +235,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
|||
currDocData,
|
||||
currDocName,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
blocks,
|
||||
isTxt,
|
||||
setCurrentDocument,
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ export default function PDFViewerPage() {
|
|||
clearCurrDoc,
|
||||
currDocPage,
|
||||
currDocPages,
|
||||
isPlaybackReady,
|
||||
parseStatus,
|
||||
parseProgress,
|
||||
documentSettings,
|
||||
|
|
@ -487,7 +488,7 @@ export default function PDFViewerPage() {
|
|||
</div>
|
||||
</div>
|
||||
) : isParseReady ? (
|
||||
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} />
|
||||
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} isPlaybackReady={isPlaybackReady} />
|
||||
) : null}
|
||||
<DocumentSettings
|
||||
isOpen={activeSidebar === 'settings'}
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ export interface PdfDocumentState {
|
|||
currDocPages: number | undefined;
|
||||
currDocPage: number;
|
||||
currDocText: string | undefined;
|
||||
isPlaybackReady: boolean;
|
||||
pdfDocument: PDFDocumentProxy | undefined;
|
||||
parsedDocument: ParsedPdfDocument | null;
|
||||
parseStatus: PdfParseStatus | null;
|
||||
|
|
@ -142,6 +143,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [currDocText, setCurrDocText] = useState<string>();
|
||||
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
||||
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
||||
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
||||
|
|
@ -169,6 +171,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
const docLoadAbortRef = useRef<AbortController | null>(null);
|
||||
const parsePollAbortRef = useRef<AbortController | null>(null);
|
||||
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
||||
const lastPreparedPlaybackPageRef = useRef<number | null>(null);
|
||||
|
||||
const fetchParsedDocument = useCallback(async (
|
||||
documentId: string,
|
||||
|
|
@ -248,6 +251,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
|
||||
if (snapshot.parseStatus === 'failed') {
|
||||
setParsedDocument(null);
|
||||
setIsPlaybackReady(false);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setActiveParseOpId(null);
|
||||
} else {
|
||||
void fetchParsedDocument(
|
||||
|
|
@ -297,6 +302,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
|
||||
useEffect(() => {
|
||||
setCurrDocPage(currDocPageNumber);
|
||||
setIsPlaybackReady(false);
|
||||
}, [currDocPageNumber]);
|
||||
|
||||
/**
|
||||
|
|
@ -324,11 +330,13 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
if (!currentPdf) return;
|
||||
const seq = ++loadSeqRef.current;
|
||||
const pageNumber = currDocPageNumber;
|
||||
setIsPlaybackReady(false);
|
||||
|
||||
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
|
||||
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
|
||||
|
||||
if (parseStatus !== 'ready' || !parsedDocument) {
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setCurrDocText(undefined);
|
||||
setTTSText('', { location: currDocPageNumber });
|
||||
return;
|
||||
|
|
@ -401,7 +409,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
return;
|
||||
}
|
||||
|
||||
if (text !== currDocText || text === '') {
|
||||
const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber;
|
||||
if (shouldPreparePlayback) {
|
||||
setCurrDocText(text);
|
||||
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
|
||||
setTTSText(text, {
|
||||
|
|
@ -414,6 +423,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
|
||||
});
|
||||
}
|
||||
lastPreparedPlaybackPageRef.current = currDocPageNumber;
|
||||
setIsPlaybackReady(true);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
|
|
@ -469,6 +480,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
setPdfDocument(undefined);
|
||||
setCurrDocPages(undefined);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setCurrDocId(id);
|
||||
setCurrDocName(undefined);
|
||||
setCurrDocData(undefined);
|
||||
|
|
@ -546,6 +559,8 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
pageTextCacheRef.current.clear();
|
||||
setParsedDocument(null);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
setParseStatus(forced.status);
|
||||
setParseProgress(null);
|
||||
setActiveParseOpId(forced.opId ?? null);
|
||||
|
|
@ -575,6 +590,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
setCurrDocName(undefined);
|
||||
setCurrDocData(undefined);
|
||||
setCurrDocText(undefined);
|
||||
setIsPlaybackReady(false);
|
||||
setCurrDocPages(undefined);
|
||||
setPdfDocument(undefined);
|
||||
setParsedDocument(null);
|
||||
|
|
@ -582,6 +598,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
setParseProgress(null);
|
||||
setActiveParseOpId(null);
|
||||
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
||||
lastPreparedPlaybackPageRef.current = null;
|
||||
pageTextCacheRef.current.clear();
|
||||
stop();
|
||||
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
|
||||
|
|
@ -681,6 +698,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
parsedDocument,
|
||||
parseStatus,
|
||||
parseProgress,
|
||||
|
|
@ -708,6 +726,7 @@ export function usePdfDocument(): PdfDocumentState {
|
|||
currDocPages,
|
||||
currDocPage,
|
||||
currDocText,
|
||||
isPlaybackReady,
|
||||
parsedDocument,
|
||||
parseStatus,
|
||||
parseProgress,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/serv
|
|||
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
|
|
@ -67,7 +68,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
|
|||
const state = await healStaleDocumentParseState({
|
||||
documentId: row.id,
|
||||
userId: row.userId,
|
||||
state: parseDocumentParseState(row.parseState),
|
||||
state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)),
|
||||
});
|
||||
const parseStatus = normalizeParseStatus(state.status);
|
||||
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
|
||||
|
|
@ -199,7 +200,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
|
||||
let initialState = await toSnapshotState(row, requestedOpId);
|
||||
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
|
||||
const state = parseDocumentParseState(row.parseState);
|
||||
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
const created = await backfillPendingPdfParseOperation({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
|
|
|
|||
|
|
@ -20,8 +20,10 @@ import {
|
|||
putParsedDocumentBlob,
|
||||
} from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
resolveParsedPdfParserVersion,
|
||||
stringifyDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill';
|
||||
|
|
@ -156,13 +158,6 @@ async function finalizeFromWorkerState(input: {
|
|||
});
|
||||
}
|
||||
|
||||
await writeParseRowState({
|
||||
documentId: input.row.id,
|
||||
userId: input.row.userId,
|
||||
parseState: stringifyDocumentParseState(workerParseState),
|
||||
parsedJsonKey,
|
||||
});
|
||||
|
||||
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||
let parsedDoc: ParsedPdfDocument | null = null;
|
||||
try {
|
||||
|
|
@ -171,6 +166,16 @@ async function finalizeFromWorkerState(input: {
|
|||
parsedDoc = null;
|
||||
}
|
||||
|
||||
await writeParseRowState({
|
||||
documentId: input.row.id,
|
||||
userId: input.row.userId,
|
||||
parseState: stringifyDocumentParseState({
|
||||
...workerParseState,
|
||||
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
|
||||
}),
|
||||
parsedJsonKey,
|
||||
});
|
||||
|
||||
if (!hasAnyParsedBlocks(parsedDoc)) {
|
||||
input.logger.warn({
|
||||
event: 'documents.parsed.no_blocks_in_output',
|
||||
|
|
@ -246,7 +251,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
});
|
||||
}
|
||||
|
||||
let state = parseDocumentParseState(row.parseState);
|
||||
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
state = await healStaleDocumentParseState({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
|
|
@ -409,7 +414,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
let state = parseDocumentParseState(row.parseState);
|
||||
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
state = await healStaleDocumentParseState({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import { isS3Configured } from '@/lib/server/storage/s3';
|
|||
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||
|
|
@ -142,7 +143,12 @@ export async function POST(req: NextRequest) {
|
|||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
||||
parseState: stringifyDocumentParseState({
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
}),
|
||||
parsedJsonKey: null,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
|
|
@ -153,7 +159,12 @@ export async function POST(req: NextRequest) {
|
|||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
||||
parseState: stringifyDocumentParseState({
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
}),
|
||||
parsedJsonKey: null,
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -17,12 +17,15 @@ import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/
|
|||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
|
@ -100,6 +103,9 @@ export async function POST(req: NextRequest) {
|
|||
|
||||
for (const doc of documentsData) {
|
||||
let headSize = doc.size;
|
||||
const reusableParsedPdf = doc.type === 'pdf'
|
||||
? await findReusableParsedPdfResult(doc.id)
|
||||
: null;
|
||||
|
||||
// Retry HEAD check to handle S3 read-after-write propagation delays.
|
||||
// The client uploads bytes directly to S3 via presigned URL, then
|
||||
|
|
@ -146,9 +152,13 @@ export async function POST(req: NextRequest) {
|
|||
lastModified: doc.lastModified,
|
||||
filePath: doc.id,
|
||||
parseState: doc.type === 'pdf'
|
||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
||||
? stringifyDocumentParseState(
|
||||
reusableParsedPdf
|
||||
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||
: { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION },
|
||||
)
|
||||
: null,
|
||||
parsedJsonKey: null,
|
||||
parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
|
|
@ -159,9 +169,13 @@ export async function POST(req: NextRequest) {
|
|||
lastModified: doc.lastModified,
|
||||
filePath: doc.id,
|
||||
parseState: doc.type === 'pdf'
|
||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
||||
? stringifyDocumentParseState(
|
||||
reusableParsedPdf
|
||||
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||
: { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION },
|
||||
)
|
||||
: null,
|
||||
parsedJsonKey: null,
|
||||
parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -191,7 +205,7 @@ export async function POST(req: NextRequest) {
|
|||
}, 'Failed to enqueue document preview');
|
||||
});
|
||||
|
||||
if (doc.type === 'pdf') {
|
||||
if (doc.type === 'pdf' && !reusableParsedPdf) {
|
||||
// Account for upload-driven parse load in the same ledger the explicit
|
||||
// re-parse limiter reads. We record (not reject) here so a legitimate
|
||||
// bulk upload always parses; the recorded load still throttles
|
||||
|
|
@ -260,13 +274,16 @@ export async function GET(req: NextRequest) {
|
|||
|
||||
const results: BaseDocument[] = rows.map((doc) => {
|
||||
const type = normalizeDocumentType(doc.type, doc.name);
|
||||
const parseState = type === 'pdf'
|
||||
? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState))
|
||||
: null;
|
||||
return {
|
||||
id: doc.id,
|
||||
name: doc.name,
|
||||
size: Number(doc.size),
|
||||
lastModified: Number(doc.lastModified),
|
||||
type,
|
||||
parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null,
|
||||
parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null,
|
||||
parsedJsonKey: doc.parsedJsonKey,
|
||||
scope: 'user',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl';
|
|||
import { Navigator } from '@/components/player/Navigator';
|
||||
import { IconButton } from '@/components/ui';
|
||||
|
||||
export default function TTSPlayer({ currentPage, numPages }: {
|
||||
export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: {
|
||||
currentPage?: number;
|
||||
numPages?: number | undefined;
|
||||
isPlaybackReady?: boolean;
|
||||
}) {
|
||||
const {
|
||||
isPlaying,
|
||||
|
|
@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
|||
<IconButton
|
||||
onClick={skipBackward}
|
||||
aria-label="Skip backward"
|
||||
disabled={isProcessing}
|
||||
disabled={isProcessing || !isPlaybackReady}
|
||||
className="relative"
|
||||
>
|
||||
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
|
||||
|
|
@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
|||
<IconButton
|
||||
onClick={togglePlay}
|
||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||
disabled={isProcessing && !isPlaying}
|
||||
disabled={!isPlaying && (!isPlaybackReady || isProcessing)}
|
||||
className="relative"
|
||||
>
|
||||
{isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />}
|
||||
{!isPlaying && !isPlaybackReady
|
||||
? <LoadingSpinner />
|
||||
: (isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />)}
|
||||
</IconButton>
|
||||
|
||||
<IconButton
|
||||
onClick={skipForward}
|
||||
aria-label="Skip forward"
|
||||
disabled={isProcessing}
|
||||
disabled={isProcessing || !isPlaybackReady}
|
||||
className="relative"
|
||||
>
|
||||
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}
|
||||
|
|
|
|||
|
|
@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
||||
const isPlayingRef = useRef(false);
|
||||
const pauseEpochRef = useRef(0);
|
||||
const pendingPlaybackStartRef = useRef(false);
|
||||
const sentencesRef = useRef<string[]>([]);
|
||||
const currentIndexRef = useRef(0);
|
||||
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
|
||||
|
|
@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const recordManualPause = useCallback(() => {
|
||||
// Cancel any queued auto-resume intent and mark an explicit user pause.
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
pendingPlaybackStartRef.current = false;
|
||||
pauseEpochRef.current += 1;
|
||||
}, []);
|
||||
|
||||
|
|
@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
* Used for external control of playback state
|
||||
*/
|
||||
const pause = useCallback(() => {
|
||||
pendingPlaybackStartRef.current = false;
|
||||
recordManualPause();
|
||||
clearPendingEpubJump();
|
||||
abortPendingTtsRequests(true);
|
||||
|
|
@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const shouldPause = normalizedOptions.shouldPause ?? false;
|
||||
const pauseEpochAtStart = pauseEpochRef.current;
|
||||
const pendingAutoResume = resumeAfterLocationChangeRef.current;
|
||||
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume);
|
||||
if (shouldPause || pendingAutoResume) {
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
}
|
||||
const pendingPlaybackStart = pendingPlaybackStartRef.current;
|
||||
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart);
|
||||
|
||||
// Keep track of previous state and pause playback
|
||||
invalidatePlaybackRun();
|
||||
|
|
@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
try {
|
||||
if (newSentences.length === 0) {
|
||||
console.warn('No sentences found in text');
|
||||
if (shouldPause || pendingAutoResume) {
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
}
|
||||
if (pendingPlaybackStart) {
|
||||
pendingPlaybackStartRef.current = false;
|
||||
}
|
||||
setIsProcessing(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (shouldPause || pendingAutoResume) {
|
||||
resumeAfterLocationChangeRef.current = false;
|
||||
}
|
||||
if (pendingPlaybackStart) {
|
||||
pendingPlaybackStartRef.current = false;
|
||||
}
|
||||
|
||||
if (!isEPUB && typeof resolvedLocation === 'number') {
|
||||
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
|
||||
if (firstFingerprint) {
|
||||
|
|
@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
// Ensure audio is unlocked while we're still in the click/tap handler.
|
||||
unlockPlaybackOnUserGesture();
|
||||
|
||||
const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]);
|
||||
if (!activeHowl && !hasPreparedSentence) {
|
||||
pendingPlaybackStartRef.current = true;
|
||||
setIsProcessing(true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Resume current sentence if we already have a paused Howl.
|
||||
if (activeHowl) {
|
||||
applyPlaybackRateToHowl(activeHowl);
|
||||
|
|
@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
setIsPlaying(true);
|
||||
}, [
|
||||
activeHowl,
|
||||
currentIndex,
|
||||
applyPlaybackRateToHowl,
|
||||
isPlaying,
|
||||
pauseActiveHowl,
|
||||
playbackSegments,
|
||||
recordManualPause,
|
||||
clearPendingEpubJump,
|
||||
abortPendingTtsRequests,
|
||||
sentences,
|
||||
unlockPlaybackOnUserGesture,
|
||||
]);
|
||||
|
||||
|
|
@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
abortAudio();
|
||||
clearWarmAudioCache();
|
||||
playbackInFlightRef.current = false;
|
||||
pendingPlaybackStartRef.current = false;
|
||||
pendingJumpTargetRef.current = null;
|
||||
clearPendingEpubJump();
|
||||
bumpEpubPreloadGeneration();
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
|
|
@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState(
|
|||
? parseProgress
|
||||
: null,
|
||||
updatedAt: nowMs,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
|
||||
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
|
||||
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
|
||||
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
|
||||
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core';
|
||||
import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||
import type {
|
||||
|
|
@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string {
|
|||
return [
|
||||
'pdf_layout',
|
||||
'v1',
|
||||
PDF_PARSER_VERSION,
|
||||
input.documentId,
|
||||
input.namespace ?? '',
|
||||
input.documentObjectKey ?? '',
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: {
|
|||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||
...(input.state.opId ? { opId: input.state.opId } : {}),
|
||||
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
|
||||
...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}),
|
||||
}));
|
||||
|
||||
await db
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
|
||||
export interface DocumentParseState {
|
||||
status: PdfParseStatus;
|
||||
|
|
@ -7,6 +8,7 @@ export interface DocumentParseState {
|
|||
error?: string | null;
|
||||
opId?: string;
|
||||
jobId?: string;
|
||||
parserVersion?: string;
|
||||
}
|
||||
|
||||
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
|
||||
|
|
@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars
|
|||
return 'pending';
|
||||
}
|
||||
|
||||
export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean {
|
||||
return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION;
|
||||
}
|
||||
|
||||
export function normalizeDocumentParseStateForCurrentParserVersion(
|
||||
state: DocumentParseState,
|
||||
nowMs = Date.now(),
|
||||
): DocumentParseState {
|
||||
const status = normalizeParseStatus(state.status);
|
||||
if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) {
|
||||
return state;
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: nowMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string {
|
||||
const parserVersion = parsed?.parserVersion?.trim();
|
||||
return parserVersion || PDF_PARSER_VERSION;
|
||||
}
|
||||
|
||||
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
||||
if (!progress || typeof progress !== 'object') return null;
|
||||
const rec = progress as Record<string, unknown>;
|
||||
|
|
@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
|||
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
||||
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
||||
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
||||
const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined;
|
||||
return {
|
||||
status,
|
||||
progress,
|
||||
|
|
@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
|||
...(error ? { error } : {}),
|
||||
...(opId ? { opId } : {}),
|
||||
...(jobId ? { jobId } : {}),
|
||||
...(parserVersion ? { parserVersion } : {}),
|
||||
};
|
||||
} catch {
|
||||
return { status: 'pending', progress: null };
|
||||
|
|
@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string {
|
|||
...(state.error ? { error: state.error } : {}),
|
||||
...(state.opId ? { opId: state.opId } : {}),
|
||||
...(state.jobId ? { jobId: state.jobId } : {}),
|
||||
...(state.parserVersion ? { parserVersion: state.parserVersion } : {}),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { eq } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
parseDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
|
||||
type ReadyParsedPdfResult = {
|
||||
parsedJsonKey: string;
|
||||
};
|
||||
|
||||
export async function findReusableParsedPdfResult(
|
||||
documentId: string,
|
||||
): Promise<ReadyParsedPdfResult | null> {
|
||||
const rows = await db
|
||||
.select({
|
||||
parseState: documents.parseState,
|
||||
parsedJsonKey: documents.parsedJsonKey,
|
||||
})
|
||||
.from(documents)
|
||||
.where(eq(documents.id, documentId));
|
||||
|
||||
for (const row of rows) {
|
||||
const parsedJsonKey = row.parsedJsonKey?.trim();
|
||||
if (!parsedJsonKey) continue;
|
||||
const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||
if (parseState.status !== 'ready') continue;
|
||||
return { parsedJsonKey };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
|
@ -1,11 +1,14 @@
|
|||
import { and, eq, inArray, isNull } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||
import { serverLogger } from '@/lib/server/logger';
|
||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
parseDocumentParseState,
|
||||
resolveParsedPdfParserVersion,
|
||||
stringifyDocumentParseState,
|
||||
type DocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
|
|
@ -13,11 +16,13 @@ import { getCompute } from '@/lib/server/compute';
|
|||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
|
||||
import type {
|
||||
ParsedPdfDocument,
|
||||
PdfLayoutJobBase,
|
||||
PdfLayoutJobResult,
|
||||
PdfLayoutProgress,
|
||||
WorkerOperationState,
|
||||
} from '@openreader/compute-core/api-contracts';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
|
||||
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
|
||||
userId: string;
|
||||
|
|
@ -41,7 +46,7 @@ function keyFor(input: UserPdfLayoutJobRequest): string {
|
|||
if (forceToken) {
|
||||
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
|
||||
}
|
||||
return `shared:${input.documentId}:${input.namespace || ''}`;
|
||||
return `shared:${input.documentId}`;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
|
|
@ -84,7 +89,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise<ParseRow[
|
|||
|
||||
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
|
||||
if (!row.parsedJsonKey) return false;
|
||||
return parseDocumentParseState(row.parseState).status === 'ready';
|
||||
return normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)).status === 'ready';
|
||||
}
|
||||
|
||||
function userIdsFromRows(rows: ParseRow[]): string[] {
|
||||
|
|
@ -95,6 +100,20 @@ function parseStateMatchCondition(expected: string | null) {
|
|||
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
|
||||
}
|
||||
|
||||
async function loadParsedDocumentForResult(
|
||||
parsed: ParsedPdfDocument | undefined,
|
||||
parsedJsonKey: string,
|
||||
): Promise<ParsedPdfDocument | null> {
|
||||
if (parsed) return parsed;
|
||||
|
||||
try {
|
||||
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||
return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateParseStateForUsers(input: {
|
||||
documentId: string;
|
||||
userIds: string[];
|
||||
|
|
@ -125,6 +144,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||
if (reusableParsed) {
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
userIds: [input.userId],
|
||||
parseState: stringifyDocumentParseState(readyState),
|
||||
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = await loadScopedRows(input);
|
||||
const ready = rows.find(isReadyRow);
|
||||
if (ready) {
|
||||
|
|
@ -132,6 +168,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
|
|
@ -142,7 +179,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
return;
|
||||
}
|
||||
|
||||
const statuses = rows.map((row) => parseDocumentParseState(row.parseState));
|
||||
const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)));
|
||||
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
|
||||
const failedState = statuses.find((state) => state.status === 'failed');
|
||||
if (failedState && !hasInFlight) {
|
||||
|
|
@ -151,6 +188,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
|||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
...(failedState.error ? { error: failedState.error } : {}),
|
||||
...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}),
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
|
|
@ -181,12 +219,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
// Non-force jobs can short-circuit by reusing existing ready output from
|
||||
// any row in the same document scope.
|
||||
if (!input.forceToken?.trim()) {
|
||||
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||
if (reusableParsed) {
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
userIds: [input.userId],
|
||||
parseState: stringifyDocumentParseState(readyState),
|
||||
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const ready = scopedRows.find(isReadyRow);
|
||||
if (ready) {
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
await updateParseStateForUsers({
|
||||
documentId: input.documentId,
|
||||
|
|
@ -205,6 +261,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
status: 'running',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
const runningState = stringifyDocumentParseState(runningStateData);
|
||||
|
||||
|
|
@ -279,6 +336,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
status: mappedStatus,
|
||||
progress: snapshot.progress ?? null,
|
||||
updatedAt: now,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
|
|
@ -305,6 +363,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
phase: progress.phase,
|
||||
},
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
...(activeOpId ? { opId: activeOpId } : {}),
|
||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||
};
|
||||
|
|
@ -327,12 +386,15 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
|
||||
}
|
||||
|
||||
const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey);
|
||||
|
||||
const finalScopedRows = await loadScopedRows(input);
|
||||
const finalUserIds = userIdsFromRows(finalScopedRows);
|
||||
const readyState: DocumentParseState = {
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
|
||||
};
|
||||
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
|
||||
await updateParseStateForUsers({
|
||||
|
|
@ -385,6 +447,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
|||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
error: message,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
};
|
||||
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
|
||||
await updateParseStateForUsers({
|
||||
|
|
|
|||
|
|
@ -188,10 +188,11 @@ async function dismissOnboardingModals(page: Page): Promise<void> {
|
|||
* Wait for the play button to be clickable and click it
|
||||
*/
|
||||
export async function waitAndClickPlay(page: Page) {
|
||||
// Wait for play button selector without disabled attribute
|
||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();
|
||||
const playButton = page.getByRole('button', { name: 'Play' });
|
||||
await expect(playButton).toBeVisible();
|
||||
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||
// Play the TTS by clicking the button
|
||||
await page.getByRole('button', { name: 'Play' }).click();
|
||||
await playButton.click();
|
||||
// Use resilient processing transition helper (tolerates fast completion)
|
||||
await expectProcessingTransition(page);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => {
|
|||
await page.waitForTimeout(750);
|
||||
|
||||
// After resize, playback should have paused (Play button visible)
|
||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });
|
||||
const playButton = page.getByRole('button', { name: 'Play' });
|
||||
await expect(playButton).toBeVisible({ timeout: 15000 });
|
||||
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||
|
||||
// Resume playback and ensure processing -> playing
|
||||
await page.getByRole('button', { name: 'Play' }).click();
|
||||
await playButton.click();
|
||||
await expectProcessingTransition(page);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import { buildPdfOpKey } from '../../src/lib/server/compute/worker';
|
||||
|
||||
describe('pdf worker op key', () => {
|
||||
test('includes parser version in the reuse boundary', () => {
|
||||
expect(buildPdfOpKey({
|
||||
documentId: 'doc-1',
|
||||
namespace: null,
|
||||
documentObjectKey: 'documents/doc-1.pdf',
|
||||
})).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`);
|
||||
});
|
||||
});
|
||||
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import {
|
||||
normalizeDocumentParseStateForCurrentParserVersion,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '../../src/lib/server/documents/parse-state';
|
||||
|
||||
describe('document parse state parser-version handling', () => {
|
||||
test('preserves parserVersion through stringify/parse', () => {
|
||||
const state = parseDocumentParseState(stringifyDocumentParseState({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1234,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
}));
|
||||
|
||||
expect(state).toEqual({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1234,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
});
|
||||
});
|
||||
|
||||
test('invalidates ready and inflight states from older parser versions', () => {
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
parserVersion: 'old-parser',
|
||||
opId: 'op-old',
|
||||
}, 2222)).toEqual({
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: 2222,
|
||||
});
|
||||
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'running',
|
||||
progress: {
|
||||
totalPages: 10,
|
||||
pagesParsed: 3,
|
||||
currentPage: 4,
|
||||
phase: 'infer',
|
||||
},
|
||||
updatedAt: 1111,
|
||||
parserVersion: 'old-parser',
|
||||
opId: 'op-old',
|
||||
}, 3333)).toEqual({
|
||||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: 3333,
|
||||
});
|
||||
});
|
||||
|
||||
test('keeps failed states and current-version ready states intact', () => {
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
error: 'no text layer',
|
||||
parserVersion: 'old-parser',
|
||||
})).toEqual({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
error: 'no text layer',
|
||||
parserVersion: 'old-parser',
|
||||
});
|
||||
|
||||
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
})).toEqual({
|
||||
status: 'ready',
|
||||
progress: null,
|
||||
updatedAt: 1111,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||
import {
|
||||
documentParseStateFromWorkerState,
|
||||
|
|
@ -11,7 +12,7 @@ function makeWorkerState(
|
|||
): WorkerOperationState<PdfLayoutJobResult> {
|
||||
return {
|
||||
opId: 'op-123',
|
||||
opKey: 'pdf_layout|v1|doc-1',
|
||||
opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`,
|
||||
kind: 'pdf_layout',
|
||||
jobId: 'job-123',
|
||||
status: 'queued',
|
||||
|
|
@ -41,6 +42,7 @@ describe('worker parse state mapping', () => {
|
|||
status: 'pending',
|
||||
progress: null,
|
||||
updatedAt: 1234,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
opId: 'op-123',
|
||||
jobId: 'job-123',
|
||||
});
|
||||
|
|
@ -66,6 +68,7 @@ describe('worker parse state mapping', () => {
|
|||
phase: 'infer',
|
||||
},
|
||||
updatedAt: 5678,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
opId: 'op-123',
|
||||
jobId: 'job-123',
|
||||
});
|
||||
|
|
@ -84,6 +87,7 @@ describe('worker parse state mapping', () => {
|
|||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: 9999,
|
||||
parserVersion: PDF_PARSER_VERSION,
|
||||
opId: 'op-123',
|
||||
jobId: 'job-123',
|
||||
error: 'layout model crashed',
|
||||
|
|
|
|||
Loading…
Reference in a new issue