feat(pdf): introduce parser versioning and playback readiness state
Add PDF_PARSER_VERSION constant and propagate parser versioning throughout the PDF parsing, job, and API layers. Implement normalization of parse state to ensure compatibility with the current parser version, and enable reuse of parsed PDF results when possible. Add isPlaybackReady state to document hooks and TTS player, improving playback UX by disabling controls until content is ready. Update tests to reflect new playback readiness logic. BREAKING CHANGE: PDF parse state and job logic now require explicit parserVersion; older parse states may be treated as pending until reprocessed.
This commit is contained in:
parent
17c69f074f
commit
2e4f36f5c5
27 changed files with 398 additions and 48 deletions
|
|
@ -16,6 +16,7 @@ export {
|
||||||
} from './config/timeout';
|
} from './config/timeout';
|
||||||
export { renderPage } from './pdf/render';
|
export { renderPage } from './pdf/render';
|
||||||
export { mergeTextWithRegions } from './pdf/merge';
|
export { mergeTextWithRegions } from './pdf/merge';
|
||||||
|
export { PDF_PARSER_VERSION } from './pdf/parser-version';
|
||||||
export { stitchCrossPageBlocks } from './pdf/stitch';
|
export { stitchCrossPageBlocks } from './pdf/stitch';
|
||||||
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
||||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import { ensureModel } from './model';
|
||||||
import { runLayoutModel } from './runLayoutModel';
|
import { runLayoutModel } from './runLayoutModel';
|
||||||
import { mergeTextWithRegions } from './merge';
|
import { mergeTextWithRegions } from './merge';
|
||||||
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||||
|
import { PDF_PARSER_VERSION } from './parser-version';
|
||||||
import { stitchCrossPageBlocks } from './stitch';
|
import { stitchCrossPageBlocks } from './stitch';
|
||||||
import { renderPage } from './render';
|
import { renderPage } from './render';
|
||||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||||
|
|
@ -143,7 +144,7 @@ export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument>
|
||||||
const doc: ParsedPdfDocument = {
|
const doc: ParsedPdfDocument = {
|
||||||
schemaVersion: 1,
|
schemaVersion: 1,
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69',
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
parsedAt: Date.now(),
|
parsedAt: Date.now(),
|
||||||
pages,
|
pages,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
1
compute/core/src/pdf/parser-version.ts
Normal file
1
compute/core/src/pdf/parser-version.ts
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||||
|
|
@ -1,12 +1,15 @@
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
let cachedStandardFontDataUrl: string | null = null;
|
let cachedStandardFontDataUrl: string | null = null;
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
export function resolvePdfjsStandardFontDataUrl(): string {
|
export function resolvePdfjsStandardFontDataUrl(): string {
|
||||||
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
||||||
|
|
||||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
const pdfjsPackageDir = path.dirname(require.resolve('pdfjs-dist/package.json'));
|
||||||
|
const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts');
|
||||||
|
|
||||||
if (!fs.existsSync(standardFontDir)) {
|
if (!fs.existsSync(standardFontDir)) {
|
||||||
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ export default function EPUBPage() {
|
||||||
const {
|
const {
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
currDocName,
|
currDocName,
|
||||||
|
isPlaybackReady,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
createFullAudioBook: createEPUBAudioBook,
|
createFullAudioBook: createEPUBAudioBook,
|
||||||
regenerateChapter: regenerateEPUBChapter,
|
regenerateChapter: regenerateEPUBChapter,
|
||||||
|
|
@ -242,7 +243,7 @@ export default function EPUBPage() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<TTSPlayer />
|
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||||
)}
|
)}
|
||||||
<DocumentSettings
|
<DocumentSettings
|
||||||
epub
|
epub
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,7 @@ export interface EpubDocumentState {
|
||||||
currDocPages: number | undefined;
|
currDocPages: number | undefined;
|
||||||
currDocPage: number | string;
|
currDocPage: number | string;
|
||||||
currDocText: string | undefined;
|
currDocText: string | undefined;
|
||||||
|
isPlaybackReady: boolean;
|
||||||
setCurrentDocument: (id: string) => Promise<void>;
|
setCurrentDocument: (id: string) => Promise<void>;
|
||||||
clearCurrDoc: () => void;
|
clearCurrDoc: () => void;
|
||||||
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
|
extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise<string>;
|
||||||
|
|
@ -106,6 +107,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
const [currDocText, setCurrDocText] = useState<string>();
|
const [currDocText, setCurrDocText] = useState<string>();
|
||||||
|
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||||
// Mirror state into a ref so resolveEpubLocator (registered once with
|
// Mirror state into a ref so resolveEpubLocator (registered once with
|
||||||
// TTSContext via a stable callback) can always read the latest page text
|
// TTSContext via a stable callback) can always read the latest page text
|
||||||
// without forcing re-registration on every page turn.
|
// without forcing re-registration on every page turn.
|
||||||
|
|
@ -160,6 +162,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
isEPUBSetOnce.current = false;
|
isEPUBSetOnce.current = false;
|
||||||
shouldPauseRef.current = true;
|
shouldPauseRef.current = true;
|
||||||
|
|
@ -179,6 +182,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
*/
|
*/
|
||||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||||
try {
|
try {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
const meta = await getDocumentMetadata(id);
|
const meta = await getDocumentMetadata(id);
|
||||||
if (!meta) {
|
if (!meta) {
|
||||||
clearCurrDoc();
|
clearCurrDoc();
|
||||||
|
|
@ -215,6 +219,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
*/
|
*/
|
||||||
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
|
const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise<string> => {
|
||||||
try {
|
try {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
const location = rendition?.location;
|
const location = rendition?.location;
|
||||||
if (!location) return '';
|
if (!location) return '';
|
||||||
const { start, end } = location;
|
const { start, end } = location;
|
||||||
|
|
@ -244,6 +249,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
nextText: continuationPreview
|
nextText: continuationPreview
|
||||||
});
|
});
|
||||||
setCurrDocText(textContent);
|
setCurrDocText(textContent);
|
||||||
|
setIsPlaybackReady(true);
|
||||||
|
|
||||||
return textContent;
|
return textContent;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -390,6 +396,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
extractPageText,
|
extractPageText,
|
||||||
walkUpcomingRenderedLocations,
|
walkUpcomingRenderedLocations,
|
||||||
|
|
@ -415,6 +422,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
extractPageText,
|
extractPageText,
|
||||||
walkUpcomingRenderedLocations,
|
walkUpcomingRenderedLocations,
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ export default function HTMLPage() {
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocName,
|
currDocName,
|
||||||
|
isPlaybackReady,
|
||||||
blocks,
|
blocks,
|
||||||
isTxt,
|
isTxt,
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
|
|
@ -233,7 +234,7 @@ export default function HTMLPage() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<TTSPlayer />
|
<TTSPlayer isPlaybackReady={isPlaybackReady} />
|
||||||
)}
|
)}
|
||||||
<DocumentSettings
|
<DocumentSettings
|
||||||
html
|
html
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ export interface HtmlDocumentState {
|
||||||
currDocData: string | undefined;
|
currDocData: string | undefined;
|
||||||
currDocName: string | undefined;
|
currDocName: string | undefined;
|
||||||
currDocText: string | undefined;
|
currDocText: string | undefined;
|
||||||
|
isPlaybackReady: boolean;
|
||||||
blocks: HtmlBlock[];
|
blocks: HtmlBlock[];
|
||||||
isTxt: boolean;
|
isTxt: boolean;
|
||||||
setCurrentDocument: (id: string) => Promise<void>;
|
setCurrentDocument: (id: string) => Promise<void>;
|
||||||
|
|
@ -73,6 +74,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
|
|
||||||
const [currDocData, setCurrDocData] = useState<string>();
|
const [currDocData, setCurrDocData] = useState<string>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
|
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||||
|
|
||||||
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
|
const isTxt = useMemo(() => isTxtName(currDocName), [currDocName]);
|
||||||
const blocks = useMemo(
|
const blocks = useMemo(
|
||||||
|
|
@ -91,22 +93,36 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
// sentence splitting + sequential advancement from there.
|
// sentence splitting + sequential advancement from there.
|
||||||
const lastFedDocRef = useRef<string | null>(null);
|
const lastFedDocRef = useRef<string | null>(null);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!currDocText) return;
|
if (currDocData === undefined) {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!currDocText) {
|
||||||
|
setIsPlaybackReady(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const key = `${currDocData ?? ''}::${currDocText.length}`;
|
const key = `${currDocData ?? ''}::${currDocText.length}`;
|
||||||
if (lastFedDocRef.current === key) return;
|
if (lastFedDocRef.current === key) {
|
||||||
|
setIsPlaybackReady(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setIsPlaybackReady(false);
|
||||||
lastFedDocRef.current = key;
|
lastFedDocRef.current = key;
|
||||||
setTTSText(currDocText);
|
setTTSText(currDocText);
|
||||||
|
setIsPlaybackReady(true);
|
||||||
}, [currDocText, currDocData, setTTSText]);
|
}, [currDocText, currDocData, setTTSText]);
|
||||||
|
|
||||||
const clearCurrDoc = useCallback(() => {
|
const clearCurrDoc = useCallback(() => {
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
lastFedDocRef.current = null;
|
lastFedDocRef.current = null;
|
||||||
stop();
|
stop();
|
||||||
}, [stop]);
|
}, [stop]);
|
||||||
|
|
||||||
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
const setCurrentDocument = useCallback(async (id: string): Promise<void> => {
|
||||||
try {
|
try {
|
||||||
|
setIsPlaybackReady(false);
|
||||||
const meta = await getDocumentMetadata(id);
|
const meta = await getDocumentMetadata(id);
|
||||||
if (!meta) {
|
if (!meta) {
|
||||||
console.error('Document not found on server');
|
console.error('Document not found on server');
|
||||||
|
|
@ -207,6 +223,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocName,
|
currDocName,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
blocks,
|
blocks,
|
||||||
isTxt,
|
isTxt,
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
|
|
@ -218,6 +235,7 @@ export function useHtmlDocument(): HtmlDocumentState {
|
||||||
currDocData,
|
currDocData,
|
||||||
currDocName,
|
currDocName,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
blocks,
|
blocks,
|
||||||
isTxt,
|
isTxt,
|
||||||
setCurrentDocument,
|
setCurrentDocument,
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,7 @@ export default function PDFViewerPage() {
|
||||||
clearCurrDoc,
|
clearCurrDoc,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocPages,
|
currDocPages,
|
||||||
|
isPlaybackReady,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
parseProgress,
|
parseProgress,
|
||||||
documentSettings,
|
documentSettings,
|
||||||
|
|
@ -487,7 +488,7 @@ export default function PDFViewerPage() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : isParseReady ? (
|
) : isParseReady ? (
|
||||||
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} />
|
<TTSPlayer currentPage={currDocPage} numPages={currDocPages} isPlaybackReady={isPlaybackReady} />
|
||||||
) : null}
|
) : null}
|
||||||
<DocumentSettings
|
<DocumentSettings
|
||||||
isOpen={activeSidebar === 'settings'}
|
isOpen={activeSidebar === 'settings'}
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,7 @@ export interface PdfDocumentState {
|
||||||
currDocPages: number | undefined;
|
currDocPages: number | undefined;
|
||||||
currDocPage: number;
|
currDocPage: number;
|
||||||
currDocText: string | undefined;
|
currDocText: string | undefined;
|
||||||
|
isPlaybackReady: boolean;
|
||||||
pdfDocument: PDFDocumentProxy | undefined;
|
pdfDocument: PDFDocumentProxy | undefined;
|
||||||
parsedDocument: ParsedPdfDocument | null;
|
parsedDocument: ParsedPdfDocument | null;
|
||||||
parseStatus: PdfParseStatus | null;
|
parseStatus: PdfParseStatus | null;
|
||||||
|
|
@ -142,6 +143,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
const [currDocData, setCurrDocData] = useState<ArrayBuffer>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
const [currDocText, setCurrDocText] = useState<string>();
|
const [currDocText, setCurrDocText] = useState<string>();
|
||||||
|
const [isPlaybackReady, setIsPlaybackReady] = useState(false);
|
||||||
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
|
||||||
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
|
||||||
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
|
||||||
|
|
@ -169,6 +171,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
const docLoadAbortRef = useRef<AbortController | null>(null);
|
const docLoadAbortRef = useRef<AbortController | null>(null);
|
||||||
const parsePollAbortRef = useRef<AbortController | null>(null);
|
const parsePollAbortRef = useRef<AbortController | null>(null);
|
||||||
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
const parseSseCloseRef = useRef<(() => void) | null>(null);
|
||||||
|
const lastPreparedPlaybackPageRef = useRef<number | null>(null);
|
||||||
|
|
||||||
const fetchParsedDocument = useCallback(async (
|
const fetchParsedDocument = useCallback(async (
|
||||||
documentId: string,
|
documentId: string,
|
||||||
|
|
@ -248,6 +251,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
|
if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') {
|
||||||
if (snapshot.parseStatus === 'failed') {
|
if (snapshot.parseStatus === 'failed') {
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setActiveParseOpId(null);
|
setActiveParseOpId(null);
|
||||||
} else {
|
} else {
|
||||||
void fetchParsedDocument(
|
void fetchParsedDocument(
|
||||||
|
|
@ -297,6 +302,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setCurrDocPage(currDocPageNumber);
|
setCurrDocPage(currDocPageNumber);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
}, [currDocPageNumber]);
|
}, [currDocPageNumber]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -324,11 +330,13 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
if (!currentPdf) return;
|
if (!currentPdf) return;
|
||||||
const seq = ++loadSeqRef.current;
|
const seq = ++loadSeqRef.current;
|
||||||
const pageNumber = currDocPageNumber;
|
const pageNumber = currDocPageNumber;
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
|
||||||
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
|
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
|
||||||
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
|
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
|
||||||
|
|
||||||
if (parseStatus !== 'ready' || !parsedDocument) {
|
if (parseStatus !== 'ready' || !parsedDocument) {
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
setTTSText('', { location: currDocPageNumber });
|
setTTSText('', { location: currDocPageNumber });
|
||||||
return;
|
return;
|
||||||
|
|
@ -401,7 +409,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (text !== currDocText || text === '') {
|
const shouldPreparePlayback = text === '' || text !== currDocText || lastPreparedPlaybackPageRef.current !== currDocPageNumber;
|
||||||
|
if (shouldPreparePlayback) {
|
||||||
setCurrDocText(text);
|
setCurrDocText(text);
|
||||||
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
|
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
|
||||||
setTTSText(text, {
|
setTTSText(text, {
|
||||||
|
|
@ -414,6 +423,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
|
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
lastPreparedPlaybackPageRef.current = currDocPageNumber;
|
||||||
|
setIsPlaybackReady(true);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||||
return;
|
return;
|
||||||
|
|
@ -469,6 +480,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setPdfDocument(undefined);
|
setPdfDocument(undefined);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setCurrDocId(id);
|
setCurrDocId(id);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
|
|
@ -546,6 +559,8 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
setParseStatus(forced.status);
|
setParseStatus(forced.status);
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
setActiveParseOpId(forced.opId ?? null);
|
setActiveParseOpId(forced.opId ?? null);
|
||||||
|
|
@ -575,6 +590,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setIsPlaybackReady(false);
|
||||||
setCurrDocPages(undefined);
|
setCurrDocPages(undefined);
|
||||||
setPdfDocument(undefined);
|
setPdfDocument(undefined);
|
||||||
setParsedDocument(null);
|
setParsedDocument(null);
|
||||||
|
|
@ -582,6 +598,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
setParseProgress(null);
|
setParseProgress(null);
|
||||||
setActiveParseOpId(null);
|
setActiveParseOpId(null);
|
||||||
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
|
||||||
|
lastPreparedPlaybackPageRef.current = null;
|
||||||
pageTextCacheRef.current.clear();
|
pageTextCacheRef.current.clear();
|
||||||
stop();
|
stop();
|
||||||
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
|
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
|
||||||
|
|
@ -681,6 +698,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
parsedDocument,
|
parsedDocument,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
parseProgress,
|
parseProgress,
|
||||||
|
|
@ -708,6 +726,7 @@ export function usePdfDocument(): PdfDocumentState {
|
||||||
currDocPages,
|
currDocPages,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
currDocText,
|
currDocText,
|
||||||
|
isPlaybackReady,
|
||||||
parsedDocument,
|
parsedDocument,
|
||||||
parseStatus,
|
parseStatus,
|
||||||
parseProgress,
|
parseProgress,
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import { isWorkerOperationStateStale, snapshotFromWorkerState } from '@/lib/serv
|
||||||
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
import { fetchWorkerOperationState } from '@/lib/server/compute/worker-op-state';
|
||||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
|
|
@ -67,7 +68,7 @@ async function toSnapshotState(row: ParseRow, preferredOpId?: string | null): Pr
|
||||||
const state = await healStaleDocumentParseState({
|
const state = await healStaleDocumentParseState({
|
||||||
documentId: row.id,
|
documentId: row.id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
state: parseDocumentParseState(row.parseState),
|
state: normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)),
|
||||||
});
|
});
|
||||||
const parseStatus = normalizeParseStatus(state.status);
|
const parseStatus = normalizeParseStatus(state.status);
|
||||||
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
|
const dbOpId = typeof state.opId === 'string' && state.opId.trim() ? state.opId.trim() : null;
|
||||||
|
|
@ -199,7 +200,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
|
|
||||||
let initialState = await toSnapshotState(row, requestedOpId);
|
let initialState = await toSnapshotState(row, requestedOpId);
|
||||||
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
|
if (!requestedOpId && !initialState.opId && initialState.snapshot.parseStatus !== 'ready' && initialState.snapshot.parseStatus !== 'failed') {
|
||||||
const state = parseDocumentParseState(row.parseState);
|
const state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
const created = await backfillPendingPdfParseOperation({
|
const created = await backfillPendingPdfParseOperation({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,10 @@ import {
|
||||||
putParsedDocumentBlob,
|
putParsedDocumentBlob,
|
||||||
} from '@/lib/server/documents/blobstore';
|
} from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
|
resolveParsedPdfParserVersion,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill';
|
import { backfillPendingPdfParseOperation } from '@/lib/server/documents/parse-state-backfill';
|
||||||
|
|
@ -156,13 +158,6 @@ async function finalizeFromWorkerState(input: {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
await writeParseRowState({
|
|
||||||
documentId: input.row.id,
|
|
||||||
userId: input.row.userId,
|
|
||||||
parseState: stringifyDocumentParseState(workerParseState),
|
|
||||||
parsedJsonKey,
|
|
||||||
});
|
|
||||||
|
|
||||||
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||||
let parsedDoc: ParsedPdfDocument | null = null;
|
let parsedDoc: ParsedPdfDocument | null = null;
|
||||||
try {
|
try {
|
||||||
|
|
@ -171,6 +166,16 @@ async function finalizeFromWorkerState(input: {
|
||||||
parsedDoc = null;
|
parsedDoc = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await writeParseRowState({
|
||||||
|
documentId: input.row.id,
|
||||||
|
userId: input.row.userId,
|
||||||
|
parseState: stringifyDocumentParseState({
|
||||||
|
...workerParseState,
|
||||||
|
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
|
||||||
|
}),
|
||||||
|
parsedJsonKey,
|
||||||
|
});
|
||||||
|
|
||||||
if (!hasAnyParsedBlocks(parsedDoc)) {
|
if (!hasAnyParsedBlocks(parsedDoc)) {
|
||||||
input.logger.warn({
|
input.logger.warn({
|
||||||
event: 'documents.parsed.no_blocks_in_output',
|
event: 'documents.parsed.no_blocks_in_output',
|
||||||
|
|
@ -246,7 +251,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let state = parseDocumentParseState(row.parseState);
|
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
state = await healStaleDocumentParseState({
|
state = await healStaleDocumentParseState({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
|
|
@ -409,7 +414,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
let state = parseDocumentParseState(row.parseState);
|
let state = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
state = await healStaleDocumentParseState({
|
state = await healStaleDocumentParseState({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
|
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
|
||||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||||
|
|
@ -142,7 +143,12 @@ export async function POST(req: NextRequest) {
|
||||||
size: pdfContent.length,
|
size: pdfContent.length,
|
||||||
lastModified,
|
lastModified,
|
||||||
filePath: id,
|
filePath: id,
|
||||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
}),
|
||||||
parsedJsonKey: null,
|
parsedJsonKey: null,
|
||||||
})
|
})
|
||||||
.onConflictDoUpdate({
|
.onConflictDoUpdate({
|
||||||
|
|
@ -153,7 +159,12 @@ export async function POST(req: NextRequest) {
|
||||||
size: pdfContent.length,
|
size: pdfContent.length,
|
||||||
lastModified,
|
lastModified,
|
||||||
filePath: id,
|
filePath: id,
|
||||||
parseState: stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }),
|
parseState: stringifyDocumentParseState({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
}),
|
||||||
parsedJsonKey: null,
|
parsedJsonKey: null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -17,12 +17,15 @@ import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/
|
||||||
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
|
||||||
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
@ -100,6 +103,9 @@ export async function POST(req: NextRequest) {
|
||||||
|
|
||||||
for (const doc of documentsData) {
|
for (const doc of documentsData) {
|
||||||
let headSize = doc.size;
|
let headSize = doc.size;
|
||||||
|
const reusableParsedPdf = doc.type === 'pdf'
|
||||||
|
? await findReusableParsedPdfResult(doc.id)
|
||||||
|
: null;
|
||||||
|
|
||||||
// Retry HEAD check to handle S3 read-after-write propagation delays.
|
// Retry HEAD check to handle S3 read-after-write propagation delays.
|
||||||
// The client uploads bytes directly to S3 via presigned URL, then
|
// The client uploads bytes directly to S3 via presigned URL, then
|
||||||
|
|
@ -146,9 +152,13 @@ export async function POST(req: NextRequest) {
|
||||||
lastModified: doc.lastModified,
|
lastModified: doc.lastModified,
|
||||||
filePath: doc.id,
|
filePath: doc.id,
|
||||||
parseState: doc.type === 'pdf'
|
parseState: doc.type === 'pdf'
|
||||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
? stringifyDocumentParseState(
|
||||||
|
reusableParsedPdf
|
||||||
|
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||||
|
: { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION },
|
||||||
|
)
|
||||||
: null,
|
: null,
|
||||||
parsedJsonKey: null,
|
parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null,
|
||||||
})
|
})
|
||||||
.onConflictDoUpdate({
|
.onConflictDoUpdate({
|
||||||
target: [documents.id, documents.userId],
|
target: [documents.id, documents.userId],
|
||||||
|
|
@ -159,9 +169,13 @@ export async function POST(req: NextRequest) {
|
||||||
lastModified: doc.lastModified,
|
lastModified: doc.lastModified,
|
||||||
filePath: doc.id,
|
filePath: doc.id,
|
||||||
parseState: doc.type === 'pdf'
|
parseState: doc.type === 'pdf'
|
||||||
? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() })
|
? stringifyDocumentParseState(
|
||||||
|
reusableParsedPdf
|
||||||
|
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||||
|
: { status: 'pending', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION },
|
||||||
|
)
|
||||||
: null,
|
: null,
|
||||||
parsedJsonKey: null,
|
parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -191,7 +205,7 @@ export async function POST(req: NextRequest) {
|
||||||
}, 'Failed to enqueue document preview');
|
}, 'Failed to enqueue document preview');
|
||||||
});
|
});
|
||||||
|
|
||||||
if (doc.type === 'pdf') {
|
if (doc.type === 'pdf' && !reusableParsedPdf) {
|
||||||
// Account for upload-driven parse load in the same ledger the explicit
|
// Account for upload-driven parse load in the same ledger the explicit
|
||||||
// re-parse limiter reads. We record (not reject) here so a legitimate
|
// re-parse limiter reads. We record (not reject) here so a legitimate
|
||||||
// bulk upload always parses; the recorded load still throttles
|
// bulk upload always parses; the recorded load still throttles
|
||||||
|
|
@ -260,13 +274,16 @@ export async function GET(req: NextRequest) {
|
||||||
|
|
||||||
const results: BaseDocument[] = rows.map((doc) => {
|
const results: BaseDocument[] = rows.map((doc) => {
|
||||||
const type = normalizeDocumentType(doc.type, doc.name);
|
const type = normalizeDocumentType(doc.type, doc.name);
|
||||||
|
const parseState = type === 'pdf'
|
||||||
|
? normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(doc.parseState))
|
||||||
|
: null;
|
||||||
return {
|
return {
|
||||||
id: doc.id,
|
id: doc.id,
|
||||||
name: doc.name,
|
name: doc.name,
|
||||||
size: Number(doc.size),
|
size: Number(doc.size),
|
||||||
lastModified: Number(doc.lastModified),
|
lastModified: Number(doc.lastModified),
|
||||||
type,
|
type,
|
||||||
parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null,
|
parseStatus: type === 'pdf' && parseState ? normalizeParseStatus(parseState.status) : null,
|
||||||
parsedJsonKey: doc.parsedJsonKey,
|
parsedJsonKey: doc.parsedJsonKey,
|
||||||
scope: 'user',
|
scope: 'user',
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,10 @@ import { SpeedControl } from '@/components/player/SpeedControl';
|
||||||
import { Navigator } from '@/components/player/Navigator';
|
import { Navigator } from '@/components/player/Navigator';
|
||||||
import { IconButton } from '@/components/ui';
|
import { IconButton } from '@/components/ui';
|
||||||
|
|
||||||
export default function TTSPlayer({ currentPage, numPages }: {
|
export default function TTSPlayer({ currentPage, numPages, isPlaybackReady = true }: {
|
||||||
currentPage?: number;
|
currentPage?: number;
|
||||||
numPages?: number | undefined;
|
numPages?: number | undefined;
|
||||||
|
isPlaybackReady?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const {
|
const {
|
||||||
isPlaying,
|
isPlaying,
|
||||||
|
|
@ -52,7 +53,7 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={skipBackward}
|
onClick={skipBackward}
|
||||||
aria-label="Skip backward"
|
aria-label="Skip backward"
|
||||||
disabled={isProcessing}
|
disabled={isProcessing || !isPlaybackReady}
|
||||||
className="relative"
|
className="relative"
|
||||||
>
|
>
|
||||||
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
|
{isProcessing ? <LoadingSpinner /> : <SkipBackwardIcon className="w-5 h-5" />}
|
||||||
|
|
@ -61,16 +62,18 @@ export default function TTSPlayer({ currentPage, numPages }: {
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={togglePlay}
|
onClick={togglePlay}
|
||||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||||
disabled={isProcessing && !isPlaying}
|
disabled={!isPlaying && (!isPlaybackReady || isProcessing)}
|
||||||
className="relative"
|
className="relative"
|
||||||
>
|
>
|
||||||
{isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />}
|
{!isPlaying && !isPlaybackReady
|
||||||
|
? <LoadingSpinner />
|
||||||
|
: (isPlaying ? <PauseIcon className="w-5 h-5" /> : <PlayIcon className="w-5 h-5" />)}
|
||||||
</IconButton>
|
</IconButton>
|
||||||
|
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={skipForward}
|
onClick={skipForward}
|
||||||
aria-label="Skip forward"
|
aria-label="Skip forward"
|
||||||
disabled={isProcessing}
|
disabled={isProcessing || !isPlaybackReady}
|
||||||
className="relative"
|
className="relative"
|
||||||
>
|
>
|
||||||
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}
|
{isProcessing ? <LoadingSpinner /> : <SkipForwardIcon className="w-5 h-5" />}
|
||||||
|
|
|
||||||
|
|
@ -624,6 +624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
||||||
const isPlayingRef = useRef(false);
|
const isPlayingRef = useRef(false);
|
||||||
const pauseEpochRef = useRef(0);
|
const pauseEpochRef = useRef(0);
|
||||||
|
const pendingPlaybackStartRef = useRef(false);
|
||||||
const sentencesRef = useRef<string[]>([]);
|
const sentencesRef = useRef<string[]>([]);
|
||||||
const currentIndexRef = useRef(0);
|
const currentIndexRef = useRef(0);
|
||||||
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
|
const plannedSegmentsByLocationRef = useRef<Map<string, CanonicalTtsSegment[]>>(new Map());
|
||||||
|
|
@ -939,6 +940,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const recordManualPause = useCallback(() => {
|
const recordManualPause = useCallback(() => {
|
||||||
// Cancel any queued auto-resume intent and mark an explicit user pause.
|
// Cancel any queued auto-resume intent and mark an explicit user pause.
|
||||||
resumeAfterLocationChangeRef.current = false;
|
resumeAfterLocationChangeRef.current = false;
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
pauseEpochRef.current += 1;
|
pauseEpochRef.current += 1;
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
|
@ -947,6 +949,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
* Used for external control of playback state
|
* Used for external control of playback state
|
||||||
*/
|
*/
|
||||||
const pause = useCallback(() => {
|
const pause = useCallback(() => {
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
recordManualPause();
|
recordManualPause();
|
||||||
clearPendingEpubJump();
|
clearPendingEpubJump();
|
||||||
abortPendingTtsRequests(true);
|
abortPendingTtsRequests(true);
|
||||||
|
|
@ -1248,10 +1251,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const shouldPause = normalizedOptions.shouldPause ?? false;
|
const shouldPause = normalizedOptions.shouldPause ?? false;
|
||||||
const pauseEpochAtStart = pauseEpochRef.current;
|
const pauseEpochAtStart = pauseEpochRef.current;
|
||||||
const pendingAutoResume = resumeAfterLocationChangeRef.current;
|
const pendingAutoResume = resumeAfterLocationChangeRef.current;
|
||||||
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume);
|
const pendingPlaybackStart = pendingPlaybackStartRef.current;
|
||||||
if (shouldPause || pendingAutoResume) {
|
const shouldResumePlayback = !shouldPause && (isPlaying || pendingAutoResume || pendingPlaybackStart);
|
||||||
resumeAfterLocationChangeRef.current = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keep track of previous state and pause playback
|
// Keep track of previous state and pause playback
|
||||||
invalidatePlaybackRun();
|
invalidatePlaybackRun();
|
||||||
|
|
@ -1262,10 +1263,23 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
try {
|
try {
|
||||||
if (newSentences.length === 0) {
|
if (newSentences.length === 0) {
|
||||||
console.warn('No sentences found in text');
|
console.warn('No sentences found in text');
|
||||||
|
if (shouldPause || pendingAutoResume) {
|
||||||
|
resumeAfterLocationChangeRef.current = false;
|
||||||
|
}
|
||||||
|
if (pendingPlaybackStart) {
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
|
}
|
||||||
setIsProcessing(false);
|
setIsProcessing(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (shouldPause || pendingAutoResume) {
|
||||||
|
resumeAfterLocationChangeRef.current = false;
|
||||||
|
}
|
||||||
|
if (pendingPlaybackStart) {
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
|
}
|
||||||
|
|
||||||
if (!isEPUB && typeof resolvedLocation === 'number') {
|
if (!isEPUB && typeof resolvedLocation === 'number') {
|
||||||
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
|
const firstFingerprint = normalizeBlockFingerprint(newSentences[0] || '');
|
||||||
if (firstFingerprint) {
|
if (firstFingerprint) {
|
||||||
|
|
@ -1388,6 +1402,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
// Ensure audio is unlocked while we're still in the click/tap handler.
|
// Ensure audio is unlocked while we're still in the click/tap handler.
|
||||||
unlockPlaybackOnUserGesture();
|
unlockPlaybackOnUserGesture();
|
||||||
|
|
||||||
|
const hasPreparedSentence = Boolean(playbackSegments[currentIndex]?.text ?? sentences[currentIndex]);
|
||||||
|
if (!activeHowl && !hasPreparedSentence) {
|
||||||
|
pendingPlaybackStartRef.current = true;
|
||||||
|
setIsProcessing(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Resume current sentence if we already have a paused Howl.
|
// Resume current sentence if we already have a paused Howl.
|
||||||
if (activeHowl) {
|
if (activeHowl) {
|
||||||
applyPlaybackRateToHowl(activeHowl);
|
applyPlaybackRateToHowl(activeHowl);
|
||||||
|
|
@ -1406,12 +1427,15 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
setIsPlaying(true);
|
setIsPlaying(true);
|
||||||
}, [
|
}, [
|
||||||
activeHowl,
|
activeHowl,
|
||||||
|
currentIndex,
|
||||||
applyPlaybackRateToHowl,
|
applyPlaybackRateToHowl,
|
||||||
isPlaying,
|
isPlaying,
|
||||||
pauseActiveHowl,
|
pauseActiveHowl,
|
||||||
|
playbackSegments,
|
||||||
recordManualPause,
|
recordManualPause,
|
||||||
clearPendingEpubJump,
|
clearPendingEpubJump,
|
||||||
abortPendingTtsRequests,
|
abortPendingTtsRequests,
|
||||||
|
sentences,
|
||||||
unlockPlaybackOnUserGesture,
|
unlockPlaybackOnUserGesture,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
@ -2878,6 +2902,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
abortAudio();
|
abortAudio();
|
||||||
clearWarmAudioCache();
|
clearWarmAudioCache();
|
||||||
playbackInFlightRef.current = false;
|
playbackInFlightRef.current = false;
|
||||||
|
pendingPlaybackStartRef.current = false;
|
||||||
pendingJumpTargetRef.current = null;
|
pendingJumpTargetRef.current = null;
|
||||||
clearPendingEpubJump();
|
clearPendingEpubJump();
|
||||||
bumpEpubPreloadGeneration();
|
bumpEpubPreloadGeneration();
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
import type { DocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
|
|
@ -42,6 +43,7 @@ export function documentParseStateFromWorkerState(
|
||||||
? parseProgress
|
? parseProgress
|
||||||
: null,
|
: null,
|
||||||
updatedAt: nowMs,
|
updatedAt: nowMs,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
|
...(typeof state.opId === 'string' && state.opId.trim() ? { opId: state.opId } : {}),
|
||||||
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
|
...(typeof state.jobId === 'string' && state.jobId.trim() ? { jobId: state.jobId } : {}),
|
||||||
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),
|
...(parseStatus === 'failed' && state.error?.message ? { error: state.error.message } : {}),
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { createHash, randomUUID } from 'node:crypto';
|
import { createHash, randomUUID } from 'node:crypto';
|
||||||
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
|
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
|
||||||
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
|
import { parseSseEventId, parseSsePayload } from '@openreader/compute-core';
|
||||||
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core';
|
import { getWorkerClientWaitTimeoutMs, PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||||
import type {
|
import type {
|
||||||
|
|
@ -190,6 +190,7 @@ export function buildPdfOpKey(input: PdfLayoutInput): string {
|
||||||
return [
|
return [
|
||||||
'pdf_layout',
|
'pdf_layout',
|
||||||
'v1',
|
'v1',
|
||||||
|
PDF_PARSER_VERSION,
|
||||||
input.documentId,
|
input.documentId,
|
||||||
input.namespace ?? '',
|
input.namespace ?? '',
|
||||||
input.documentObjectKey ?? '',
|
input.documentObjectKey ?? '',
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ export async function healStaleDocumentParseState(input: {
|
||||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||||
...(input.state.opId ? { opId: input.state.opId } : {}),
|
...(input.state.opId ? { opId: input.state.opId } : {}),
|
||||||
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
|
...(input.state.jobId ? { jobId: input.state.jobId } : {}),
|
||||||
|
...(input.state.parserVersion ? { parserVersion: input.state.parserVersion } : {}),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await db
|
await db
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
|
|
||||||
export interface DocumentParseState {
|
export interface DocumentParseState {
|
||||||
status: PdfParseStatus;
|
status: PdfParseStatus;
|
||||||
|
|
@ -7,6 +8,7 @@ export interface DocumentParseState {
|
||||||
error?: string | null;
|
error?: string | null;
|
||||||
opId?: string;
|
opId?: string;
|
||||||
jobId?: string;
|
jobId?: string;
|
||||||
|
parserVersion?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
|
export function isInProgressParseStatus(status: PdfParseStatus): status is 'pending' | 'running' {
|
||||||
|
|
@ -32,6 +34,31 @@ export function normalizeParseStatus(status: string | null | undefined): PdfPars
|
||||||
return 'pending';
|
return 'pending';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function hasCurrentPdfParserVersion(parserVersion: string | null | undefined): boolean {
|
||||||
|
return typeof parserVersion === 'string' && parserVersion.trim() === PDF_PARSER_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeDocumentParseStateForCurrentParserVersion(
|
||||||
|
state: DocumentParseState,
|
||||||
|
nowMs = Date.now(),
|
||||||
|
): DocumentParseState {
|
||||||
|
const status = normalizeParseStatus(state.status);
|
||||||
|
if (status === 'failed' || hasCurrentPdfParserVersion(state.parserVersion)) {
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: nowMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveParsedPdfParserVersion(parsed: ParsedPdfDocument | null | undefined): string {
|
||||||
|
const parserVersion = parsed?.parserVersion?.trim();
|
||||||
|
return parserVersion || PDF_PARSER_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
function normalizeProgress(progress: unknown): PdfParseProgress | null {
|
||||||
if (!progress || typeof progress !== 'object') return null;
|
if (!progress || typeof progress !== 'object') return null;
|
||||||
const rec = progress as Record<string, unknown>;
|
const rec = progress as Record<string, unknown>;
|
||||||
|
|
@ -63,6 +90,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
||||||
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
const error = typeof parsed.error === 'string' ? parsed.error : null;
|
||||||
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined;
|
||||||
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined;
|
||||||
|
const parserVersion = typeof parsed.parserVersion === 'string' ? parsed.parserVersion : undefined;
|
||||||
return {
|
return {
|
||||||
status,
|
status,
|
||||||
progress,
|
progress,
|
||||||
|
|
@ -70,6 +98,7 @@ export function parseDocumentParseState(value: string | null): DocumentParseStat
|
||||||
...(error ? { error } : {}),
|
...(error ? { error } : {}),
|
||||||
...(opId ? { opId } : {}),
|
...(opId ? { opId } : {}),
|
||||||
...(jobId ? { jobId } : {}),
|
...(jobId ? { jobId } : {}),
|
||||||
|
...(parserVersion ? { parserVersion } : {}),
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return { status: 'pending', progress: null };
|
return { status: 'pending', progress: null };
|
||||||
|
|
@ -84,5 +113,6 @@ export function stringifyDocumentParseState(state: DocumentParseState): string {
|
||||||
...(state.error ? { error: state.error } : {}),
|
...(state.error ? { error: state.error } : {}),
|
||||||
...(state.opId ? { opId: state.opId } : {}),
|
...(state.opId ? { opId: state.opId } : {}),
|
||||||
...(state.jobId ? { jobId: state.jobId } : {}),
|
...(state.jobId ? { jobId: state.jobId } : {}),
|
||||||
|
...(state.parserVersion ? { parserVersion: state.parserVersion } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
33
src/lib/server/documents/parsed-pdf-reuse.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
import { eq } from 'drizzle-orm';
|
||||||
|
import { db } from '@/db';
|
||||||
|
import { documents } from '@/db/schema';
|
||||||
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
|
parseDocumentParseState,
|
||||||
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
|
||||||
|
type ReadyParsedPdfResult = {
|
||||||
|
parsedJsonKey: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function findReusableParsedPdfResult(
|
||||||
|
documentId: string,
|
||||||
|
): Promise<ReadyParsedPdfResult | null> {
|
||||||
|
const rows = await db
|
||||||
|
.select({
|
||||||
|
parseState: documents.parseState,
|
||||||
|
parsedJsonKey: documents.parsedJsonKey,
|
||||||
|
})
|
||||||
|
.from(documents)
|
||||||
|
.where(eq(documents.id, documentId));
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const parsedJsonKey = row.parsedJsonKey?.trim();
|
||||||
|
if (!parsedJsonKey) continue;
|
||||||
|
const parseState = normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState));
|
||||||
|
if (parseState.status !== 'ready') continue;
|
||||||
|
return { parsedJsonKey };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import { and, eq, inArray, isNull } from 'drizzle-orm';
|
import { and, eq, inArray, isNull } from 'drizzle-orm';
|
||||||
import { db } from '@/db';
|
import { db } from '@/db';
|
||||||
import { documents } from '@/db/schema';
|
import { documents } from '@/db/schema';
|
||||||
import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
import { documentKey, getParsedDocumentBlobByKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||||
|
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||||
import { serverLogger } from '@/lib/server/logger';
|
import { serverLogger } from '@/lib/server/logger';
|
||||||
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
import { logDegraded, logServerError } from '@/lib/server/errors/logging';
|
||||||
import {
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
|
resolveParsedPdfParserVersion,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
type DocumentParseState,
|
type DocumentParseState,
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
|
@ -13,11 +16,13 @@ import { getCompute } from '@/lib/server/compute';
|
||||||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||||
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
|
import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy';
|
||||||
import type {
|
import type {
|
||||||
|
ParsedPdfDocument,
|
||||||
PdfLayoutJobBase,
|
PdfLayoutJobBase,
|
||||||
PdfLayoutJobResult,
|
PdfLayoutJobResult,
|
||||||
PdfLayoutProgress,
|
PdfLayoutProgress,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
} from '@openreader/compute-core/api-contracts';
|
} from '@openreader/compute-core/api-contracts';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
|
||||||
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
|
type UserPdfLayoutJobRequest = PdfLayoutJobBase & {
|
||||||
userId: string;
|
userId: string;
|
||||||
|
|
@ -41,7 +46,7 @@ function keyFor(input: UserPdfLayoutJobRequest): string {
|
||||||
if (forceToken) {
|
if (forceToken) {
|
||||||
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
|
return `force:${input.documentId}:${input.namespace || ''}:${forceToken}`;
|
||||||
}
|
}
|
||||||
return `shared:${input.documentId}:${input.namespace || ''}`;
|
return `shared:${input.documentId}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sleep(ms: number): Promise<void> {
|
function sleep(ms: number): Promise<void> {
|
||||||
|
|
@ -84,7 +89,7 @@ async function loadScopedRows(input: UserPdfLayoutJobRequest): Promise<ParseRow[
|
||||||
|
|
||||||
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
|
function isReadyRow(row: ParseRow): row is ParseRow & { parsedJsonKey: string } {
|
||||||
if (!row.parsedJsonKey) return false;
|
if (!row.parsedJsonKey) return false;
|
||||||
return parseDocumentParseState(row.parseState).status === 'ready';
|
return normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)).status === 'ready';
|
||||||
}
|
}
|
||||||
|
|
||||||
function userIdsFromRows(rows: ParseRow[]): string[] {
|
function userIdsFromRows(rows: ParseRow[]): string[] {
|
||||||
|
|
@ -95,6 +100,20 @@ function parseStateMatchCondition(expected: string | null) {
|
||||||
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
|
return expected === null ? isNull(documents.parseState) : eq(documents.parseState, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadParsedDocumentForResult(
|
||||||
|
parsed: ParsedPdfDocument | undefined,
|
||||||
|
parsedJsonKey: string,
|
||||||
|
): Promise<ParsedPdfDocument | null> {
|
||||||
|
if (parsed) return parsed;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const json = await getParsedDocumentBlobByKey(parsedJsonKey);
|
||||||
|
return JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function updateParseStateForUsers(input: {
|
async function updateParseStateForUsers(input: {
|
||||||
documentId: string;
|
documentId: string;
|
||||||
userIds: string[];
|
userIds: string[];
|
||||||
|
|
@ -125,6 +144,23 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
|
const deadline = Date.now() + FOLLOWER_WAIT_TIMEOUT_MS;
|
||||||
|
|
||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
|
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||||
|
if (reusableParsed) {
|
||||||
|
const readyState: DocumentParseState = {
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
};
|
||||||
|
await updateParseStateForUsers({
|
||||||
|
documentId: input.documentId,
|
||||||
|
userIds: [input.userId],
|
||||||
|
parseState: stringifyDocumentParseState(readyState),
|
||||||
|
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const rows = await loadScopedRows(input);
|
const rows = await loadScopedRows(input);
|
||||||
const ready = rows.find(isReadyRow);
|
const ready = rows.find(isReadyRow);
|
||||||
if (ready) {
|
if (ready) {
|
||||||
|
|
@ -132,6 +168,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
};
|
};
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
|
@ -142,7 +179,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const statuses = rows.map((row) => parseDocumentParseState(row.parseState));
|
const statuses = rows.map((row) => normalizeDocumentParseStateForCurrentParserVersion(parseDocumentParseState(row.parseState)));
|
||||||
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
|
const hasInFlight = statuses.some((state) => state.status === 'pending' || state.status === 'running');
|
||||||
const failedState = statuses.find((state) => state.status === 'failed');
|
const failedState = statuses.find((state) => state.status === 'failed');
|
||||||
if (failedState && !hasInFlight) {
|
if (failedState && !hasInFlight) {
|
||||||
|
|
@ -151,6 +188,7 @@ async function syncCallerToSharedResult(input: UserPdfLayoutJobRequest): Promise
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
...(failedState.error ? { error: failedState.error } : {}),
|
...(failedState.error ? { error: failedState.error } : {}),
|
||||||
|
...(failedState.parserVersion ? { parserVersion: failedState.parserVersion } : {}),
|
||||||
};
|
};
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
|
@ -181,12 +219,30 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
// Non-force jobs can short-circuit by reusing existing ready output from
|
// Non-force jobs can short-circuit by reusing existing ready output from
|
||||||
// any row in the same document scope.
|
// any row in the same document scope.
|
||||||
if (!input.forceToken?.trim()) {
|
if (!input.forceToken?.trim()) {
|
||||||
|
const reusableParsed = await findReusableParsedPdfResult(input.documentId);
|
||||||
|
if (reusableParsed) {
|
||||||
|
const readyState: DocumentParseState = {
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
};
|
||||||
|
await updateParseStateForUsers({
|
||||||
|
documentId: input.documentId,
|
||||||
|
userIds: [input.userId],
|
||||||
|
parseState: stringifyDocumentParseState(readyState),
|
||||||
|
parsedJsonKey: reusableParsed.parsedJsonKey,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const ready = scopedRows.find(isReadyRow);
|
const ready = scopedRows.find(isReadyRow);
|
||||||
if (ready) {
|
if (ready) {
|
||||||
const readyState: DocumentParseState = {
|
const readyState: DocumentParseState = {
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
};
|
};
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
documentId: input.documentId,
|
documentId: input.documentId,
|
||||||
|
|
@ -205,6 +261,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
status: 'running',
|
status: 'running',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
};
|
};
|
||||||
const runningState = stringifyDocumentParseState(runningStateData);
|
const runningState = stringifyDocumentParseState(runningStateData);
|
||||||
|
|
||||||
|
|
@ -279,6 +336,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
status: mappedStatus,
|
status: mappedStatus,
|
||||||
progress: snapshot.progress ?? null,
|
progress: snapshot.progress ?? null,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
...(activeOpId ? { opId: activeOpId } : {}),
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
|
|
@ -305,6 +363,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
phase: progress.phase,
|
phase: progress.phase,
|
||||||
},
|
},
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
...(activeOpId ? { opId: activeOpId } : {}),
|
...(activeOpId ? { opId: activeOpId } : {}),
|
||||||
...(activeJobId ? { jobId: activeJobId } : {}),
|
...(activeJobId ? { jobId: activeJobId } : {}),
|
||||||
};
|
};
|
||||||
|
|
@ -327,12 +386,15 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
|
parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const parsedDoc = await loadParsedDocumentForResult(layout.parsed, parsedJsonKey);
|
||||||
|
|
||||||
const finalScopedRows = await loadScopedRows(input);
|
const finalScopedRows = await loadScopedRows(input);
|
||||||
const finalUserIds = userIdsFromRows(finalScopedRows);
|
const finalUserIds = userIdsFromRows(finalScopedRows);
|
||||||
const readyState: DocumentParseState = {
|
const readyState: DocumentParseState = {
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
|
parserVersion: resolveParsedPdfParserVersion(parsedDoc),
|
||||||
};
|
};
|
||||||
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
|
const readyUserIds = finalUserIds.length > 0 ? finalUserIds : cohortUserIds;
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
|
|
@ -385,6 +447,7 @@ export async function parsePdfJob(input: UserPdfLayoutJobRequest): Promise<void>
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
error: message,
|
error: message,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
};
|
};
|
||||||
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
|
const failedUserIds = scopedUserIds.length > 0 ? scopedUserIds : [input.userId];
|
||||||
await updateParseStateForUsers({
|
await updateParseStateForUsers({
|
||||||
|
|
|
||||||
|
|
@ -188,10 +188,11 @@ async function dismissOnboardingModals(page: Page): Promise<void> {
|
||||||
* Wait for the play button to be clickable and click it
|
* Wait for the play button to be clickable and click it
|
||||||
*/
|
*/
|
||||||
export async function waitAndClickPlay(page: Page) {
|
export async function waitAndClickPlay(page: Page) {
|
||||||
// Wait for play button selector without disabled attribute
|
const playButton = page.getByRole('button', { name: 'Play' });
|
||||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();
|
await expect(playButton).toBeVisible();
|
||||||
|
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||||
// Play the TTS by clicking the button
|
// Play the TTS by clicking the button
|
||||||
await page.getByRole('button', { name: 'Play' }).click();
|
await playButton.click();
|
||||||
// Use resilient processing transition helper (tolerates fast completion)
|
// Use resilient processing transition helper (tolerates fast completion)
|
||||||
await expectProcessingTransition(page);
|
await expectProcessingTransition(page);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -115,10 +115,12 @@ test.describe('EPUB resize pauses TTS', () => {
|
||||||
await page.waitForTimeout(750);
|
await page.waitForTimeout(750);
|
||||||
|
|
||||||
// After resize, playback should have paused (Play button visible)
|
// After resize, playback should have paused (Play button visible)
|
||||||
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });
|
const playButton = page.getByRole('button', { name: 'Play' });
|
||||||
|
await expect(playButton).toBeVisible({ timeout: 15000 });
|
||||||
|
await expect(playButton).toBeEnabled({ timeout: 15000 });
|
||||||
|
|
||||||
// Resume playback and ensure processing -> playing
|
// Resume playback and ensure processing -> playing
|
||||||
await page.getByRole('button', { name: 'Play' }).click();
|
await playButton.click();
|
||||||
await expectProcessingTransition(page);
|
await expectProcessingTransition(page);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
13
tests/unit/pdf-op-key.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
import { describe, expect, test } from 'vitest';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import { buildPdfOpKey } from '../../src/lib/server/compute/worker';
|
||||||
|
|
||||||
|
describe('pdf worker op key', () => {
|
||||||
|
test('includes parser version in the reuse boundary', () => {
|
||||||
|
expect(buildPdfOpKey({
|
||||||
|
documentId: 'doc-1',
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: 'documents/doc-1.pdf',
|
||||||
|
})).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`);
|
||||||
|
});
|
||||||
|
});
|
||||||
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
84
tests/unit/pdf-parse-state.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
import { describe, expect, test } from 'vitest';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
|
import {
|
||||||
|
normalizeDocumentParseStateForCurrentParserVersion,
|
||||||
|
parseDocumentParseState,
|
||||||
|
stringifyDocumentParseState,
|
||||||
|
} from '../../src/lib/server/documents/parse-state';
|
||||||
|
|
||||||
|
describe('document parse state parser-version handling', () => {
|
||||||
|
test('preserves parserVersion through stringify/parse', () => {
|
||||||
|
const state = parseDocumentParseState(stringifyDocumentParseState({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1234,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
}));
|
||||||
|
|
||||||
|
expect(state).toEqual({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1234,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('invalidates ready and inflight states from older parser versions', () => {
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
opId: 'op-old',
|
||||||
|
}, 2222)).toEqual({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 2222,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'running',
|
||||||
|
progress: {
|
||||||
|
totalPages: 10,
|
||||||
|
pagesParsed: 3,
|
||||||
|
currentPage: 4,
|
||||||
|
phase: 'infer',
|
||||||
|
},
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
opId: 'op-old',
|
||||||
|
}, 3333)).toEqual({
|
||||||
|
status: 'pending',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 3333,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('keeps failed states and current-version ready states intact', () => {
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'failed',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
error: 'no text layer',
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
})).toEqual({
|
||||||
|
status: 'failed',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
error: 'no text layer',
|
||||||
|
parserVersion: 'old-parser',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(normalizeDocumentParseStateForCurrentParserVersion({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
})).toEqual({
|
||||||
|
status: 'ready',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: 1111,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
|
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||||
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts';
|
||||||
import {
|
import {
|
||||||
documentParseStateFromWorkerState,
|
documentParseStateFromWorkerState,
|
||||||
|
|
@ -11,7 +12,7 @@ function makeWorkerState(
|
||||||
): WorkerOperationState<PdfLayoutJobResult> {
|
): WorkerOperationState<PdfLayoutJobResult> {
|
||||||
return {
|
return {
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
opKey: 'pdf_layout|v1|doc-1',
|
opKey: `pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1`,
|
||||||
kind: 'pdf_layout',
|
kind: 'pdf_layout',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
status: 'queued',
|
status: 'queued',
|
||||||
|
|
@ -41,6 +42,7 @@ describe('worker parse state mapping', () => {
|
||||||
status: 'pending',
|
status: 'pending',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: 1234,
|
updatedAt: 1234,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
});
|
});
|
||||||
|
|
@ -66,6 +68,7 @@ describe('worker parse state mapping', () => {
|
||||||
phase: 'infer',
|
phase: 'infer',
|
||||||
},
|
},
|
||||||
updatedAt: 5678,
|
updatedAt: 5678,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
});
|
});
|
||||||
|
|
@ -84,6 +87,7 @@ describe('worker parse state mapping', () => {
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
progress: null,
|
progress: null,
|
||||||
updatedAt: 9999,
|
updatedAt: 9999,
|
||||||
|
parserVersion: PDF_PARSER_VERSION,
|
||||||
opId: 'op-123',
|
opId: 'op-123',
|
||||||
jobId: 'job-123',
|
jobId: 'job-123',
|
||||||
error: 'layout model crashed',
|
error: 'layout model crashed',
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue