refactor(types,api): centralize PDF and TTS type exports to compute-core/types
Move and re-export all PDF parsing and TTS-related types from @openreader/compute-core/types, eliminating local type duplication. Update imports throughout the codebase to use the new centralized type module. Remove obsolete worker-contract file and update tsconfig paths and compute-core exports for the new types entry point. Refactor API routes and job logic to use the new type imports and shared parse-state healing utility. This streamlines type management and improves consistency between compute and app layers.
This commit is contained in:
parent
09e8898683
commit
b6be71e3b1
13 changed files with 116 additions and 162 deletions
|
|
@ -13,6 +13,7 @@
|
|||
},
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./local-runtime": "./src/local-runtime.ts"
|
||||
"./local-runtime": "./src/local-runtime.ts",
|
||||
"./types": "./src/types/index.ts"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
35
compute/core/src/types/index.ts
Normal file
35
compute/core/src/types/index.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
export type {
|
||||
TTSAudioBuffer,
|
||||
TTSAudioBytes,
|
||||
TTSSentenceAlignment,
|
||||
TTSSentenceWord,
|
||||
} from './tts';
|
||||
|
||||
export type {
|
||||
ParsedPdfBlock,
|
||||
ParsedPdfBlockFragment,
|
||||
ParsedPdfBlockKind,
|
||||
ParsedPdfDocument,
|
||||
ParsedPdfPage,
|
||||
PdfParsePhase,
|
||||
PdfParseProgress,
|
||||
PdfParseStatus,
|
||||
} from './parsed-pdf';
|
||||
|
||||
export type {
|
||||
PdfLayoutJobRequest,
|
||||
PdfLayoutJobResult,
|
||||
PdfLayoutProgress,
|
||||
PdfLayoutProgressPhase,
|
||||
PdfLayoutOperationRequest,
|
||||
WhisperAlignJobRequest,
|
||||
WhisperAlignJobResult,
|
||||
WhisperAlignOperationRequest,
|
||||
WorkerJobErrorShape,
|
||||
WorkerJobState,
|
||||
WorkerJobStatusResponse,
|
||||
WorkerJobTiming,
|
||||
WorkerOperationKind,
|
||||
WorkerOperationRequest,
|
||||
WorkerOperationState,
|
||||
} from '../contracts';
|
||||
|
|
@ -51,3 +51,13 @@ export interface ParsedPdfDocument {
|
|||
parsedAt: number;
|
||||
pages: ParsedPdfPage[];
|
||||
}
|
||||
|
||||
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed';
|
||||
export type PdfParsePhase = 'infer' | 'merge';
|
||||
|
||||
export interface PdfParseProgress {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
phase: PdfParsePhase;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,13 +8,8 @@ import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/li
|
|||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
isDocumentParseStateStale,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
||||
import { normalizeParseStatus, parseDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -42,32 +37,12 @@ function sleep(ms: number): Promise<void> {
|
|||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function maybeHealStaleParseState(row: ParseRow): Promise<ParseRow> {
|
||||
const state = parseDocumentParseState(row.parseState);
|
||||
const staleMs = getComputeOpStaleMs();
|
||||
if (!isDocumentParseStateStale(state, staleMs)) return row;
|
||||
|
||||
const nextState = stringifyDocumentParseState({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||
});
|
||||
|
||||
await db
|
||||
.update(documents)
|
||||
.set({ parseState: nextState })
|
||||
.where(and(eq(documents.id, row.id), eq(documents.userId, row.userId)));
|
||||
|
||||
return {
|
||||
...row,
|
||||
parseState: nextState,
|
||||
};
|
||||
}
|
||||
|
||||
async function toSnapshot(row: ParseRow): Promise<ParsedSnapshot> {
|
||||
const healed = await maybeHealStaleParseState(row);
|
||||
const state = parseDocumentParseState(healed.parseState);
|
||||
const state = await healStaleDocumentParseState({
|
||||
documentId: row.id,
|
||||
userId: row.userId,
|
||||
state: parseDocumentParseState(row.parseState),
|
||||
});
|
||||
const parseStatus = normalizeParseStatus(state.status);
|
||||
return {
|
||||
parseStatus,
|
||||
|
|
|
|||
|
|
@ -12,15 +12,14 @@ import {
|
|||
} from '@/lib/server/documents/blobstore';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
||||
import {
|
||||
isDocumentParseStateStale,
|
||||
normalizeParseStatus,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -36,29 +35,6 @@ function hasAnyParsedBlocks(doc: ParsedPdfDocument | null): boolean {
|
|||
return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0);
|
||||
}
|
||||
|
||||
async function healStaleInProgressParseState(input: {
|
||||
documentId: string;
|
||||
userId: string;
|
||||
state: ReturnType<typeof parseDocumentParseState>;
|
||||
}): Promise<ReturnType<typeof parseDocumentParseState>> {
|
||||
const staleMs = getComputeOpStaleMs();
|
||||
if (!isDocumentParseStateStale(input.state, staleMs)) return input.state;
|
||||
|
||||
const nextState = parseDocumentParseState(stringifyDocumentParseState({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||
}));
|
||||
|
||||
await db
|
||||
.update(documents)
|
||||
.set({ parseState: stringifyDocumentParseState(nextState) })
|
||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||
|
||||
return nextState;
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
|
@ -99,7 +75,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
}
|
||||
|
||||
let state = parseDocumentParseState(row.parseState);
|
||||
state = await healStaleInProgressParseState({
|
||||
state = await healStaleDocumentParseState({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
state,
|
||||
|
|
@ -216,7 +192,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
}
|
||||
|
||||
let state = parseDocumentParseState(row.parseState);
|
||||
state = await healStaleInProgressParseState({
|
||||
state = await healStaleDocumentParseState({
|
||||
documentId: id,
|
||||
userId: row.userId,
|
||||
state,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import type {
|
|||
TTSSentenceAlignment,
|
||||
ParsedPdfDocument,
|
||||
PdfLayoutProgress,
|
||||
} from '@openreader/compute-core';
|
||||
} from '@openreader/compute-core/types';
|
||||
|
||||
export type ComputeMode = 'local' | 'worker';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
export {
|
||||
type PdfLayoutJobRequest,
|
||||
type PdfLayoutJobResult,
|
||||
type PdfLayoutProgress,
|
||||
type PdfLayoutProgressPhase,
|
||||
type PdfLayoutOperationRequest,
|
||||
type WhisperAlignJobRequest,
|
||||
type WhisperAlignJobResult,
|
||||
type WhisperAlignOperationRequest,
|
||||
type WorkerJobErrorShape,
|
||||
type WorkerOperationKind,
|
||||
type WorkerOperationRequest,
|
||||
type WorkerOperationState,
|
||||
type WorkerJobState,
|
||||
type WorkerJobTiming,
|
||||
type WorkerJobStatusResponse,
|
||||
} from '@openreader/compute-core';
|
||||
|
|
@ -7,7 +7,7 @@ import type {
|
|||
WhisperAlignJobRequest,
|
||||
WhisperAlignJobResult,
|
||||
WorkerOperationState,
|
||||
} from '@/lib/server/compute/worker-contract';
|
||||
} from '@openreader/compute-core/types';
|
||||
|
||||
class WorkerHttpError extends Error {
|
||||
status: number;
|
||||
|
|
|
|||
34
src/lib/server/documents/parse-state-healing.ts
Normal file
34
src/lib/server/documents/parse-state-healing.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { and, eq } from 'drizzle-orm';
|
||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import {
|
||||
isDocumentParseStateStale,
|
||||
parseDocumentParseState,
|
||||
stringifyDocumentParseState,
|
||||
type DocumentParseState,
|
||||
} from '@/lib/server/documents/parse-state';
|
||||
|
||||
export async function healStaleDocumentParseState(input: {
|
||||
documentId: string;
|
||||
userId: string;
|
||||
state: DocumentParseState;
|
||||
}): Promise<DocumentParseState> {
|
||||
const staleMs = getComputeOpStaleMs();
|
||||
if (!isDocumentParseStateStale(input.state, staleMs)) return input.state;
|
||||
|
||||
const nextState = parseDocumentParseState(stringifyDocumentParseState({
|
||||
status: 'failed',
|
||||
progress: null,
|
||||
updatedAt: Date.now(),
|
||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||
}));
|
||||
|
||||
await db
|
||||
.update(documents)
|
||||
.set({ parseState: stringifyDocumentParseState(nextState) })
|
||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||
|
||||
return nextState;
|
||||
}
|
||||
|
||||
|
|
@ -5,7 +5,7 @@ import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobs
|
|||
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import { getCompute } from '@/lib/server/compute';
|
||||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
import type { PdfLayoutProgress } from '@openreader/compute-core';
|
||||
import type { PdfLayoutProgress } from '@openreader/compute-core/types';
|
||||
|
||||
interface ParsePdfJobInput {
|
||||
documentId: string;
|
||||
|
|
|
|||
|
|
@ -1,25 +1,12 @@
|
|||
export type ParsedPdfBlockKind =
|
||||
| 'abstract'
|
||||
| 'algorithm'
|
||||
| 'aside_text'
|
||||
| 'chart'
|
||||
| 'content'
|
||||
| 'formula'
|
||||
| 'doc_title'
|
||||
| 'figure_title'
|
||||
| 'footer'
|
||||
| 'footnote'
|
||||
| 'formula_number'
|
||||
| 'header'
|
||||
| 'image'
|
||||
| 'number'
|
||||
| 'paragraph_title'
|
||||
| 'reference'
|
||||
| 'reference_content'
|
||||
| 'seal'
|
||||
| 'table'
|
||||
| 'text'
|
||||
| 'vision_footnote';
|
||||
import type { ParsedPdfBlockKind } from '@openreader/compute-core/types';
|
||||
|
||||
export {
|
||||
type ParsedPdfBlock,
|
||||
type ParsedPdfBlockFragment,
|
||||
type ParsedPdfBlockKind,
|
||||
type ParsedPdfDocument,
|
||||
type ParsedPdfPage,
|
||||
} from '@openreader/compute-core/types';
|
||||
|
||||
export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [
|
||||
'abstract',
|
||||
|
|
@ -45,43 +32,8 @@ export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [
|
|||
'vision_footnote',
|
||||
];
|
||||
|
||||
export interface ParsedPdfBlockFragment {
|
||||
page: number;
|
||||
bbox: [number, number, number, number];
|
||||
text: string;
|
||||
readingOrder: number;
|
||||
modelConfidence?: number;
|
||||
}
|
||||
|
||||
export interface ParsedPdfBlock {
|
||||
id: string;
|
||||
kind: ParsedPdfBlockKind;
|
||||
fragments: ParsedPdfBlockFragment[];
|
||||
text: string;
|
||||
parentSectionId?: string;
|
||||
}
|
||||
|
||||
export interface ParsedPdfPage {
|
||||
pageNumber: number;
|
||||
width: number;
|
||||
height: number;
|
||||
blocks: ParsedPdfBlock[];
|
||||
}
|
||||
|
||||
export interface ParsedPdfDocument {
|
||||
schemaVersion: 1;
|
||||
documentId: string;
|
||||
parserVersion: string;
|
||||
parsedAt: number;
|
||||
pages: ParsedPdfPage[];
|
||||
}
|
||||
|
||||
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed';
|
||||
export type PdfParsePhase = 'infer' | 'merge';
|
||||
|
||||
export interface PdfParseProgress {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
phase: PdfParsePhase;
|
||||
}
|
||||
export type {
|
||||
PdfParsePhase,
|
||||
PdfParseProgress,
|
||||
PdfParseStatus,
|
||||
} from '@openreader/compute-core/types';
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
|
||||
export type {
|
||||
TTSAudioBuffer,
|
||||
TTSAudioBytes,
|
||||
TTSSentenceAlignment,
|
||||
TTSSentenceWord,
|
||||
} from '@openreader/compute-core/types';
|
||||
|
||||
export type TTSLocation = string | number;
|
||||
|
||||
// Core audio representations used across TTS and audiobook flows
|
||||
export type TTSAudioBuffer = ArrayBuffer;
|
||||
export type TTSAudioBytes = number[]; // JSON-safe representation (Array.from(new Uint8Array(buffer)))
|
||||
|
||||
// Core playback state exposed by the TTS context
|
||||
export interface TTSPlaybackState {
|
||||
isPlaying: boolean;
|
||||
|
|
@ -24,21 +26,6 @@ export interface TTSPageTurnEstimate {
|
|||
fraction: number;
|
||||
}
|
||||
|
||||
// Word-level alignment within a single spoken sentence/block
|
||||
export interface TTSSentenceWord {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
charStart: number;
|
||||
charEnd: number;
|
||||
}
|
||||
|
||||
// Alignment metadata for a single TTS sentence/block
|
||||
export interface TTSSentenceAlignment {
|
||||
sentence: string;
|
||||
sentenceIndex: number;
|
||||
words: TTSSentenceWord[];
|
||||
}
|
||||
|
||||
interface EpubRenderedLocationWalkItem {
|
||||
/** Page-start CFI from the rendition — best-effort jump hint only. */
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@
|
|||
"paths": {
|
||||
"@/*": ["./src/*"],
|
||||
"@openreader/compute-core": ["./compute/core/src/index.ts"],
|
||||
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"]
|
||||
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"],
|
||||
"@openreader/compute-core/types": ["./compute/core/src/types/index.ts"]
|
||||
}
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
|
|
|
|||
Loading…
Reference in a new issue