refactor(types,api): centralize PDF and TTS type exports to compute-core/types
Move and re-export all PDF parsing and TTS-related types from @openreader/compute-core/types, eliminating local type duplication. Update imports throughout the codebase to use the new centralized type module. Remove obsolete worker-contract file and update tsconfig paths and compute-core exports for the new types entry point. Refactor API routes and job logic to use the new type imports and shared parse-state healing utility. This streamlines type management and improves consistency between compute and app layers.
This commit is contained in:
parent
09e8898683
commit
b6be71e3b1
13 changed files with 116 additions and 162 deletions
|
|
@ -13,6 +13,7 @@
|
||||||
},
|
},
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./src/index.ts",
|
".": "./src/index.ts",
|
||||||
"./local-runtime": "./src/local-runtime.ts"
|
"./local-runtime": "./src/local-runtime.ts",
|
||||||
|
"./types": "./src/types/index.ts"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
35
compute/core/src/types/index.ts
Normal file
35
compute/core/src/types/index.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
export type {
|
||||||
|
TTSAudioBuffer,
|
||||||
|
TTSAudioBytes,
|
||||||
|
TTSSentenceAlignment,
|
||||||
|
TTSSentenceWord,
|
||||||
|
} from './tts';
|
||||||
|
|
||||||
|
export type {
|
||||||
|
ParsedPdfBlock,
|
||||||
|
ParsedPdfBlockFragment,
|
||||||
|
ParsedPdfBlockKind,
|
||||||
|
ParsedPdfDocument,
|
||||||
|
ParsedPdfPage,
|
||||||
|
PdfParsePhase,
|
||||||
|
PdfParseProgress,
|
||||||
|
PdfParseStatus,
|
||||||
|
} from './parsed-pdf';
|
||||||
|
|
||||||
|
export type {
|
||||||
|
PdfLayoutJobRequest,
|
||||||
|
PdfLayoutJobResult,
|
||||||
|
PdfLayoutProgress,
|
||||||
|
PdfLayoutProgressPhase,
|
||||||
|
PdfLayoutOperationRequest,
|
||||||
|
WhisperAlignJobRequest,
|
||||||
|
WhisperAlignJobResult,
|
||||||
|
WhisperAlignOperationRequest,
|
||||||
|
WorkerJobErrorShape,
|
||||||
|
WorkerJobState,
|
||||||
|
WorkerJobStatusResponse,
|
||||||
|
WorkerJobTiming,
|
||||||
|
WorkerOperationKind,
|
||||||
|
WorkerOperationRequest,
|
||||||
|
WorkerOperationState,
|
||||||
|
} from '../contracts';
|
||||||
|
|
@ -51,3 +51,13 @@ export interface ParsedPdfDocument {
|
||||||
parsedAt: number;
|
parsedAt: number;
|
||||||
pages: ParsedPdfPage[];
|
pages: ParsedPdfPage[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed';
|
||||||
|
export type PdfParsePhase = 'infer' | 'merge';
|
||||||
|
|
||||||
|
export interface PdfParseProgress {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
phase: PdfParsePhase;
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,13 +8,8 @@ import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/li
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf';
|
||||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import { normalizeParseStatus, parseDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
isDocumentParseStateStale,
|
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
||||||
normalizeParseStatus,
|
|
||||||
parseDocumentParseState,
|
|
||||||
stringifyDocumentParseState,
|
|
||||||
} from '@/lib/server/documents/parse-state';
|
|
||||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
|
@ -42,32 +37,12 @@ function sleep(ms: number): Promise<void> {
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function maybeHealStaleParseState(row: ParseRow): Promise<ParseRow> {
|
|
||||||
const state = parseDocumentParseState(row.parseState);
|
|
||||||
const staleMs = getComputeOpStaleMs();
|
|
||||||
if (!isDocumentParseStateStale(state, staleMs)) return row;
|
|
||||||
|
|
||||||
const nextState = stringifyDocumentParseState({
|
|
||||||
status: 'failed',
|
|
||||||
progress: null,
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
|
||||||
});
|
|
||||||
|
|
||||||
await db
|
|
||||||
.update(documents)
|
|
||||||
.set({ parseState: nextState })
|
|
||||||
.where(and(eq(documents.id, row.id), eq(documents.userId, row.userId)));
|
|
||||||
|
|
||||||
return {
|
|
||||||
...row,
|
|
||||||
parseState: nextState,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function toSnapshot(row: ParseRow): Promise<ParsedSnapshot> {
|
async function toSnapshot(row: ParseRow): Promise<ParsedSnapshot> {
|
||||||
const healed = await maybeHealStaleParseState(row);
|
const state = await healStaleDocumentParseState({
|
||||||
const state = parseDocumentParseState(healed.parseState);
|
documentId: row.id,
|
||||||
|
userId: row.userId,
|
||||||
|
state: parseDocumentParseState(row.parseState),
|
||||||
|
});
|
||||||
const parseStatus = normalizeParseStatus(state.status);
|
const parseStatus = normalizeParseStatus(state.status);
|
||||||
return {
|
return {
|
||||||
parseStatus,
|
parseStatus,
|
||||||
|
|
|
||||||
|
|
@ -12,15 +12,14 @@ import {
|
||||||
} from '@/lib/server/documents/blobstore';
|
} from '@/lib/server/documents/blobstore';
|
||||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
|
||||||
import {
|
import {
|
||||||
isDocumentParseStateStale,
|
|
||||||
normalizeParseStatus,
|
normalizeParseStatus,
|
||||||
parseDocumentParseState,
|
parseDocumentParseState,
|
||||||
stringifyDocumentParseState,
|
stringifyDocumentParseState,
|
||||||
} from '@/lib/server/documents/parse-state';
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing';
|
||||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
|
||||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||||
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
|
||||||
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
|
@ -36,29 +35,6 @@ function hasAnyParsedBlocks(doc: ParsedPdfDocument | null): boolean {
|
||||||
return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0);
|
return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function healStaleInProgressParseState(input: {
|
|
||||||
documentId: string;
|
|
||||||
userId: string;
|
|
||||||
state: ReturnType<typeof parseDocumentParseState>;
|
|
||||||
}): Promise<ReturnType<typeof parseDocumentParseState>> {
|
|
||||||
const staleMs = getComputeOpStaleMs();
|
|
||||||
if (!isDocumentParseStateStale(input.state, staleMs)) return input.state;
|
|
||||||
|
|
||||||
const nextState = parseDocumentParseState(stringifyDocumentParseState({
|
|
||||||
status: 'failed',
|
|
||||||
progress: null,
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await db
|
|
||||||
.update(documents)
|
|
||||||
.set({ parseState: stringifyDocumentParseState(nextState) })
|
|
||||||
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
|
||||||
|
|
||||||
return nextState;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||||
try {
|
try {
|
||||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||||
|
|
@ -99,7 +75,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
}
|
}
|
||||||
|
|
||||||
let state = parseDocumentParseState(row.parseState);
|
let state = parseDocumentParseState(row.parseState);
|
||||||
state = await healStaleInProgressParseState({
|
state = await healStaleDocumentParseState({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
state,
|
state,
|
||||||
|
|
@ -216,7 +192,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
}
|
}
|
||||||
|
|
||||||
let state = parseDocumentParseState(row.parseState);
|
let state = parseDocumentParseState(row.parseState);
|
||||||
state = await healStaleInProgressParseState({
|
state = await healStaleDocumentParseState({
|
||||||
documentId: id,
|
documentId: id,
|
||||||
userId: row.userId,
|
userId: row.userId,
|
||||||
state,
|
state,
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import type {
|
||||||
TTSSentenceAlignment,
|
TTSSentenceAlignment,
|
||||||
ParsedPdfDocument,
|
ParsedPdfDocument,
|
||||||
PdfLayoutProgress,
|
PdfLayoutProgress,
|
||||||
} from '@openreader/compute-core';
|
} from '@openreader/compute-core/types';
|
||||||
|
|
||||||
export type ComputeMode = 'local' | 'worker';
|
export type ComputeMode = 'local' | 'worker';
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
export {
|
|
||||||
type PdfLayoutJobRequest,
|
|
||||||
type PdfLayoutJobResult,
|
|
||||||
type PdfLayoutProgress,
|
|
||||||
type PdfLayoutProgressPhase,
|
|
||||||
type PdfLayoutOperationRequest,
|
|
||||||
type WhisperAlignJobRequest,
|
|
||||||
type WhisperAlignJobResult,
|
|
||||||
type WhisperAlignOperationRequest,
|
|
||||||
type WorkerJobErrorShape,
|
|
||||||
type WorkerOperationKind,
|
|
||||||
type WorkerOperationRequest,
|
|
||||||
type WorkerOperationState,
|
|
||||||
type WorkerJobState,
|
|
||||||
type WorkerJobTiming,
|
|
||||||
type WorkerJobStatusResponse,
|
|
||||||
} from '@openreader/compute-core';
|
|
||||||
|
|
@ -7,7 +7,7 @@ import type {
|
||||||
WhisperAlignJobRequest,
|
WhisperAlignJobRequest,
|
||||||
WhisperAlignJobResult,
|
WhisperAlignJobResult,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
} from '@/lib/server/compute/worker-contract';
|
} from '@openreader/compute-core/types';
|
||||||
|
|
||||||
class WorkerHttpError extends Error {
|
class WorkerHttpError extends Error {
|
||||||
status: number;
|
status: number;
|
||||||
|
|
|
||||||
34
src/lib/server/documents/parse-state-healing.ts
Normal file
34
src/lib/server/documents/parse-state-healing.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
import { and, eq } from 'drizzle-orm';
|
||||||
|
import { getComputeOpStaleMs } from '@openreader/compute-core';
|
||||||
|
import { db } from '@/db';
|
||||||
|
import { documents } from '@/db/schema';
|
||||||
|
import {
|
||||||
|
isDocumentParseStateStale,
|
||||||
|
parseDocumentParseState,
|
||||||
|
stringifyDocumentParseState,
|
||||||
|
type DocumentParseState,
|
||||||
|
} from '@/lib/server/documents/parse-state';
|
||||||
|
|
||||||
|
export async function healStaleDocumentParseState(input: {
|
||||||
|
documentId: string;
|
||||||
|
userId: string;
|
||||||
|
state: DocumentParseState;
|
||||||
|
}): Promise<DocumentParseState> {
|
||||||
|
const staleMs = getComputeOpStaleMs();
|
||||||
|
if (!isDocumentParseStateStale(input.state, staleMs)) return input.state;
|
||||||
|
|
||||||
|
const nextState = parseDocumentParseState(stringifyDocumentParseState({
|
||||||
|
status: 'failed',
|
||||||
|
progress: null,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
error: `Parse state stale for more than ${staleMs}ms; marked failed for retry`,
|
||||||
|
}));
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(documents)
|
||||||
|
.set({ parseState: stringifyDocumentParseState(nextState) })
|
||||||
|
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
|
||||||
|
|
||||||
|
return nextState;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -5,7 +5,7 @@ import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobs
|
||||||
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||||
import { getCompute } from '@/lib/server/compute';
|
import { getCompute } from '@/lib/server/compute';
|
||||||
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||||
import type { PdfLayoutProgress } from '@openreader/compute-core';
|
import type { PdfLayoutProgress } from '@openreader/compute-core/types';
|
||||||
|
|
||||||
interface ParsePdfJobInput {
|
interface ParsePdfJobInput {
|
||||||
documentId: string;
|
documentId: string;
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,12 @@
|
||||||
export type ParsedPdfBlockKind =
|
import type { ParsedPdfBlockKind } from '@openreader/compute-core/types';
|
||||||
| 'abstract'
|
|
||||||
| 'algorithm'
|
export {
|
||||||
| 'aside_text'
|
type ParsedPdfBlock,
|
||||||
| 'chart'
|
type ParsedPdfBlockFragment,
|
||||||
| 'content'
|
type ParsedPdfBlockKind,
|
||||||
| 'formula'
|
type ParsedPdfDocument,
|
||||||
| 'doc_title'
|
type ParsedPdfPage,
|
||||||
| 'figure_title'
|
} from '@openreader/compute-core/types';
|
||||||
| 'footer'
|
|
||||||
| 'footnote'
|
|
||||||
| 'formula_number'
|
|
||||||
| 'header'
|
|
||||||
| 'image'
|
|
||||||
| 'number'
|
|
||||||
| 'paragraph_title'
|
|
||||||
| 'reference'
|
|
||||||
| 'reference_content'
|
|
||||||
| 'seal'
|
|
||||||
| 'table'
|
|
||||||
| 'text'
|
|
||||||
| 'vision_footnote';
|
|
||||||
|
|
||||||
export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [
|
export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [
|
||||||
'abstract',
|
'abstract',
|
||||||
|
|
@ -45,43 +32,8 @@ export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [
|
||||||
'vision_footnote',
|
'vision_footnote',
|
||||||
];
|
];
|
||||||
|
|
||||||
export interface ParsedPdfBlockFragment {
|
export type {
|
||||||
page: number;
|
PdfParsePhase,
|
||||||
bbox: [number, number, number, number];
|
PdfParseProgress,
|
||||||
text: string;
|
PdfParseStatus,
|
||||||
readingOrder: number;
|
} from '@openreader/compute-core/types';
|
||||||
modelConfidence?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ParsedPdfBlock {
|
|
||||||
id: string;
|
|
||||||
kind: ParsedPdfBlockKind;
|
|
||||||
fragments: ParsedPdfBlockFragment[];
|
|
||||||
text: string;
|
|
||||||
parentSectionId?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ParsedPdfPage {
|
|
||||||
pageNumber: number;
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
blocks: ParsedPdfBlock[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ParsedPdfDocument {
|
|
||||||
schemaVersion: 1;
|
|
||||||
documentId: string;
|
|
||||||
parserVersion: string;
|
|
||||||
parsedAt: number;
|
|
||||||
pages: ParsedPdfPage[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed';
|
|
||||||
export type PdfParsePhase = 'infer' | 'merge';
|
|
||||||
|
|
||||||
export interface PdfParseProgress {
|
|
||||||
totalPages: number;
|
|
||||||
pagesParsed: number;
|
|
||||||
currentPage?: number;
|
|
||||||
phase: PdfParsePhase;
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,13 @@
|
||||||
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
|
import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan';
|
||||||
|
export type {
|
||||||
|
TTSAudioBuffer,
|
||||||
|
TTSAudioBytes,
|
||||||
|
TTSSentenceAlignment,
|
||||||
|
TTSSentenceWord,
|
||||||
|
} from '@openreader/compute-core/types';
|
||||||
|
|
||||||
export type TTSLocation = string | number;
|
export type TTSLocation = string | number;
|
||||||
|
|
||||||
// Core audio representations used across TTS and audiobook flows
|
|
||||||
export type TTSAudioBuffer = ArrayBuffer;
|
|
||||||
export type TTSAudioBytes = number[]; // JSON-safe representation (Array.from(new Uint8Array(buffer)))
|
|
||||||
|
|
||||||
// Core playback state exposed by the TTS context
|
// Core playback state exposed by the TTS context
|
||||||
export interface TTSPlaybackState {
|
export interface TTSPlaybackState {
|
||||||
isPlaying: boolean;
|
isPlaying: boolean;
|
||||||
|
|
@ -24,21 +26,6 @@ export interface TTSPageTurnEstimate {
|
||||||
fraction: number;
|
fraction: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Word-level alignment within a single spoken sentence/block
|
|
||||||
export interface TTSSentenceWord {
|
|
||||||
text: string;
|
|
||||||
startSec: number;
|
|
||||||
endSec: number;
|
|
||||||
charStart: number;
|
|
||||||
charEnd: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Alignment metadata for a single TTS sentence/block
|
|
||||||
export interface TTSSentenceAlignment {
|
|
||||||
sentence: string;
|
|
||||||
sentenceIndex: number;
|
|
||||||
words: TTSSentenceWord[];
|
|
||||||
}
|
|
||||||
|
|
||||||
interface EpubRenderedLocationWalkItem {
|
interface EpubRenderedLocationWalkItem {
|
||||||
/** Page-start CFI from the rendition — best-effort jump hint only. */
|
/** Page-start CFI from the rendition — best-effort jump hint only. */
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,8 @@
|
||||||
"paths": {
|
"paths": {
|
||||||
"@/*": ["./src/*"],
|
"@/*": ["./src/*"],
|
||||||
"@openreader/compute-core": ["./compute/core/src/index.ts"],
|
"@openreader/compute-core": ["./compute/core/src/index.ts"],
|
||||||
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"]
|
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"],
|
||||||
|
"@openreader/compute-core/types": ["./compute/core/src/types/index.ts"]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue