Refactor PDF parsing and rendering logic; migrate to compute-worker protocol
- Removed deprecated functions related to document parsing and blob storage in blobstore.ts. - Introduced new PDF rendering logic in pdf-preview-renderer.ts and pdf-preview-pdfjs-runtime.ts. - Updated previews-render.ts to utilize the new PDF rendering functions. - Refactored user-whisper-align-job.ts to use the compute-worker client for alignment requests. - Enhanced artifact.ts and operation.ts to validate parsed PDF artifacts and resolve current PDF parses. - Updated snapshot.ts to align with new worker operation types. - Adjusted runtime-config.ts to check for compute-worker availability. - Modified types in parsed-pdf.ts and tts.ts to reflect changes in the compute-worker protocol. - Added unit tests for PDF artifact validation and compute-worker client contract. - Removed obsolete pdf-op-key.vitest.spec.ts test file.
This commit is contained in:
parent
4bfd020125
commit
a56aaa2be9
122 changed files with 4285 additions and 1528 deletions
|
|
@ -1,5 +1,6 @@
|
|||
.env
|
||||
.env.*
|
||||
**/*.creds
|
||||
README.md
|
||||
.next
|
||||
node_modules
|
||||
|
|
|
|||
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -62,13 +62,13 @@ jobs:
|
|||
platform: linux/amd64
|
||||
runner: ubuntu-24.04
|
||||
context: .
|
||||
dockerfile: ./compute/worker/Dockerfile
|
||||
dockerfile: ./compute-worker/Dockerfile
|
||||
- image_target: compute-worker
|
||||
arch: arm64
|
||||
platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
context: .
|
||||
dockerfile: ./compute/worker/Dockerfile
|
||||
dockerfile: ./compute-worker/Dockerfile
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
|
|
|
|||
2
.github/workflows/vitest.yml
vendored
2
.github/workflows/vitest.yml
vendored
|
|
@ -25,3 +25,5 @@ jobs:
|
|||
run: pnpm migrate
|
||||
- name: Run Vitest suites
|
||||
run: pnpm test:unit
|
||||
- name: Verify compute worker contract and boundary
|
||||
run: pnpm compute:openapi:check && pnpm check:compute-boundary
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ WORKDIR /app
|
|||
|
||||
# Copy workspace manifests needed for dependency installation
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY compute/core/package.json ./compute/core/package.json
|
||||
COPY compute/worker/package.json ./compute/worker/package.json
|
||||
COPY compute-worker/package.json ./compute-worker/package.json
|
||||
COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json
|
||||
|
||||
# Install dependencies
|
||||
|
|
@ -88,7 +87,7 @@ COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses
|
|||
# Include SeaweedFS license text for the copied weed binary.
|
||||
COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt
|
||||
# Include static model notices for runtime-downloaded assets.
|
||||
COPY --from=app-builder /app/compute/core/src/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
|
||||
COPY --from=app-builder /app/compute-worker/src/compute/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
|
||||
|
||||
# Copy seaweedfs weed binary for optional embedded local S3.
|
||||
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
|
||||
|
|
@ -98,7 +97,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server
|
|||
RUN chmod +x /usr/local/bin/nats-server
|
||||
|
||||
# Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts.
|
||||
COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
||||
COPY --from=app-builder /app/compute-worker/src/compute/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
||||
|
||||
# Match the app's historical container port now that standalone server.js
|
||||
# is started directly instead of `next start -p 3003`.
|
||||
|
|
|
|||
|
|
@ -5,11 +5,8 @@ RUN npm install -g pnpm@11.1.2
|
|||
WORKDIR /workspace
|
||||
|
||||
COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY compute/worker/package.json compute/worker/package.json
|
||||
COPY compute/core/package.json compute/core/package.json
|
||||
|
||||
COPY compute/worker compute/worker
|
||||
COPY compute/core compute/core
|
||||
COPY compute-worker/package.json compute-worker/package.json
|
||||
COPY compute-worker compute-worker
|
||||
|
||||
RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker
|
||||
|
||||
|
|
@ -11,13 +11,14 @@ services:
|
|||
|
||||
compute-worker:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: compute/worker/Dockerfile
|
||||
context: ..
|
||||
dockerfile: compute-worker/Dockerfile
|
||||
container_name: openreader-compute-worker
|
||||
depends_on:
|
||||
- nats
|
||||
env_file:
|
||||
- ./.env
|
||||
- path: ./.env
|
||||
required: false
|
||||
environment:
|
||||
NATS_URL: ${NATS_URL:-nats://nats:4222}
|
||||
COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0}
|
||||
|
|
@ -32,16 +33,12 @@ services:
|
|||
- action: sync+restart
|
||||
path: .
|
||||
target: /workspace
|
||||
- action: rebuild
|
||||
path: ../../compute/core
|
||||
- action: rebuild
|
||||
path: ./package.json
|
||||
- action: rebuild
|
||||
path: ../../compute/core/package.json
|
||||
path: ../pnpm-lock.yaml
|
||||
- action: rebuild
|
||||
path: ../../pnpm-lock.yaml
|
||||
- action: rebuild
|
||||
path: ../../pnpm-workspace.yaml
|
||||
path: ../pnpm-workspace.yaml
|
||||
|
||||
volumes:
|
||||
nats-data:
|
||||
1766
compute-worker/openapi.json
Normal file
1766
compute-worker/openapi.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -5,15 +5,22 @@
|
|||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"start": "tsx src/server.ts"
|
||||
"start": "tsx src/server.ts",
|
||||
"openapi:generate": "tsx scripts/generate-openapi.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1061.0",
|
||||
"@fastify/swagger": "^9.0.0",
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"@napi-rs/canvas": "^0.1.100",
|
||||
"@nats-io/jetstream": "^3.4.0",
|
||||
"@nats-io/kv": "^3.4.0",
|
||||
"@nats-io/transport-node": "^3.4.0",
|
||||
"@openreader/compute-core": "workspace:*",
|
||||
"ffmpeg-static": "^5.3.0",
|
||||
"fastify": "^5.8.5",
|
||||
"jszip": "^3.10.1",
|
||||
"onnxruntime-node": "^1.26.0",
|
||||
"pdfjs-dist": "4.8.69",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"tsx": "^4.22.4",
|
||||
20
compute-worker/scripts/generate-openapi.ts
Normal file
20
compute-worker/scripts/generate-openapi.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { createComputeWorkerApp } from '../src/runtime';
|
||||
|
||||
process.env.COMPUTE_WORKER_TOKEN ||= 'openapi-generation-token';
|
||||
process.env.NATS_URL ||= 'nats://127.0.0.1:4222';
|
||||
|
||||
const outputPath = resolve(process.cwd(), 'openapi.json');
|
||||
const runtime = await createComputeWorkerApp({
|
||||
workerToken: process.env.COMPUTE_WORKER_TOKEN,
|
||||
disableWorkers: true,
|
||||
});
|
||||
|
||||
try {
|
||||
await runtime.app.ready();
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(runtime.app.swagger(), null, 2)}\n`, 'utf8');
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
56
compute-worker/src/api/operation-keys.ts
Normal file
56
compute-worker/src/api/operation-keys.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version';
|
||||
|
||||
function sha256Hex(input: string): string {
|
||||
return createHash('sha256').update(input).digest('hex');
|
||||
}
|
||||
|
||||
export function buildWhisperOperationKey(input: {
|
||||
audioObjectKey: string;
|
||||
text: string;
|
||||
lang?: string;
|
||||
cacheKey?: string;
|
||||
}): string {
|
||||
const cacheKey = input.cacheKey?.trim();
|
||||
if (cacheKey) {
|
||||
return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`;
|
||||
}
|
||||
return [
|
||||
'whisper_align',
|
||||
'v1',
|
||||
input.audioObjectKey,
|
||||
input.lang ?? '',
|
||||
sha256Hex(input.text),
|
||||
].join('|');
|
||||
}
|
||||
|
||||
export function buildPdfOperationKey(input: {
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
documentObjectKey: string;
|
||||
replaceToken?: string;
|
||||
}, parserVersion = PDF_PARSER_VERSION): string {
|
||||
return [
|
||||
'pdf_layout',
|
||||
'v1',
|
||||
parserVersion,
|
||||
input.documentId,
|
||||
input.namespace ?? '',
|
||||
input.documentObjectKey,
|
||||
input.replaceToken?.trim() || '',
|
||||
].join('|');
|
||||
}
|
||||
|
||||
export function pdfSubjectFromOperationKey(opKey: string): {
|
||||
kind: 'pdf_layout';
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
} | null {
|
||||
const [kind, version, , documentId, namespace] = opKey.split('|');
|
||||
if (kind !== 'pdf_layout' || version !== 'v1' || !documentId) return null;
|
||||
return {
|
||||
kind: 'pdf_layout',
|
||||
documentId,
|
||||
namespace: namespace || null,
|
||||
};
|
||||
}
|
||||
44
compute-worker/src/api/public-operation.ts
Normal file
44
compute-worker/src/api/public-operation.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import type { WorkerOperationState } from '../compute/api-contracts';
|
||||
import { pdfSubjectFromOperationKey } from './operation-keys';
|
||||
|
||||
export type PublicOperationSubject =
|
||||
| { kind: 'whisper_align' }
|
||||
| { kind: 'pdf_layout'; documentId: string; namespace: string | null };
|
||||
|
||||
export interface PublicOperation<Result = unknown> {
|
||||
opId: string;
|
||||
subject: PublicOperationSubject;
|
||||
status: WorkerOperationState['status'];
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: Result;
|
||||
error?: WorkerOperationState['error'];
|
||||
timing?: WorkerOperationState['timing'];
|
||||
progress?: WorkerOperationState['progress'];
|
||||
}
|
||||
|
||||
export interface PublicOperationEvent<Result = unknown> {
|
||||
eventId: number;
|
||||
snapshot: PublicOperation<Result>;
|
||||
}
|
||||
|
||||
export function toPublicOperation<Result>(
|
||||
state: WorkerOperationState<Result>,
|
||||
): PublicOperation<Result> {
|
||||
const subject = state.kind === 'pdf_layout'
|
||||
? (pdfSubjectFromOperationKey(state.opKey) ?? { kind: 'pdf_layout', documentId: '', namespace: null })
|
||||
: { kind: 'whisper_align' as const };
|
||||
return {
|
||||
opId: state.opId,
|
||||
subject,
|
||||
status: state.status,
|
||||
queuedAt: state.queuedAt,
|
||||
updatedAt: state.updatedAt,
|
||||
...(state.startedAt === undefined ? {} : { startedAt: state.startedAt }),
|
||||
...(state.result === undefined ? {} : { result: state.result }),
|
||||
...(state.error === undefined ? {} : { error: state.error }),
|
||||
...(state.timing === undefined ? {} : { timing: state.timing }),
|
||||
...(state.progress === undefined ? {} : { progress: state.progress }),
|
||||
};
|
||||
}
|
||||
134
compute-worker/src/api/schemas.ts
Normal file
134
compute-worker/src/api/schemas.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
const bboxSchema = z.tuple([z.number(), z.number(), z.number(), z.number()]);
|
||||
const parsedPdfBlockKindSchema = z.enum([
|
||||
'abstract', 'algorithm', 'aside_text', 'chart', 'content', 'formula',
|
||||
'doc_title', 'figure_title', 'footer', 'footnote', 'formula_number',
|
||||
'header', 'image', 'number', 'paragraph_title', 'reference',
|
||||
'reference_content', 'seal', 'table', 'text', 'vision_footnote',
|
||||
]);
|
||||
const documentIdSchema = z.string().trim().regex(/^[a-f0-9]{64}$/i);
|
||||
const namespaceSchema = z.string().trim().regex(/^[a-zA-Z0-9._-]{1,128}$/).nullable();
|
||||
|
||||
export const parsedPdfDocumentSchema = z.object({
|
||||
schemaVersion: z.literal(1),
|
||||
documentId: z.string(),
|
||||
parserVersion: z.string(),
|
||||
parsedAt: z.number(),
|
||||
pages: z.array(z.object({
|
||||
pageNumber: z.number(),
|
||||
width: z.number(),
|
||||
height: z.number(),
|
||||
blocks: z.array(z.object({
|
||||
id: z.string(),
|
||||
kind: parsedPdfBlockKindSchema,
|
||||
fragments: z.array(z.object({
|
||||
page: z.number(),
|
||||
bbox: bboxSchema,
|
||||
text: z.string(),
|
||||
readingOrder: z.number(),
|
||||
modelConfidence: z.number().optional(),
|
||||
})),
|
||||
text: z.string(),
|
||||
parentSectionId: z.string().optional(),
|
||||
})),
|
||||
})),
|
||||
});
|
||||
|
||||
export const ttsSentenceAlignmentSchema = z.object({
|
||||
sentence: z.string(),
|
||||
sentenceIndex: z.number(),
|
||||
words: z.array(z.object({
|
||||
text: z.string(),
|
||||
startSec: z.number(),
|
||||
endSec: z.number(),
|
||||
charStart: z.number(),
|
||||
charEnd: z.number(),
|
||||
})),
|
||||
});
|
||||
|
||||
export const whisperOperationCreateSchema = z.object({
|
||||
text: z.string().trim().min(1),
|
||||
lang: z.string().trim().min(1).max(16).optional(),
|
||||
cacheKey: z.string().trim().min(1).max(256).optional(),
|
||||
audioObjectKey: z.string().trim().min(1).max(2048),
|
||||
});
|
||||
|
||||
export const pdfOperationCreateSchema = z.object({
|
||||
documentId: documentIdSchema,
|
||||
namespace: namespaceSchema,
|
||||
documentObjectKey: z.string().trim().min(1).max(2048),
|
||||
replaceToken: z.string().trim().min(1).max(256).optional(),
|
||||
});
|
||||
|
||||
export const pdfResolveSchema = z.object({
|
||||
documentId: documentIdSchema,
|
||||
namespace: namespaceSchema,
|
||||
documentObjectKey: z.string().trim().min(1).max(2048),
|
||||
});
|
||||
|
||||
export const operationParamsSchema = z.object({
|
||||
opId: z.string().trim().min(1),
|
||||
});
|
||||
|
||||
export const operationEventsQuerySchema = z.object({
|
||||
sinceEventId: z.union([z.string(), z.number()]).optional(),
|
||||
});
|
||||
|
||||
export const apiErrorResponseSchema = z.object({
|
||||
error: z.string(),
|
||||
}).passthrough();
|
||||
|
||||
export const operationErrorSchema = z.object({
|
||||
message: z.string(),
|
||||
code: z.string().optional(),
|
||||
});
|
||||
|
||||
export const artifactReferenceSchema = z.object({
|
||||
objectKey: z.string(),
|
||||
});
|
||||
|
||||
export const pdfLayoutProgressSchema = z.object({
|
||||
totalPages: z.number(),
|
||||
pagesParsed: z.number(),
|
||||
currentPage: z.number().optional(),
|
||||
phase: z.enum(['infer', 'merge']),
|
||||
});
|
||||
|
||||
export const publicOperationSchema = z.object({
|
||||
opId: z.string(),
|
||||
subject: z.discriminatedUnion('kind', [
|
||||
z.object({ kind: z.literal('whisper_align') }),
|
||||
z.object({
|
||||
kind: z.literal('pdf_layout'),
|
||||
documentId: z.string(),
|
||||
namespace: z.string().nullable(),
|
||||
}),
|
||||
]),
|
||||
status: z.enum(['queued', 'running', 'succeeded', 'failed']),
|
||||
queuedAt: z.number(),
|
||||
updatedAt: z.number(),
|
||||
startedAt: z.number().optional(),
|
||||
result: z.unknown().optional(),
|
||||
error: operationErrorSchema.optional(),
|
||||
timing: z.object({
|
||||
queueWaitMs: z.number().optional(),
|
||||
s3FetchMs: z.number().optional(),
|
||||
computeMs: z.number().optional(),
|
||||
}).optional(),
|
||||
progress: pdfLayoutProgressSchema.optional(),
|
||||
});
|
||||
|
||||
export const publicOperationEventSchema = z.object({
|
||||
eventId: z.number(),
|
||||
snapshot: publicOperationSchema,
|
||||
});
|
||||
|
||||
export const pdfLayoutResolutionSchema = z.object({
|
||||
artifact: artifactReferenceSchema.nullable(),
|
||||
operation: publicOperationSchema.nullable(),
|
||||
});
|
||||
|
||||
export function jsonSchema(schema: z.ZodType): Record<string, unknown> {
|
||||
return z.toJSONSchema(schema, { target: 'draft-7' }) as Record<string, unknown>;
|
||||
}
|
||||
44
compute-worker/src/compute/whisper/alignment-map.ts
Normal file
44
compute-worker/src/compute/whisper/alignment-map.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
||||
|
||||
export interface WhisperWord {
|
||||
start: number;
|
||||
end: number;
|
||||
word: string;
|
||||
}
|
||||
|
||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||
const lowerSentence = sentence.toLowerCase();
|
||||
let cursor = 0;
|
||||
|
||||
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
||||
const token = w.word.trim();
|
||||
if (!token) {
|
||||
return {
|
||||
text: '',
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: cursor,
|
||||
charEnd: cursor,
|
||||
};
|
||||
}
|
||||
|
||||
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||
const start = idx >= 0 ? idx : cursor;
|
||||
const end = Math.min(sentence.length, start + token.length);
|
||||
cursor = Math.max(cursor, end);
|
||||
|
||||
return {
|
||||
text: token,
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: start,
|
||||
charEnd: end,
|
||||
};
|
||||
}).filter((word) => word.text.length > 0);
|
||||
|
||||
return {
|
||||
sentence,
|
||||
sentenceIndex: 0,
|
||||
words: alignedWords,
|
||||
};
|
||||
}
|
||||
|
|
@ -8,12 +8,12 @@ import type {
|
|||
OperationState,
|
||||
OperationStateStore,
|
||||
QueuedOperation,
|
||||
} from '@openreader/compute-core/control-plane';
|
||||
} from '../compute/control-plane';
|
||||
import type {
|
||||
PdfLayoutJobRequest,
|
||||
WhisperAlignJobRequest,
|
||||
WorkerOperationKind,
|
||||
} from '@openreader/compute-core/api-contracts';
|
||||
} from '../compute/api-contracts';
|
||||
import { createJsonCodec } from './json-codec';
|
||||
|
||||
export interface KvEntryLike {
|
||||
|
|
@ -4,7 +4,7 @@ import type {
|
|||
WorkerJobTiming,
|
||||
WorkerJobState,
|
||||
WorkerOperationState,
|
||||
} from '@openreader/compute-core/api-contracts';
|
||||
} from './compute/api-contracts';
|
||||
|
||||
export type StreamedOperationState = WorkerOperationState<WhisperAlignJobResult | PdfLayoutJobResult>;
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { PdfLayoutProgress } from '@openreader/compute-core/api-contracts';
|
||||
import type { PdfLayoutProgress } from './compute/api-contracts';
|
||||
|
||||
export function buildInferProgressForPageStart(input: {
|
||||
pageNumber: number;
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
||||
import swagger from '@fastify/swagger';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
connect,
|
||||
|
|
@ -24,18 +25,16 @@ import {
|
|||
ensureComputeModels,
|
||||
runPdfLayoutFromPdfBuffer,
|
||||
runWhisperAlignmentFromAudioBuffer,
|
||||
} from '@openreader/compute-core/local-runtime';
|
||||
} from './compute/local-runtime';
|
||||
import {
|
||||
getComputeTimeoutConfig,
|
||||
getComputeOpStaleMs,
|
||||
getAvailableCpuCores,
|
||||
getOnnxThreadsPerJob,
|
||||
PDF_PARSER_VERSION,
|
||||
encodeParserVersion,
|
||||
withIdleTimeoutAndHardCap,
|
||||
withTimeout,
|
||||
} from '@openreader/compute-core';
|
||||
import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane';
|
||||
} from './compute';
|
||||
import { encodeSseFrame, OperationOrchestrator } from './compute/control-plane';
|
||||
import type {
|
||||
PdfLayoutJobRequest,
|
||||
PdfLayoutJobResult,
|
||||
|
|
@ -48,7 +47,7 @@ import type {
|
|||
WorkerOperationRequest,
|
||||
WorkerOperationState,
|
||||
PdfLayoutProgress,
|
||||
} from '@openreader/compute-core/api-contracts';
|
||||
} from './compute/api-contracts';
|
||||
import {
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
|
|
@ -71,6 +70,26 @@ import {
|
|||
import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress';
|
||||
import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy';
|
||||
import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence';
|
||||
import { parsedPdfArtifactKey } from './storage/artifact-addressing';
|
||||
import { buildPdfOperationKey, buildWhisperOperationKey } from './api/operation-keys';
|
||||
import { toPublicOperation, type PublicOperationEvent } from './api/public-operation';
|
||||
import {
|
||||
apiErrorResponseSchema,
|
||||
artifactReferenceSchema,
|
||||
jsonSchema,
|
||||
operationEventsQuerySchema,
|
||||
operationParamsSchema,
|
||||
operationErrorSchema,
|
||||
parsedPdfDocumentSchema,
|
||||
pdfLayoutProgressSchema,
|
||||
pdfLayoutResolutionSchema,
|
||||
pdfOperationCreateSchema,
|
||||
pdfResolveSchema,
|
||||
publicOperationEventSchema,
|
||||
publicOperationSchema,
|
||||
ttsSentenceAlignmentSchema,
|
||||
whisperOperationCreateSchema,
|
||||
} from './api/schemas';
|
||||
|
||||
const JOBS_STREAM_NAME = 'compute_jobs';
|
||||
const WHISPER_JOBS_SUBJECT = 'jobs.whisper';
|
||||
|
|
@ -88,8 +107,6 @@ const OP_EVENTS_KEEPALIVE_MS = 15_000;
|
|||
// client from immediately reconnecting and re-waking the worker; instead it
|
||||
// reconnects on a slow cadence so the container stays asleep most of the time.
|
||||
const OP_EVENTS_RECONNECT_HINT_MS = 120_000;
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const WHISPER_MAX_DELIVER = 1;
|
||||
const NATS_API_TIMEOUT_MS = 60_000;
|
||||
// Disconnect from NATS after this much continuous idle so the worker stops
|
||||
|
|
@ -182,6 +199,7 @@ export interface ComputeWorkerRouteDeps {
|
|||
orchestrator: OrchestratorLike;
|
||||
operationStateStore: OperationStateStoreLike;
|
||||
operationEventStream: OperationEventStreamLike;
|
||||
artifactExists?: (key: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface CreateComputeWorkerAppOptions {
|
||||
|
|
@ -254,20 +272,6 @@ function normalizeS3Prefix(prefix: string | undefined): string {
|
|||
return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader';
|
||||
}
|
||||
|
||||
function sanitizeNamespace(namespace: string | null): string | null {
|
||||
if (!namespace) return null;
|
||||
return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null;
|
||||
}
|
||||
|
||||
function documentParsedKey(id: string, namespace: string | null, prefix: string): string {
|
||||
if (!DOCUMENT_ID_REGEX.test(id)) {
|
||||
throw new Error(`Invalid document id: ${id}`);
|
||||
}
|
||||
const ns = sanitizeNamespace(namespace);
|
||||
const nsSegment = ns ? `ns/${ns}/` : '';
|
||||
return `${prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(PDF_PARSER_VERSION)}.json`;
|
||||
}
|
||||
|
||||
function buildS3Client(): S3Client {
|
||||
const bucket = requireEnv('S3_BUCKET');
|
||||
const region = requireEnv('S3_REGION');
|
||||
|
|
@ -357,7 +361,7 @@ function extractOpId(request: FastifyRequest, path: string): string | null {
|
|||
if (params && typeof params.opId === 'string' && params.opId.trim()) {
|
||||
return params.opId.trim();
|
||||
}
|
||||
const match = path.match(/^\/ops\/([^/]+)/);
|
||||
const match = path.match(/^\/v1\/operations\/([^/]+)/);
|
||||
if (!match?.[1]) return null;
|
||||
try {
|
||||
return decodeURIComponent(match[1]);
|
||||
|
|
@ -428,22 +432,7 @@ const layoutSchema = z.object({
|
|||
documentObjectKey: z.string().trim().min(1).max(2048),
|
||||
});
|
||||
|
||||
const operationCreateSchema = z.discriminatedUnion('kind', [
|
||||
z.object({
|
||||
kind: z.literal('whisper_align'),
|
||||
opKey: z.string().trim().min(1).max(1024),
|
||||
payload: alignSchema,
|
||||
}),
|
||||
z.object({
|
||||
kind: z.literal('pdf_layout'),
|
||||
opKey: z.string().trim().min(1).max(1024),
|
||||
payload: layoutSchema,
|
||||
}),
|
||||
]);
|
||||
|
||||
const operationLookupSchema = z.object({
|
||||
opKey: z.string().trim().min(1).max(1024),
|
||||
});
|
||||
const errorResponseSchema = jsonSchema(apiErrorResponseSchema);
|
||||
|
||||
async function ensureJetStreamResources(
|
||||
jsm: JetStreamManager,
|
||||
|
|
@ -567,7 +556,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
// Lazy NATS connection lifecycle. The worker connects on demand (first request
|
||||
// that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle
|
||||
// so it stops emitting outbound traffic and Railway can sleep it. Reconnect is
|
||||
// transparent: any inbound /ops request both wakes the container and re-establishes
|
||||
// transparent: any inbound operation request both wakes the container and re-establishes
|
||||
// the session via ensureConnected().
|
||||
let session: NatsSession | null = null;
|
||||
let connecting: Promise<NatsSession> | null = null;
|
||||
|
|
@ -785,7 +774,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
if (!s3) {
|
||||
throw new Error('S3 access is disabled for this worker app instance');
|
||||
}
|
||||
const key = documentParsedKey(documentId, namespace, s3Prefix);
|
||||
const key = parsedPdfArtifactKey({ documentId, namespace, prefix: s3Prefix });
|
||||
const body = Buffer.from(JSON.stringify(parsed));
|
||||
await s3.send(new PutObjectCommand({
|
||||
Bucket: s3Bucket,
|
||||
|
|
@ -805,6 +794,35 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
logger: buildLoggerConfig(),
|
||||
disableRequestLogging: true,
|
||||
});
|
||||
await app.register(swagger, {
|
||||
openapi: {
|
||||
openapi: '3.0.3',
|
||||
info: {
|
||||
title: 'OpenReader Compute Worker API',
|
||||
version: '1.0.0',
|
||||
},
|
||||
components: {
|
||||
securitySchemes: {
|
||||
bearerAuth: {
|
||||
type: 'http',
|
||||
scheme: 'bearer',
|
||||
},
|
||||
},
|
||||
schemas: {
|
||||
ParsedPdfDocument: jsonSchema(parsedPdfDocumentSchema),
|
||||
TTSSentenceAlignment: jsonSchema(ttsSentenceAlignmentSchema),
|
||||
ArtifactReference: jsonSchema(artifactReferenceSchema),
|
||||
ErrorResponse: jsonSchema(apiErrorResponseSchema),
|
||||
OperationError: jsonSchema(operationErrorSchema),
|
||||
PdfLayoutProgress: jsonSchema(pdfLayoutProgressSchema),
|
||||
PublicOperation: jsonSchema(publicOperationSchema),
|
||||
PublicOperationEvent: jsonSchema(publicOperationEventSchema),
|
||||
PdfLayoutResolution: jsonSchema(pdfLayoutResolutionSchema),
|
||||
},
|
||||
},
|
||||
security: [{ bearerAuth: [] }],
|
||||
},
|
||||
});
|
||||
app.log.info({
|
||||
jobConcurrency,
|
||||
whisperTimeoutMs,
|
||||
|
|
@ -956,16 +974,37 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
}
|
||||
});
|
||||
|
||||
app.get('/health/live', async () => ({ ok: true }));
|
||||
app.get('/health/live', {
|
||||
schema: {
|
||||
security: [],
|
||||
response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } },
|
||||
},
|
||||
}, async () => ({ ok: true }));
|
||||
|
||||
// Reports readiness without forcing a NATS round-trip. Probing NATS here would
|
||||
// reconnect (and keep) the connection open, defeating idle sleep, so we only
|
||||
// report the current connection state. The worker reconnects lazily on the next
|
||||
// /ops request regardless of what this returns.
|
||||
app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null }));
|
||||
// operation request regardless of what this returns.
|
||||
app.get('/health/ready', {
|
||||
schema: {
|
||||
security: [],
|
||||
response: {
|
||||
200: {
|
||||
type: 'object',
|
||||
properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } },
|
||||
required: ['ok', 'natsConnected'],
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async () => ({ ok: true, natsConnected: session !== null }));
|
||||
|
||||
app.post('/ops', async (request, reply) => {
|
||||
const parsed = operationCreateSchema.safeParse(request.body);
|
||||
app.post('/v1/whisper-align/operations', {
|
||||
schema: {
|
||||
body: jsonSchema(whisperOperationCreateSchema),
|
||||
response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema },
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
const parsed = whisperOperationCreateSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
reply.code(400);
|
||||
return {
|
||||
|
|
@ -974,7 +1013,11 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
};
|
||||
}
|
||||
|
||||
const requestOp = parsed.data as WorkerOperationRequest;
|
||||
const requestOp: WorkerOperationRequest = {
|
||||
kind: 'whisper_align',
|
||||
opKey: buildWhisperOperationKey(parsed.data),
|
||||
payload: parsed.data,
|
||||
};
|
||||
await ensureOrphanedOpRecovery();
|
||||
const op = await orchestrator.enqueueOrReuse(requestOp);
|
||||
app.log.info({
|
||||
|
|
@ -985,11 +1028,16 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16),
|
||||
}, 'op.accepted');
|
||||
reply.code(202);
|
||||
return op;
|
||||
return toPublicOperation(op);
|
||||
});
|
||||
|
||||
app.post('/ops/lookup', async (request, reply) => {
|
||||
const parsed = operationLookupSchema.safeParse(request.body);
|
||||
app.post('/v1/pdf-layout/operations', {
|
||||
schema: {
|
||||
body: jsonSchema(pdfOperationCreateSchema),
|
||||
response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema },
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
const parsed = pdfOperationCreateSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
reply.code(400);
|
||||
return {
|
||||
|
|
@ -998,28 +1046,58 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
};
|
||||
}
|
||||
|
||||
const requestOp: WorkerOperationRequest = {
|
||||
kind: 'pdf_layout',
|
||||
opKey: buildPdfOperationKey(parsed.data),
|
||||
payload: {
|
||||
documentId: parsed.data.documentId,
|
||||
namespace: parsed.data.namespace,
|
||||
documentObjectKey: parsed.data.documentObjectKey,
|
||||
},
|
||||
};
|
||||
await ensureOrphanedOpRecovery();
|
||||
if (typeof operationStateStore.getOpIndex !== 'function') {
|
||||
reply.code(501);
|
||||
return { error: 'Operation lookup by opKey is not supported by this state store' };
|
||||
}
|
||||
const indexEntry = await operationStateStore.getOpIndex(parsed.data.opKey);
|
||||
if (!indexEntry?.opId) {
|
||||
reply.code(404);
|
||||
return { error: 'Operation not found' };
|
||||
}
|
||||
|
||||
const state = await operationStateStore.getOpState(indexEntry.opId);
|
||||
if (!state) {
|
||||
reply.code(404);
|
||||
return { error: 'Operation not found' };
|
||||
}
|
||||
|
||||
return state;
|
||||
const op = await orchestrator.enqueueOrReuse(requestOp);
|
||||
reply.code(202);
|
||||
return toPublicOperation(op);
|
||||
});
|
||||
|
||||
app.get('/ops/:opId', async (request, reply) => {
|
||||
const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params);
|
||||
app.post('/v1/pdf-layout/resolve', {
|
||||
schema: {
|
||||
body: jsonSchema(pdfResolveSchema),
|
||||
response: {
|
||||
200: jsonSchema(pdfLayoutResolutionSchema),
|
||||
400: errorResponseSchema,
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
const parsed = pdfResolveSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
reply.code(400);
|
||||
return { error: 'Invalid request body', issues: parsed.error.issues };
|
||||
}
|
||||
await ensureOrphanedOpRecovery();
|
||||
const artifactKey = parsedPdfArtifactKey({
|
||||
documentId: parsed.data.documentId,
|
||||
namespace: parsed.data.namespace,
|
||||
prefix: s3Prefix,
|
||||
});
|
||||
const hasArtifact = await (options.routeDeps?.artifactExists ?? objectExists)(artifactKey);
|
||||
const opKey = buildPdfOperationKey(parsed.data);
|
||||
const index = await operationStateStore.getOpIndex?.(opKey);
|
||||
const operation = index?.opId ? await operationStateStore.getOpState(index.opId) : null;
|
||||
return {
|
||||
artifact: hasArtifact ? { objectKey: artifactKey } : null,
|
||||
operation: operation ? toPublicOperation(operation) : null,
|
||||
};
|
||||
});
|
||||
|
||||
app.get('/v1/operations/:opId', {
|
||||
schema: {
|
||||
params: jsonSchema(operationParamsSchema),
|
||||
response: { 200: jsonSchema(publicOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema },
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
const params = operationParamsSchema.safeParse(request.params);
|
||||
if (!params.success) {
|
||||
reply.code(400);
|
||||
return { error: 'Invalid op id' };
|
||||
|
|
@ -1031,11 +1109,21 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
return { error: 'Operation not found' };
|
||||
}
|
||||
|
||||
return state;
|
||||
return toPublicOperation(state);
|
||||
});
|
||||
|
||||
app.get('/ops/:opId/events', async (request, reply) => {
|
||||
const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params);
|
||||
app.get('/v1/operations/:opId/events', {
|
||||
schema: {
|
||||
params: jsonSchema(operationParamsSchema),
|
||||
querystring: jsonSchema(operationEventsQuerySchema),
|
||||
response: {
|
||||
200: { type: 'string', description: 'Server-sent PublicOperationEvent stream' },
|
||||
400: errorResponseSchema,
|
||||
404: errorResponseSchema,
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
const params = operationParamsSchema.safeParse(request.params);
|
||||
if (!params.success) {
|
||||
reply.code(400);
|
||||
return { error: 'Invalid op id' };
|
||||
|
|
@ -1080,9 +1168,9 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
|||
|
||||
const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => {
|
||||
if (closed || reply.raw.writableEnded) return;
|
||||
const frameEvent: WorkerOperationEvent<WhisperAlignJobResult | PdfLayoutJobResult> = {
|
||||
const frameEvent: PublicOperationEvent<WhisperAlignJobResult | PdfLayoutJobResult> = {
|
||||
eventId,
|
||||
snapshot,
|
||||
snapshot: toPublicOperation(snapshot),
|
||||
};
|
||||
reply.raw.write(encodeSseFrame({
|
||||
id: eventId,
|
||||
21
compute-worker/src/storage/artifact-addressing.ts
Normal file
21
compute-worker/src/storage/artifact-addressing.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version';
|
||||
import { encodeParserVersion } from '../compute/pdf/parser-version-key';
|
||||
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
export function parsedPdfArtifactKey(input: {
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
prefix: string;
|
||||
parserVersion?: string;
|
||||
}): string {
|
||||
if (!DOCUMENT_ID_REGEX.test(input.documentId)) {
|
||||
throw new Error(`Invalid document id: ${input.documentId}`);
|
||||
}
|
||||
const namespace = input.namespace && SAFE_NAMESPACE_REGEX.test(input.namespace)
|
||||
? input.namespace
|
||||
: null;
|
||||
const namespaceSegment = namespace ? `ns/${namespace}/` : '';
|
||||
return `${input.prefix}/documents_v1/parsed_v2/${namespaceSegment}${input.documentId}/${encodeParserVersion(input.parserVersion ?? PDF_PARSER_VERSION)}.json`;
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts';
|
||||
import type { WorkerOperationKind } from './compute/api-contracts';
|
||||
|
||||
export type RetryAction = 'nak_retry' | 'term_fail';
|
||||
|
||||
|
|
@ -25,53 +25,51 @@ describe('compute worker API routes', () => {
|
|||
const live = await runtime.app.inject({ method: 'GET', url: '/health/live' });
|
||||
expect(live.statusCode).toBe(200);
|
||||
|
||||
const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' });
|
||||
const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/v1/operations/op-1' });
|
||||
expect(protectedRoute.statusCode).toBe(401);
|
||||
});
|
||||
|
||||
test('validates operation creation body and returns 400 for invalid payload', async () => {
|
||||
const response = await runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/ops',
|
||||
url: '/v1/pdf-layout/operations',
|
||||
headers: AUTH,
|
||||
payload: {
|
||||
kind: 'pdf_layout',
|
||||
opKey: '',
|
||||
payload: {
|
||||
documentId: 'd1',
|
||||
documentId: '',
|
||||
namespace: null,
|
||||
documentObjectKey: 's3://bucket/doc.pdf',
|
||||
},
|
||||
documentObjectKey: 'openreader/doc.pdf',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(response.json()).toMatchObject({ error: 'Invalid request body' });
|
||||
expect(response.json()).toMatchObject({ error: 'Bad Request' });
|
||||
});
|
||||
|
||||
test('creates operation and fetches state by op id', async () => {
|
||||
const documentId = 'c'.repeat(64);
|
||||
const create = await runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/ops',
|
||||
url: '/v1/pdf-layout/operations',
|
||||
headers: AUTH,
|
||||
payload: {
|
||||
kind: 'pdf_layout',
|
||||
opKey: 'doc-1:layout',
|
||||
payload: {
|
||||
documentId: 'doc-1',
|
||||
documentId,
|
||||
namespace: null,
|
||||
documentObjectKey: 'openreader/doc-1.pdf',
|
||||
},
|
||||
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(create.statusCode).toBe(202);
|
||||
const created = create.json();
|
||||
expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' });
|
||||
expect(created).toMatchObject({
|
||||
subject: { kind: 'pdf_layout', documentId, namespace: null },
|
||||
status: 'queued',
|
||||
});
|
||||
expect(created).not.toHaveProperty('opKey');
|
||||
expect(created).not.toHaveProperty('jobId');
|
||||
|
||||
const fetch = await runtime.app.inject({
|
||||
method: 'GET',
|
||||
url: `/ops/${created.opId}`,
|
||||
url: `/v1/operations/${created.opId}`,
|
||||
headers: AUTH,
|
||||
});
|
||||
|
||||
|
|
@ -79,17 +77,120 @@ describe('compute worker API routes', () => {
|
|||
expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' });
|
||||
});
|
||||
|
||||
test('reuses idempotent PDF requests and replaces them only with an explicit token', async () => {
|
||||
const documentId = 'e'.repeat(64);
|
||||
const payload = {
|
||||
documentId,
|
||||
namespace: null,
|
||||
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||
};
|
||||
const create = (body: typeof payload & { replaceToken?: string }) => runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/pdf-layout/operations',
|
||||
headers: AUTH,
|
||||
payload: body,
|
||||
});
|
||||
|
||||
const initial = (await create(payload)).json();
|
||||
const reused = (await create(payload)).json();
|
||||
const replacement = (await create({ ...payload, replaceToken: 'replace-1' })).json();
|
||||
|
||||
expect(reused.opId).toBe(initial.opId);
|
||||
expect(replacement.opId).not.toBe(initial.opId);
|
||||
expect(replacement.subject).toEqual(initial.subject);
|
||||
});
|
||||
|
||||
test('creates whisper alignment operations without exposing internal keys', async () => {
|
||||
const response = await runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/whisper-align/operations',
|
||||
headers: AUTH,
|
||||
payload: {
|
||||
text: 'Canonical worker text',
|
||||
audioObjectKey: 'openreader/audio.mp3',
|
||||
lang: 'en',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(202);
|
||||
expect(response.json()).toMatchObject({
|
||||
subject: { kind: 'whisper_align' },
|
||||
status: 'queued',
|
||||
});
|
||||
expect(response.json()).not.toHaveProperty('opKey');
|
||||
});
|
||||
|
||||
test('resolves the current PDF artifact and operation without exposing parser identity', async () => {
|
||||
const documentId = 'a'.repeat(64);
|
||||
const create = await runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/pdf-layout/operations',
|
||||
headers: AUTH,
|
||||
payload: {
|
||||
documentId,
|
||||
namespace: null,
|
||||
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||
},
|
||||
});
|
||||
expect(create.statusCode).toBe(202);
|
||||
|
||||
const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`;
|
||||
fake.seedArtifact(artifactKey);
|
||||
const resolve = await runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/pdf-layout/resolve',
|
||||
headers: AUTH,
|
||||
payload: {
|
||||
documentId,
|
||||
namespace: null,
|
||||
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolve.statusCode).toBe(200);
|
||||
expect(resolve.json()).toMatchObject({
|
||||
artifact: { objectKey: artifactKey },
|
||||
operation: {
|
||||
subject: { kind: 'pdf_layout', documentId, namespace: null },
|
||||
},
|
||||
});
|
||||
expect(resolve.json().operation).not.toHaveProperty('opKey');
|
||||
});
|
||||
|
||||
test('resolves a current PDF artifact after operation state has expired', async () => {
|
||||
const documentId = 'd'.repeat(64);
|
||||
const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`;
|
||||
fake.seedArtifact(artifactKey);
|
||||
|
||||
const resolve = await runtime.app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/pdf-layout/resolve',
|
||||
headers: AUTH,
|
||||
payload: {
|
||||
documentId,
|
||||
namespace: null,
|
||||
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolve.statusCode).toBe(200);
|
||||
expect(resolve.json()).toEqual({
|
||||
artifact: { objectKey: artifactKey },
|
||||
operation: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('returns not found for unknown operation and event stream lookups', async () => {
|
||||
const opResponse = await runtime.app.inject({
|
||||
method: 'GET',
|
||||
url: '/ops/missing',
|
||||
url: '/v1/operations/missing',
|
||||
headers: AUTH,
|
||||
});
|
||||
expect(opResponse.statusCode).toBe(404);
|
||||
|
||||
const eventsResponse = await runtime.app.inject({
|
||||
method: 'GET',
|
||||
url: '/ops/missing/events',
|
||||
url: '/v1/operations/missing/events',
|
||||
headers: AUTH,
|
||||
});
|
||||
expect(eventsResponse.statusCode).toBe(404);
|
||||
|
|
@ -109,7 +210,7 @@ describe('compute worker API routes', () => {
|
|||
|
||||
const stream = await runtime.app.inject({
|
||||
method: 'GET',
|
||||
url: '/ops/op-terminal/events?sinceEventId=7',
|
||||
url: '/v1/operations/op-terminal/events?sinceEventId=7',
|
||||
headers: AUTH,
|
||||
});
|
||||
|
||||
|
|
@ -159,11 +260,11 @@ describe('compute worker API routes', () => {
|
|||
updatedAt: now - 310_000,
|
||||
});
|
||||
|
||||
// GET /ops/:opId resolves via getOpState(), which first awaits the shared
|
||||
// GET /v1/operations/:opId resolves via getOpState(), which first awaits the shared
|
||||
// orphanRecoveryPromise path through ensureOrphanedOpRecovery().
|
||||
const fetch = await runtime.app.inject({
|
||||
method: 'GET',
|
||||
url: '/ops/op-stale-whisper-running',
|
||||
url: '/v1/operations/op-stale-whisper-running',
|
||||
headers: AUTH,
|
||||
});
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { mergeTextWithRegions } from '@openreader/compute-core';
|
||||
import { mergeTextWithRegions } from '../../../src/compute';
|
||||
|
||||
describe('mergeTextWithRegions', () => {
|
||||
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
|
||||
import { normalizeTextItemsForLayout } from '@openreader/compute-core';
|
||||
import { normalizeTextItemsForLayout } from '../../../src/compute';
|
||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
|
||||
function makeTextItem(
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { stitchCrossPageBlocks } from '@openreader/compute-core';
|
||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf';
|
||||
import { stitchCrossPageBlocks } from '../../../src/compute';
|
||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/compute/types';
|
||||
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
||||
|
||||
function makeBlock(
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
import type { BaseDocument } from '../../../src/types/documents';
|
||||
import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/compute/types';
|
||||
|
||||
export function makeBaseDocument(overrides: Partial<BaseDocument> = {}): BaseDocument {
|
||||
return {
|
||||
id: 'doc-1',
|
||||
name: 'document.pdf',
|
||||
size: 1_024,
|
||||
lastModified: 1_700_000_000_000,
|
||||
type: 'pdf',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeParsedPdfBlock(input: {
|
||||
id: string;
|
||||
kind: ParsedPdfBlockKind;
|
||||
text: string;
|
||||
page: number;
|
||||
readingOrder: number;
|
||||
}): ParsedPdfBlock {
|
||||
return {
|
||||
id: input.id,
|
||||
kind: input.kind,
|
||||
text: input.text,
|
||||
fragments: [
|
||||
{
|
||||
page: input.page,
|
||||
bbox: [0, 0, 100, 10],
|
||||
text: input.text,
|
||||
readingOrder: input.readingOrder,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage {
|
||||
return {
|
||||
pageNumber,
|
||||
width: 800,
|
||||
height: 1_200,
|
||||
blocks,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
documentId: 'doc-fixture',
|
||||
parserVersion: 'test',
|
||||
parsedAt: 1_700_000_000_000,
|
||||
pages,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import {
|
||||
mapWordsToSentenceOffsets,
|
||||
} from '@openreader/compute-core';
|
||||
} from '../../../src/compute';
|
||||
|
||||
describe('whisper alignment mapping', () => {
|
||||
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from '../../../src/compute';
|
||||
|
||||
function dftPower(samples: Float32Array, k: number): number {
|
||||
const n = samples.length;
|
||||
|
|
@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node';
|
|||
import {
|
||||
buildWordsFromTimestampedTokens,
|
||||
extractTokenStartTimestamps,
|
||||
} from '@openreader/compute-core';
|
||||
} from '../../../src/compute';
|
||||
|
||||
describe('whisper token timestamp alignment', () => {
|
||||
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import type { WorkerOperationRequest } from '../../src/api-contracts';
|
||||
import { OperationOrchestrator } from '../../src/control-plane';
|
||||
import type { WorkerOperationRequest } from '../../../src/compute/api-contracts';
|
||||
import { OperationOrchestrator } from '../../../src/compute/control-plane';
|
||||
import {
|
||||
InMemoryOperationEventStream,
|
||||
InMemoryOperationQueue,
|
||||
|
|
@ -70,13 +70,13 @@ vi.mock('@napi-rs/canvas', () => {
|
|||
};
|
||||
});
|
||||
|
||||
vi.mock('../../src/pdf/model', () => ({
|
||||
vi.mock('../../../src/compute/pdf/model', () => ({
|
||||
ensureModel: vi.fn(async () => '/tmp/model.onnx'),
|
||||
MODEL_CONFIG_PATH: '/tmp/model-config.json',
|
||||
MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json',
|
||||
}));
|
||||
|
||||
vi.mock('../../src/config/cpu-budget', () => ({
|
||||
vi.mock('../../../src/compute/config/cpu-budget', () => ({
|
||||
getOnnxThreadsPerJob: vi.fn(() => 1),
|
||||
}));
|
||||
|
||||
|
|
@ -107,7 +107,7 @@ describe('runLayoutModel', () => {
|
|||
},
|
||||
};
|
||||
|
||||
const { runLayoutModel } = await import('../../src/pdf/runLayoutModel');
|
||||
const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel');
|
||||
const regions = await runLayoutModel({
|
||||
pageWidth: 100,
|
||||
pageHeight: 100,
|
||||
|
|
@ -154,7 +154,7 @@ describe('runLayoutModel', () => {
|
|||
},
|
||||
};
|
||||
|
||||
const { runLayoutModel } = await import('../../src/pdf/runLayoutModel');
|
||||
const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel');
|
||||
const regions = await runLayoutModel({
|
||||
pageWidth: 100,
|
||||
pageHeight: 100,
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse';
|
||||
import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/compute/control-plane/sse';
|
||||
|
||||
describe('sse codec', () => {
|
||||
test('encodes event id and payload and decodes both reliably', () => {
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import type { WorkerOperationState } from '../../src/api-contracts';
|
||||
import type { WorkerOperationState } from '../../../src/compute/api-contracts';
|
||||
import {
|
||||
explainReplacementReason,
|
||||
isInflightStatus,
|
||||
isTerminalStatus,
|
||||
shouldReuseExistingOperation,
|
||||
} from '../../src/control-plane/state-machine';
|
||||
} from '../../../src/compute/control-plane/state-machine';
|
||||
|
||||
function runningState(overrides: Partial<WorkerOperationState> = {}): WorkerOperationState {
|
||||
return {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { EventEmitter } from 'node:events';
|
||||
import type { WorkerOperationKind } from '../../src/api-contracts';
|
||||
import type { WorkerOperationKind } from '../../../src/compute/api-contracts';
|
||||
import type {
|
||||
OperationEvent,
|
||||
OperationEventStream,
|
||||
|
|
@ -8,7 +8,7 @@ import type {
|
|||
OperationState,
|
||||
OperationStateStore,
|
||||
QueuedOperation,
|
||||
} from '../../src/control-plane/types';
|
||||
} from '../../../src/compute/control-plane/types';
|
||||
|
||||
function topicFor(opId: string): string {
|
||||
return `op.${opId}`;
|
||||
|
|
@ -4,7 +4,7 @@ import type {
|
|||
WorkerOperationEvent,
|
||||
WorkerOperationRequest,
|
||||
WorkerOperationState,
|
||||
} from '@openreader/compute-core/api-contracts';
|
||||
} from '../../src/compute/api-contracts';
|
||||
import type { ComputeWorkerRouteDeps } from '../../src/runtime';
|
||||
|
||||
type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult;
|
||||
|
|
@ -16,6 +16,7 @@ export class FakeControlPlane {
|
|||
private readonly revisionByOpId = new Map<string, number>();
|
||||
private readonly opIdByOpKey = new Map<string, string>();
|
||||
private readonly eventsByOpId = new Map<string, ComputeEvent[]>();
|
||||
private readonly artifactKeys = new Set<string>();
|
||||
private nextOpId = 1;
|
||||
|
||||
readonly deps: ComputeWorkerRouteDeps = {
|
||||
|
|
@ -58,6 +59,10 @@ export class FakeControlPlane {
|
|||
};
|
||||
},
|
||||
listOpStates: async () => Array.from(this.stateByOpId.values()),
|
||||
getOpIndex: async (opKey) => {
|
||||
const opId = this.opIdByOpKey.get(opKey);
|
||||
return opId ? { opId } : null;
|
||||
},
|
||||
},
|
||||
operationEventStream: {
|
||||
subscribe: async ({ opId, sinceEventId, onEvent }) => {
|
||||
|
|
@ -69,6 +74,7 @@ export class FakeControlPlane {
|
|||
return () => undefined;
|
||||
},
|
||||
},
|
||||
artifactExists: async (key) => this.artifactKeys.has(key),
|
||||
};
|
||||
|
||||
seedState(state: ComputeState): void {
|
||||
|
|
@ -83,6 +89,10 @@ export class FakeControlPlane {
|
|||
this.eventsByOpId.set(opId, list);
|
||||
}
|
||||
|
||||
seedArtifact(key: string): void {
|
||||
this.artifactKeys.add(key);
|
||||
}
|
||||
|
||||
getState(opId: string): ComputeState | null {
|
||||
return this.stateByOpId.get(opId) ?? null;
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { OperationOrchestrator } from '@openreader/compute-core/control-plane';
|
||||
import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts';
|
||||
import { OperationOrchestrator } from '../../src/compute/control-plane';
|
||||
import type { WorkerOperationRequest } from '../../src/compute/api-contracts';
|
||||
import {
|
||||
JetStreamOperationEventStream,
|
||||
JetStreamOperationQueue,
|
||||
|
|
@ -4,6 +4,8 @@ import {
|
|||
buildInferProgressForPageParsed,
|
||||
buildInferProgressForPageStart,
|
||||
} from '../../src/pdf-progress';
|
||||
import { buildPdfOperationKey } from '../../src/api/operation-keys';
|
||||
import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing';
|
||||
|
||||
describe('compute worker helpers', () => {
|
||||
test('hash and kv key helpers are stable and deterministic', () => {
|
||||
|
|
@ -34,4 +36,17 @@ describe('compute worker helpers', () => {
|
|||
phase: 'infer',
|
||||
});
|
||||
});
|
||||
|
||||
test('parser-version changes rotate worker-owned operation and artifact identities', () => {
|
||||
const request = {
|
||||
documentId: 'a'.repeat(64),
|
||||
namespace: null,
|
||||
documentObjectKey: `openreader/${'a'.repeat(64)}.pdf`,
|
||||
};
|
||||
|
||||
expect(buildPdfOperationKey(request, 'parser-v1'))
|
||||
.not.toBe(buildPdfOperationKey(request, 'parser-v2'));
|
||||
expect(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v1' }))
|
||||
.not.toBe(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v2' }));
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"extends": "../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": true,
|
||||
},
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{
|
||||
"name": "@openreader/compute-core",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"@napi-rs/canvas": "^0.1.100",
|
||||
"ffmpeg-static": "^5.3.0",
|
||||
"jszip": "^3.10.1",
|
||||
"onnxruntime-node": "^1.26.0",
|
||||
"pdfjs-dist": "4.8.69"
|
||||
},
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./local-runtime": "./src/local-runtime.ts",
|
||||
"./api-contracts": "./src/api-contracts/index.ts",
|
||||
"./control-plane": "./src/control-plane/index.ts",
|
||||
"./types": "./src/types/index.ts"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
||||
|
||||
// Worker-side mirror of the app's canonical audio-text cleaning rules in
|
||||
// `src/lib/shared/audio-text.ts`. This is a separate build target and cannot
|
||||
// import from `@/lib`, so the rules are duplicated here on purpose. The word
|
||||
// `charStart`/`charEnd` offsets this module emits are consumed against text
|
||||
// normalized by that shared module, so any divergence shifts viewer highlights
|
||||
// off-word — keep this byte-for-byte in sync with `audio-text.ts`.
|
||||
const STRIPPED_GLYPHS = /[*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖]/g;
|
||||
|
||||
function preprocessSentenceForAudio(text: string): string {
|
||||
return text
|
||||
.replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -')
|
||||
.replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2')
|
||||
.replace(STRIPPED_GLYPHS, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export interface WhisperWord {
|
||||
start: number;
|
||||
end: number;
|
||||
word: string;
|
||||
}
|
||||
|
||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||
const normalizedSentence = preprocessSentenceForAudio(sentence);
|
||||
const lowerSentence = normalizedSentence.toLowerCase();
|
||||
let cursor = 0;
|
||||
|
||||
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
||||
const token = w.word.trim();
|
||||
if (!token) {
|
||||
return {
|
||||
text: '',
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: cursor,
|
||||
charEnd: cursor,
|
||||
};
|
||||
}
|
||||
|
||||
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||
const start = idx >= 0 ? idx : cursor;
|
||||
const end = Math.min(normalizedSentence.length, start + token.length);
|
||||
cursor = Math.max(cursor, end);
|
||||
|
||||
return {
|
||||
text: token,
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: start,
|
||||
charEnd: end,
|
||||
};
|
||||
}).filter((word) => word.text.length > 0);
|
||||
|
||||
return {
|
||||
sentence,
|
||||
sentenceIndex: 0,
|
||||
words: alignedWords,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
|
|
@ -11,7 +11,8 @@ Use this guide when OpenReader runs compute as a separate service. For the defau
|
|||
- Runs PDF layout parsing jobs
|
||||
- Stores durable job state in NATS JetStream and NATS KV
|
||||
|
||||
The app server submits work to `POST /ops` and listens for updates on `GET /ops/:opId/events`.
|
||||
The app server submits resource-specific operations under `/v1` and listens for updates on
|
||||
`GET /v1/operations/:opId/events`.
|
||||
|
||||
## When to use it
|
||||
|
||||
|
|
@ -37,7 +38,7 @@ S3_SECRET_ACCESS_KEY=...
|
|||
```
|
||||
|
||||
:::important
|
||||
`compute/worker/.env*` is only for standalone worker deployments.
|
||||
`compute-worker/.env*` is only for standalone worker deployments.
|
||||
|
||||
- Embedded/local mode: configure the root `.env` only.
|
||||
- External worker mode: set `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` on the app, and worker runtime values on the worker service.
|
||||
|
|
|
|||
|
|
@ -163,17 +163,17 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i
|
|||
<summary><strong>External compute worker dev stack (optional)</strong></summary>
|
||||
|
||||
Use this only when you intentionally run compute-worker as a separate service.
|
||||
Default local flow does not need `compute/worker/.env`; embedded worker startup reads root `.env`.
|
||||
Default local flow does not need `compute-worker/.env`; embedded worker startup reads root `.env`.
|
||||
Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker).
|
||||
|
||||
Start only NATS + compute-worker via compose watch:
|
||||
|
||||
```bash
|
||||
docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch
|
||||
# or: pnpm compute:dev:watch
|
||||
docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --watch
|
||||
# or: pnpm compute:dev:compose
|
||||
```
|
||||
|
||||
`compute/worker/.env.example` contains a starter config for standalone worker service deployments.
|
||||
`compute-worker/.env.example` contains a starter config for standalone worker service deployments.
|
||||
|
||||
Run the main app separately on the host:
|
||||
|
||||
|
|
@ -190,9 +190,9 @@ COMPUTE_WORKER_TOKEN=<same-token-used-by-worker>
|
|||
|
||||
Ownership in external worker mode:
|
||||
- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS`
|
||||
- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
||||
- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
||||
|
||||
For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute/worker/.env*` is ignored in that mode.
|
||||
For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute-worker/.env*` is ignored in that mode.
|
||||
|
||||
Worker mode requires worker-reachable shared object storage (S3-compatible endpoint).
|
||||
For external worker mode, object storage must be shared/reachable by both app and worker services.
|
||||
|
|
@ -240,7 +240,7 @@ COMPUTE_WORKER_TOKEN=<same-token-used-by-worker>
|
|||
|
||||
Use the same ownership split:
|
||||
- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale overrides
|
||||
- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
||||
- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
||||
|
||||
Use one of these `.env` mode templates:
|
||||
|
||||
|
|
|
|||
|
|
@ -45,21 +45,20 @@ title: Stack
|
|||
|
||||
## External compute worker (optional)
|
||||
|
||||
Monorepo packages under `compute/`:
|
||||
Standalone worker package:
|
||||
|
||||
- **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts
|
||||
- **`@openreader/compute-worker`** — standalone Node.js compute service containing its private inference and queue runtime
|
||||
- ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers`
|
||||
- Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps
|
||||
- PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing
|
||||
- PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization
|
||||
- Utilities: `jszip`, `ffmpeg-static`
|
||||
- **`@openreader/compute-worker`** — standalone Node.js worker service
|
||||
- HTTP server: [Fastify](https://fastify.dev/) v5
|
||||
- HTTP server: [Fastify](https://fastify.dev/) v5 with a versioned OpenAPI contract
|
||||
- Job queue + state: [NATS](https://nats.io/) JetStream WorkQueue pull consumers + NATS KV (`jobs.whisper`, `jobs.layout`)
|
||||
- Storage: AWS SDK v3 S3 client for reading/writing blobs
|
||||
- Logging: [Pino](https://getpino.io/)
|
||||
- Validation: [Zod](https://zod.dev/)
|
||||
- Heavy compute is worker-backed via `COMPUTE_WORKER_URL` + `COMPUTE_WORKER_TOKEN` (remote queue via HTTP + NATS)
|
||||
- The Next.js app communicates with the worker only through the versioned HTTP API generated from OpenAPI.
|
||||
|
||||
## Tooling and testing
|
||||
|
||||
|
|
|
|||
|
|
@ -34,16 +34,14 @@ const UI_ARCHITECTURE_FILES = [
|
|||
"src/components/player/**/*.{ts,tsx}",
|
||||
"src/components/reader/**/*.{ts,tsx}",
|
||||
];
|
||||
const COMPUTE_CORE_IMPORT_PATTERNS = [
|
||||
const COMPUTE_WORKER_IMPORT_PATTERNS = [
|
||||
{
|
||||
group: [
|
||||
"@openreader/compute-core/*",
|
||||
"!@openreader/compute-core/local-runtime",
|
||||
"!@openreader/compute-core/types",
|
||||
"!@openreader/compute-core/api-contracts",
|
||||
"@openreader/compute-worker",
|
||||
"@openreader/compute-worker/*",
|
||||
],
|
||||
message:
|
||||
"Use '@openreader/compute-core' root imports for light APIs. Allowed subpaths are '@openreader/compute-core/local-runtime', '@openreader/compute-core/types', and '@openreader/compute-core/api-contracts'.",
|
||||
"The app must communicate with compute-worker through the app-owned HTTP client and generated protocol types.",
|
||||
},
|
||||
];
|
||||
const UI_ARCHITECTURE_IMPORT_PATHS = [
|
||||
|
|
@ -124,7 +122,7 @@ const eslintConfig = [
|
|||
"no-restricted-imports": [
|
||||
"error",
|
||||
{
|
||||
patterns: COMPUTE_CORE_IMPORT_PATTERNS,
|
||||
patterns: COMPUTE_WORKER_IMPORT_PATTERNS,
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
@ -142,7 +140,7 @@ const eslintConfig = [
|
|||
"error",
|
||||
{
|
||||
paths: UI_ARCHITECTURE_IMPORT_PATHS,
|
||||
patterns: COMPUTE_CORE_IMPORT_PATTERNS,
|
||||
patterns: COMPUTE_WORKER_IMPORT_PATTERNS,
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ const securityHeaders = [
|
|||
},
|
||||
];
|
||||
|
||||
const bundleWorkerCompute = true;
|
||||
const pdfjsTraceFiles = [
|
||||
'./node_modules/pdfjs-dist/package.json',
|
||||
'./node_modules/pdfjs-dist/legacy/build/pdf.mjs',
|
||||
|
|
@ -29,7 +28,6 @@ const serverExternalPackages = [
|
|||
// Keep pdfjs-dist as a real package in node_modules. Server-side preview
|
||||
// rendering resolves pdf.js runtime assets from the filesystem at runtime.
|
||||
'pdfjs-dist',
|
||||
...(!bundleWorkerCompute ? ['onnxruntime-node', '@huggingface/tokenizers'] : []),
|
||||
];
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
|
|
@ -88,14 +86,6 @@ const nextConfig: NextConfig = {
|
|||
},
|
||||
];
|
||||
}
|
||||
if (isServer && bundleWorkerCompute) {
|
||||
config.resolve.alias = {
|
||||
...(config.resolve.alias || {}),
|
||||
'@openreader/compute-core/local-runtime$': false,
|
||||
'onnxruntime-node$': false,
|
||||
'@huggingface/tokenizers$': false,
|
||||
};
|
||||
}
|
||||
return config;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
10
package.json
10
package.json
|
|
@ -8,13 +8,16 @@
|
|||
"dev:raw": "next dev --turbopack -p 3003",
|
||||
"build": "next build",
|
||||
"build:bundle-guard": "node scripts/check-next-server-bundle.mjs",
|
||||
"check:compute-boundary": "node scripts/check-compute-boundary.mjs",
|
||||
"start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw",
|
||||
"start:raw": "next start -p 3003",
|
||||
"lint": "next lint",
|
||||
"test": "playwright test",
|
||||
"test:e2e": "playwright test",
|
||||
"test:unit": "vitest run",
|
||||
"test:compute": "vitest run --project compute-core --project compute-worker",
|
||||
"test:compute": "vitest run --project compute-worker",
|
||||
"compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts",
|
||||
"compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts",
|
||||
"migrate": "node drizzle/scripts/migrate.mjs",
|
||||
"migrate-fs": "node scripts/migrate-fs-v2.mjs",
|
||||
"migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true",
|
||||
|
|
@ -27,14 +30,13 @@
|
|||
"docs:clear": "pnpm --dir docs-site clear",
|
||||
"compute:worker:dev": "pnpm --filter @openreader/compute-worker dev",
|
||||
"compute:worker:start": "pnpm --filter @openreader/compute-worker start",
|
||||
"compute:dev:compose": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --build",
|
||||
"compute:dev:compose": "docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --build",
|
||||
"lint:route-errors": "node scripts/check-route-error-responses.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1061.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
||||
"@headlessui/react": "^2.2.10",
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"@microsoft/antissrf": "^1.0.0",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"@napi-rs/canvas": "^0.1.100",
|
||||
|
|
@ -61,7 +63,6 @@
|
|||
"linkedom": "^0.18.12",
|
||||
"lru-cache": "^11.5.1",
|
||||
"next": "^15.5.19",
|
||||
"onnxruntime-node": "^1.26.0",
|
||||
"openai": "^6.41.0",
|
||||
"pdfjs-dist": "4.8.69",
|
||||
"pg": "^8.21.0",
|
||||
|
|
@ -98,6 +99,7 @@
|
|||
"eslint": "^9.39.4",
|
||||
"eslint-config-next": "^15.5.19",
|
||||
"postcss": "^8.5.15",
|
||||
"openapi-typescript": "^7.10.1",
|
||||
"tailwindcss": "^3.4.19",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.1.8"
|
||||
|
|
|
|||
316
pnpm-lock.yaml
316
pnpm-lock.yaml
|
|
@ -20,9 +20,6 @@ importers:
|
|||
'@headlessui/react':
|
||||
specifier: ^2.2.10
|
||||
version: 2.2.10(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
|
||||
'@huggingface/tokenizers':
|
||||
specifier: ^0.1.3
|
||||
version: 0.1.3
|
||||
'@microsoft/antissrf':
|
||||
specifier: ^1.0.0
|
||||
version: 1.0.0
|
||||
|
|
@ -49,7 +46,7 @@ importers:
|
|||
version: 7.0.1
|
||||
better-auth:
|
||||
specifier: ^1.6.14
|
||||
version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)))
|
||||
version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)))
|
||||
better-sqlite3:
|
||||
specifier: ^12.10.0
|
||||
version: 12.10.0
|
||||
|
|
@ -101,9 +98,6 @@ importers:
|
|||
next:
|
||||
specifier: ^15.5.19
|
||||
version: 15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
|
||||
onnxruntime-node:
|
||||
specifier: ^1.26.0
|
||||
version: 1.26.0
|
||||
openai:
|
||||
specifier: ^6.41.0
|
||||
version: 6.41.0(zod@4.4.3)
|
||||
|
|
@ -170,7 +164,7 @@ importers:
|
|||
version: 1.60.0
|
||||
'@tailwindcss/typography':
|
||||
specifier: ^0.5.19
|
||||
version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4))
|
||||
version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))
|
||||
'@types/better-sqlite3':
|
||||
specifier: ^7.6.13
|
||||
version: 7.6.13
|
||||
|
|
@ -204,27 +198,48 @@ importers:
|
|||
eslint-config-next:
|
||||
specifier: ^15.5.19
|
||||
version: 15.5.19(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3)
|
||||
openapi-typescript:
|
||||
specifier: ^7.10.1
|
||||
version: 7.13.0(typescript@5.9.3)
|
||||
postcss:
|
||||
specifier: ^8.5.15
|
||||
version: 8.5.15
|
||||
tailwindcss:
|
||||
specifier: ^3.4.19
|
||||
version: 3.4.19(tsx@4.22.4)
|
||||
version: 3.4.19(tsx@4.22.4)(yaml@2.9.0)
|
||||
typescript:
|
||||
specifier: ^5.9.3
|
||||
version: 5.9.3
|
||||
vitest:
|
||||
specifier: ^4.1.8
|
||||
version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))
|
||||
version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))
|
||||
|
||||
compute/core:
|
||||
compute-worker:
|
||||
dependencies:
|
||||
'@aws-sdk/client-s3':
|
||||
specifier: ^3.1061.0
|
||||
version: 3.1061.0
|
||||
'@fastify/swagger':
|
||||
specifier: ^9.0.0
|
||||
version: 9.7.0
|
||||
'@huggingface/tokenizers':
|
||||
specifier: ^0.1.3
|
||||
version: 0.1.3
|
||||
'@napi-rs/canvas':
|
||||
specifier: ^0.1.100
|
||||
version: 0.1.100
|
||||
'@nats-io/jetstream':
|
||||
specifier: ^3.4.0
|
||||
version: 3.4.0
|
||||
'@nats-io/kv':
|
||||
specifier: ^3.4.0
|
||||
version: 3.4.0
|
||||
'@nats-io/transport-node':
|
||||
specifier: ^3.4.0
|
||||
version: 3.4.0
|
||||
fastify:
|
||||
specifier: ^5.8.5
|
||||
version: 5.8.5
|
||||
ffmpeg-static:
|
||||
specifier: ^5.3.0
|
||||
version: 5.3.0
|
||||
|
|
@ -237,27 +252,6 @@ importers:
|
|||
pdfjs-dist:
|
||||
specifier: 4.8.69
|
||||
version: 4.8.69
|
||||
|
||||
compute/worker:
|
||||
dependencies:
|
||||
'@aws-sdk/client-s3':
|
||||
specifier: ^3.1061.0
|
||||
version: 3.1061.0
|
||||
'@nats-io/jetstream':
|
||||
specifier: ^3.4.0
|
||||
version: 3.4.0
|
||||
'@nats-io/kv':
|
||||
specifier: ^3.4.0
|
||||
version: 3.4.0
|
||||
'@nats-io/transport-node':
|
||||
specifier: ^3.4.0
|
||||
version: 3.4.0
|
||||
'@openreader/compute-core':
|
||||
specifier: workspace:*
|
||||
version: link:../core
|
||||
fastify:
|
||||
specifier: ^5.8.5
|
||||
version: 5.8.5
|
||||
pino:
|
||||
specifier: ^10.3.1
|
||||
version: 10.3.1
|
||||
|
|
@ -424,6 +418,14 @@ packages:
|
|||
resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@babel/code-frame@7.29.7':
|
||||
resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
|
||||
'@babel/helper-validator-identifier@7.29.7':
|
||||
resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
|
||||
'@babel/runtime@7.29.7':
|
||||
resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
|
|
@ -1031,6 +1033,9 @@ packages:
|
|||
'@fastify/proxy-addr@5.1.0':
|
||||
resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==}
|
||||
|
||||
'@fastify/swagger@9.7.0':
|
||||
resolution: {integrity: sha512-Vp1SC1GC2Hrkd3faFILv86BzUNyFz5N4/xdExqtCgkGASOzn/x+eMe4qXIGq7cdT6wif/P/oa6r1Ruqx19paZA==}
|
||||
|
||||
'@floating-ui/core@1.7.5':
|
||||
resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==}
|
||||
|
||||
|
|
@ -1508,6 +1513,16 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1
|
||||
|
||||
'@redocly/ajv@8.11.2':
|
||||
resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==}
|
||||
|
||||
'@redocly/config@0.22.0':
|
||||
resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==}
|
||||
|
||||
'@redocly/openapi-core@1.34.15':
|
||||
resolution: {integrity: sha512-HAwCnNyKcs5XGQqms+9t7OdAPM/5TDstmhF+0i7tdCFato2QKuYIlyWETwkXd8c5zbltr1oB+6y9NTeQLr2d6Q==}
|
||||
engines: {node: '>=18.17.0', npm: '>=9.5.0'}
|
||||
|
||||
'@rolldown/binding-android-arm64@1.0.2':
|
||||
resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
|
|
@ -2036,6 +2051,10 @@ packages:
|
|||
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
|
||||
engines: {node: '>= 6.0.0'}
|
||||
|
||||
agent-base@7.1.4:
|
||||
resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==}
|
||||
engines: {node: '>= 14'}
|
||||
|
||||
ajv-formats@3.0.1:
|
||||
resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==}
|
||||
peerDependencies:
|
||||
|
|
@ -2050,6 +2069,10 @@ packages:
|
|||
ajv@8.20.0:
|
||||
resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==}
|
||||
|
||||
ansi-colors@4.1.3:
|
||||
resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
ansi-regex@5.0.1:
|
||||
resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -2384,6 +2407,9 @@ packages:
|
|||
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
change-case@5.4.4:
|
||||
resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==}
|
||||
|
||||
character-entities-html4@2.1.0:
|
||||
resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==}
|
||||
|
||||
|
|
@ -2421,6 +2447,9 @@ packages:
|
|||
color-name@1.1.4:
|
||||
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
|
||||
|
||||
colorette@1.4.0:
|
||||
resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==}
|
||||
|
||||
colorette@2.0.20:
|
||||
resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==}
|
||||
|
||||
|
|
@ -3011,6 +3040,9 @@ packages:
|
|||
resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==}
|
||||
hasBin: true
|
||||
|
||||
fastify-plugin@5.1.0:
|
||||
resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==}
|
||||
|
||||
fastify@5.8.5:
|
||||
resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==}
|
||||
|
||||
|
|
@ -3212,6 +3244,10 @@ packages:
|
|||
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
https-proxy-agent@7.0.6:
|
||||
resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}
|
||||
engines: {node: '>= 14'}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
|
|
@ -3234,6 +3270,10 @@ packages:
|
|||
resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==}
|
||||
engines: {node: '>=0.8.19'}
|
||||
|
||||
index-to-position@1.2.0:
|
||||
resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
inherits@2.0.4:
|
||||
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
||||
|
||||
|
|
@ -3413,9 +3453,17 @@ packages:
|
|||
resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
js-levenshtein@1.1.6:
|
||||
resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
js-tokens@4.0.0:
|
||||
resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==}
|
||||
|
||||
js-yaml@4.1.1:
|
||||
resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==}
|
||||
hasBin: true
|
||||
|
||||
js-yaml@4.2.0:
|
||||
resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==}
|
||||
hasBin: true
|
||||
|
|
@ -3426,6 +3474,10 @@ packages:
|
|||
json-schema-ref-resolver@3.0.0:
|
||||
resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==}
|
||||
|
||||
json-schema-resolver@3.0.0:
|
||||
resolution: {integrity: sha512-HqMnbz0tz2DaEJ3ntsqtx3ezzZyDE7G56A/pPY/NGmrPu76UzsWquOpHFRAf5beTNXoH2LU5cQePVvRli1nchA==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
json-schema-traverse@0.4.1:
|
||||
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
|
||||
|
||||
|
|
@ -3927,6 +3979,15 @@ packages:
|
|||
zod:
|
||||
optional: true
|
||||
|
||||
openapi-types@12.1.3:
|
||||
resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==}
|
||||
|
||||
openapi-typescript@7.13.0:
|
||||
resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
typescript: ^5.x
|
||||
|
||||
optionator@0.9.4:
|
||||
resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
|
||||
engines: {node: '>= 0.8.0'}
|
||||
|
|
@ -3963,6 +4024,10 @@ packages:
|
|||
parse-entities@4.0.2:
|
||||
resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
|
||||
|
||||
parse-json@8.3.0:
|
||||
resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
path-exists@4.0.0:
|
||||
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -4073,6 +4138,10 @@ packages:
|
|||
engines: {node: '>=18'}
|
||||
hasBin: true
|
||||
|
||||
pluralize@8.0.0:
|
||||
resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==}
|
||||
engines: {node: '>=4'}
|
||||
|
||||
possible-typed-array-names@1.1.0:
|
||||
resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
|
@ -4615,6 +4684,10 @@ packages:
|
|||
suffix-thumb@5.0.2:
|
||||
resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==}
|
||||
|
||||
supports-color@10.2.2:
|
||||
resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
supports-color@7.2.0:
|
||||
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -4728,6 +4801,10 @@ packages:
|
|||
resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
type-fest@4.41.0:
|
||||
resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
type@2.7.3:
|
||||
resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==}
|
||||
|
||||
|
|
@ -4786,6 +4863,9 @@ packages:
|
|||
unrs-resolver@1.12.2:
|
||||
resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==}
|
||||
|
||||
uri-js-replace@1.0.1:
|
||||
resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==}
|
||||
|
||||
uri-js@4.4.1:
|
||||
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
|
||||
|
||||
|
|
@ -4943,6 +5023,18 @@ packages:
|
|||
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
|
||||
engines: {node: '>=0.4'}
|
||||
|
||||
yaml-ast-parser@0.0.43:
|
||||
resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==}
|
||||
|
||||
yaml@2.9.0:
|
||||
resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==}
|
||||
engines: {node: '>= 14.6'}
|
||||
hasBin: true
|
||||
|
||||
yargs-parser@21.1.1:
|
||||
resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
yocto-queue@0.1.0:
|
||||
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
|
||||
engines: {node: '>=10'}
|
||||
|
|
@ -5224,6 +5316,14 @@ snapshots:
|
|||
|
||||
'@aws/lambda-invoke-store@0.2.4': {}
|
||||
|
||||
'@babel/code-frame@7.29.7':
|
||||
dependencies:
|
||||
'@babel/helper-validator-identifier': 7.29.7
|
||||
js-tokens: 4.0.0
|
||||
picocolors: 1.1.1
|
||||
|
||||
'@babel/helper-validator-identifier@7.29.7': {}
|
||||
|
||||
'@babel/runtime@7.29.7': {}
|
||||
|
||||
'@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)':
|
||||
|
|
@ -5546,7 +5646,7 @@ snapshots:
|
|||
'@eslint/config-array@0.21.2':
|
||||
dependencies:
|
||||
'@eslint/object-schema': 2.1.7
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
minimatch: 3.1.5
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
|
@ -5562,7 +5662,7 @@ snapshots:
|
|||
'@eslint/eslintrc@3.3.5':
|
||||
dependencies:
|
||||
ajv: 6.15.0
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
espree: 10.4.0
|
||||
globals: 14.0.0
|
||||
ignore: 5.3.2
|
||||
|
|
@ -5605,6 +5705,16 @@ snapshots:
|
|||
'@fastify/forwarded': 3.0.1
|
||||
ipaddr.js: 2.4.0
|
||||
|
||||
'@fastify/swagger@9.7.0':
|
||||
dependencies:
|
||||
fastify-plugin: 5.1.0
|
||||
json-schema-resolver: 3.0.0
|
||||
openapi-types: 12.1.3
|
||||
rfdc: 1.4.1
|
||||
yaml: 2.9.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@floating-ui/core@1.7.5':
|
||||
dependencies:
|
||||
'@floating-ui/utils': 0.2.11
|
||||
|
|
@ -5968,6 +6078,29 @@ snapshots:
|
|||
dependencies:
|
||||
react: 19.2.7
|
||||
|
||||
'@redocly/ajv@8.11.2':
|
||||
dependencies:
|
||||
fast-deep-equal: 3.1.3
|
||||
json-schema-traverse: 1.0.0
|
||||
require-from-string: 2.0.2
|
||||
uri-js-replace: 1.0.1
|
||||
|
||||
'@redocly/config@0.22.0': {}
|
||||
|
||||
'@redocly/openapi-core@1.34.15(supports-color@10.2.2)':
|
||||
dependencies:
|
||||
'@redocly/ajv': 8.11.2
|
||||
'@redocly/config': 0.22.0
|
||||
colorette: 1.4.0
|
||||
https-proxy-agent: 7.0.6(supports-color@10.2.2)
|
||||
js-levenshtein: 1.1.6
|
||||
js-yaml: 4.1.1
|
||||
minimatch: 5.1.9
|
||||
pluralize: 8.0.0
|
||||
yaml-ast-parser: 0.0.43
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@rolldown/binding-android-arm64@1.0.2':
|
||||
optional: true
|
||||
|
||||
|
|
@ -6088,10 +6221,10 @@ snapshots:
|
|||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4))':
|
||||
'@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
postcss-selector-parser: 6.0.10
|
||||
tailwindcss: 3.4.19(tsx@4.22.4)
|
||||
tailwindcss: 3.4.19(tsx@4.22.4)(yaml@2.9.0)
|
||||
|
||||
'@tanstack/query-core@5.101.0': {}
|
||||
|
||||
|
|
@ -6220,7 +6353,7 @@ snapshots:
|
|||
'@typescript-eslint/types': 8.60.1
|
||||
'@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3)
|
||||
'@typescript-eslint/visitor-keys': 8.60.1
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
eslint: 9.39.4(jiti@1.21.7)
|
||||
typescript: 5.9.3
|
||||
transitivePeerDependencies:
|
||||
|
|
@ -6230,7 +6363,7 @@ snapshots:
|
|||
dependencies:
|
||||
'@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3)
|
||||
'@typescript-eslint/types': 8.60.1
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
typescript: 5.9.3
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
|
@ -6249,7 +6382,7 @@ snapshots:
|
|||
'@typescript-eslint/types': 8.60.1
|
||||
'@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3)
|
||||
'@typescript-eslint/utils': 8.60.1(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3)
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
eslint: 9.39.4(jiti@1.21.7)
|
||||
ts-api-utils: 2.5.0(typescript@5.9.3)
|
||||
typescript: 5.9.3
|
||||
|
|
@ -6264,7 +6397,7 @@ snapshots:
|
|||
'@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3)
|
||||
'@typescript-eslint/types': 8.60.1
|
||||
'@typescript-eslint/visitor-keys': 8.60.1
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
minimatch: 10.2.5
|
||||
semver: 7.8.1
|
||||
tinyglobby: 0.2.17
|
||||
|
|
@ -6375,13 +6508,13 @@ snapshots:
|
|||
chai: 6.2.2
|
||||
tinyrainbow: 3.1.0
|
||||
|
||||
'@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))':
|
||||
'@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
'@vitest/spy': 4.1.8
|
||||
estree-walker: 3.0.3
|
||||
magic-string: 0.30.21
|
||||
optionalDependencies:
|
||||
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)
|
||||
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)
|
||||
|
||||
'@vitest/pretty-format@4.1.8':
|
||||
dependencies:
|
||||
|
|
@ -6425,10 +6558,12 @@ snapshots:
|
|||
|
||||
agent-base@6.0.2:
|
||||
dependencies:
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
agent-base@7.1.4: {}
|
||||
|
||||
ajv-formats@3.0.1(ajv@8.20.0):
|
||||
optionalDependencies:
|
||||
ajv: 8.20.0
|
||||
|
|
@ -6447,6 +6582,8 @@ snapshots:
|
|||
json-schema-traverse: 1.0.0
|
||||
require-from-string: 2.0.2
|
||||
|
||||
ansi-colors@4.1.3: {}
|
||||
|
||||
ansi-regex@5.0.1: {}
|
||||
|
||||
ansi-regex@6.2.2: {}
|
||||
|
|
@ -6632,7 +6769,7 @@ snapshots:
|
|||
|
||||
base64-js@1.5.1: {}
|
||||
|
||||
better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))):
|
||||
better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))):
|
||||
dependencies:
|
||||
'@better-auth/core': 1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)
|
||||
'@better-auth/drizzle-adapter': 1.6.14(@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.1)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))
|
||||
|
|
@ -6659,7 +6796,7 @@ snapshots:
|
|||
pg: 8.21.0
|
||||
react: 19.2.7
|
||||
react-dom: 19.2.7(react@19.2.7)
|
||||
vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))
|
||||
vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))
|
||||
transitivePeerDependencies:
|
||||
- '@cloudflare/workers-types'
|
||||
- '@opentelemetry/api'
|
||||
|
|
@ -6765,6 +6902,8 @@ snapshots:
|
|||
ansi-styles: 4.3.0
|
||||
supports-color: 7.2.0
|
||||
|
||||
change-case@5.4.4: {}
|
||||
|
||||
character-entities-html4@2.1.0: {}
|
||||
|
||||
character-entities-legacy@3.0.0: {}
|
||||
|
|
@ -6799,6 +6938,8 @@ snapshots:
|
|||
|
||||
color-name@1.1.4: {}
|
||||
|
||||
colorette@1.4.0: {}
|
||||
|
||||
colorette@2.0.20: {}
|
||||
|
||||
comma-separated-tokens@2.0.3: {}
|
||||
|
|
@ -6896,9 +7037,11 @@ snapshots:
|
|||
dependencies:
|
||||
ms: 2.1.3
|
||||
|
||||
debug@4.4.3:
|
||||
debug@4.4.3(supports-color@10.2.2):
|
||||
dependencies:
|
||||
ms: 2.1.3
|
||||
optionalDependencies:
|
||||
supports-color: 10.2.2
|
||||
|
||||
decode-named-character-reference@1.3.0:
|
||||
dependencies:
|
||||
|
|
@ -7265,7 +7408,7 @@ snapshots:
|
|||
eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@1.21.7)):
|
||||
dependencies:
|
||||
'@nolyfill/is-core-module': 1.0.39
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
eslint: 9.39.4(jiti@1.21.7)
|
||||
get-tsconfig: 4.14.0
|
||||
is-bun-module: 2.0.0
|
||||
|
|
@ -7390,7 +7533,7 @@ snapshots:
|
|||
ajv: 6.15.0
|
||||
chalk: 4.1.2
|
||||
cross-spawn: 7.0.6
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
escape-string-regexp: 4.0.0
|
||||
eslint-scope: 8.4.0
|
||||
eslint-visitor-keys: 4.2.1
|
||||
|
|
@ -7535,6 +7678,8 @@ snapshots:
|
|||
strnum: 2.3.0
|
||||
xml-naming: 0.1.0
|
||||
|
||||
fastify-plugin@5.1.0: {}
|
||||
|
||||
fastify@5.8.5:
|
||||
dependencies:
|
||||
'@fastify/ajv-compiler': 4.0.5
|
||||
|
|
@ -7777,7 +7922,14 @@ snapshots:
|
|||
https-proxy-agent@5.0.1:
|
||||
dependencies:
|
||||
agent-base: 6.0.2
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
https-proxy-agent@7.0.6(supports-color@10.2.2):
|
||||
dependencies:
|
||||
agent-base: 7.1.4
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
|
|
@ -7796,6 +7948,8 @@ snapshots:
|
|||
|
||||
imurmurhash@0.1.4: {}
|
||||
|
||||
index-to-position@1.2.0: {}
|
||||
|
||||
inherits@2.0.4: {}
|
||||
|
||||
ini@1.3.8: {}
|
||||
|
|
@ -7972,8 +8126,14 @@ snapshots:
|
|||
|
||||
joycon@3.1.1: {}
|
||||
|
||||
js-levenshtein@1.1.6: {}
|
||||
|
||||
js-tokens@4.0.0: {}
|
||||
|
||||
js-yaml@4.1.1:
|
||||
dependencies:
|
||||
argparse: 2.0.1
|
||||
|
||||
js-yaml@4.2.0:
|
||||
dependencies:
|
||||
argparse: 2.0.1
|
||||
|
|
@ -7984,6 +8144,14 @@ snapshots:
|
|||
dependencies:
|
||||
dequal: 2.0.3
|
||||
|
||||
json-schema-resolver@3.0.0:
|
||||
dependencies:
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
fast-uri: 3.1.2
|
||||
rfdc: 1.4.1
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
json-schema-traverse@0.4.1: {}
|
||||
|
||||
json-schema-traverse@1.0.0: {}
|
||||
|
|
@ -8482,7 +8650,7 @@ snapshots:
|
|||
micromark@4.0.2:
|
||||
dependencies:
|
||||
'@types/debug': 4.1.13
|
||||
debug: 4.4.3
|
||||
debug: 4.4.3(supports-color@10.2.2)
|
||||
decode-named-character-reference: 1.3.0
|
||||
devlop: 1.1.0
|
||||
micromark-core-commonmark: 2.0.3
|
||||
|
|
@ -8658,6 +8826,18 @@ snapshots:
|
|||
optionalDependencies:
|
||||
zod: 4.4.3
|
||||
|
||||
openapi-types@12.1.3: {}
|
||||
|
||||
openapi-typescript@7.13.0(typescript@5.9.3):
|
||||
dependencies:
|
||||
'@redocly/openapi-core': 1.34.15(supports-color@10.2.2)
|
||||
ansi-colors: 4.1.3
|
||||
change-case: 5.4.4
|
||||
parse-json: 8.3.0
|
||||
supports-color: 10.2.2
|
||||
typescript: 5.9.3
|
||||
yargs-parser: 21.1.1
|
||||
|
||||
optionator@0.9.4:
|
||||
dependencies:
|
||||
deep-is: 0.1.4
|
||||
|
|
@ -8705,6 +8885,12 @@ snapshots:
|
|||
is-decimal: 2.0.1
|
||||
is-hexadecimal: 2.0.1
|
||||
|
||||
parse-json@8.3.0:
|
||||
dependencies:
|
||||
'@babel/code-frame': 7.29.7
|
||||
index-to-position: 1.2.0
|
||||
type-fest: 4.41.0
|
||||
|
||||
path-exists@4.0.0: {}
|
||||
|
||||
path-expression-matcher@1.5.0: {}
|
||||
|
|
@ -8819,6 +9005,8 @@ snapshots:
|
|||
optionalDependencies:
|
||||
fsevents: 2.3.2
|
||||
|
||||
pluralize@8.0.0: {}
|
||||
|
||||
possible-typed-array-names@1.1.0: {}
|
||||
|
||||
postcss-import@15.1.0(postcss@8.5.15):
|
||||
|
|
@ -8833,13 +9021,14 @@ snapshots:
|
|||
camelcase-css: 2.0.1
|
||||
postcss: 8.5.15
|
||||
|
||||
postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4):
|
||||
postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0):
|
||||
dependencies:
|
||||
lilconfig: 3.1.3
|
||||
optionalDependencies:
|
||||
jiti: 1.21.7
|
||||
postcss: 8.5.15
|
||||
tsx: 4.22.4
|
||||
yaml: 2.9.0
|
||||
|
||||
postcss-nested@6.2.0(postcss@8.5.15):
|
||||
dependencies:
|
||||
|
|
@ -9504,6 +9693,8 @@ snapshots:
|
|||
|
||||
suffix-thumb@5.0.2: {}
|
||||
|
||||
supports-color@10.2.2: {}
|
||||
|
||||
supports-color@7.2.0:
|
||||
dependencies:
|
||||
has-flag: 4.0.0
|
||||
|
|
@ -9512,7 +9703,7 @@ snapshots:
|
|||
|
||||
tabbable@6.4.0: {}
|
||||
|
||||
tailwindcss@3.4.19(tsx@4.22.4):
|
||||
tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0):
|
||||
dependencies:
|
||||
'@alloc/quick-lru': 5.2.0
|
||||
arg: 5.0.2
|
||||
|
|
@ -9531,7 +9722,7 @@ snapshots:
|
|||
postcss: 8.5.15
|
||||
postcss-import: 15.1.0(postcss@8.5.15)
|
||||
postcss-js: 4.1.0(postcss@8.5.15)
|
||||
postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)
|
||||
postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0)
|
||||
postcss-nested: 6.2.0(postcss@8.5.15)
|
||||
postcss-selector-parser: 6.1.2
|
||||
resolve: 1.22.12
|
||||
|
|
@ -9651,6 +9842,8 @@ snapshots:
|
|||
|
||||
type-fest@0.20.2: {}
|
||||
|
||||
type-fest@4.41.0: {}
|
||||
|
||||
type@2.7.3: {}
|
||||
|
||||
typed-array-buffer@1.0.3:
|
||||
|
|
@ -9761,6 +9954,8 @@ snapshots:
|
|||
'@unrs/resolver-binding-win32-ia32-msvc': 1.12.2
|
||||
'@unrs/resolver-binding-win32-x64-msvc': 1.12.2
|
||||
|
||||
uri-js-replace@1.0.1: {}
|
||||
|
||||
uri-js@4.4.1:
|
||||
dependencies:
|
||||
punycode: 2.3.1
|
||||
|
|
@ -9783,7 +9978,7 @@ snapshots:
|
|||
'@types/unist': 3.0.3
|
||||
vfile-message: 4.0.3
|
||||
|
||||
vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4):
|
||||
vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0):
|
||||
dependencies:
|
||||
lightningcss: 1.32.0
|
||||
picomatch: 4.0.4
|
||||
|
|
@ -9796,11 +9991,12 @@ snapshots:
|
|||
fsevents: 2.3.3
|
||||
jiti: 1.21.7
|
||||
tsx: 4.22.4
|
||||
yaml: 2.9.0
|
||||
|
||||
vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)):
|
||||
vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)):
|
||||
dependencies:
|
||||
'@vitest/expect': 4.1.8
|
||||
'@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))
|
||||
'@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))
|
||||
'@vitest/pretty-format': 4.1.8
|
||||
'@vitest/runner': 4.1.8
|
||||
'@vitest/snapshot': 4.1.8
|
||||
|
|
@ -9817,7 +10013,7 @@ snapshots:
|
|||
tinyexec: 1.2.4
|
||||
tinyglobby: 0.2.17
|
||||
tinyrainbow: 3.1.0
|
||||
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)
|
||||
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)
|
||||
why-is-node-running: 2.3.0
|
||||
optionalDependencies:
|
||||
'@types/node': 20.19.41
|
||||
|
|
@ -9898,6 +10094,12 @@ snapshots:
|
|||
|
||||
xtend@4.0.2: {}
|
||||
|
||||
yaml-ast-parser@0.0.43: {}
|
||||
|
||||
yaml@2.9.0: {}
|
||||
|
||||
yargs-parser@21.1.1: {}
|
||||
|
||||
yocto-queue@0.1.0: {}
|
||||
|
||||
zip-stream@6.0.1:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
packages:
|
||||
- .
|
||||
- compute/*
|
||||
- compute-worker
|
||||
- docker/*
|
||||
|
||||
allowBuilds:
|
||||
|
|
|
|||
39
scripts/check-compute-boundary.mjs
Normal file
39
scripts/check-compute-boundary.mjs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const root = process.cwd();
|
||||
const forbidden = [
|
||||
'@openreader/compute-core',
|
||||
'@openreader/compute-worker',
|
||||
'compute/core',
|
||||
'compute/worker',
|
||||
'compute-worker/src',
|
||||
];
|
||||
const failures = [];
|
||||
|
||||
function walk(dir) {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile() || !/\.(ts|tsx|js|mjs|cjs)$/.test(entry.name)) continue;
|
||||
const text = fs.readFileSync(fullPath, 'utf8');
|
||||
for (const pattern of forbidden) {
|
||||
if (text.includes(pattern)) failures.push({ file: fullPath, pattern });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(path.join(root, 'src'));
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error('[compute-boundary] Forbidden app-to-worker source coupling detected:');
|
||||
for (const failure of failures) {
|
||||
console.error(`- ${failure.pattern} in ${path.relative(root, failure.file)}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.info('[compute-boundary] OK: app communicates with compute worker through protocol/client only');
|
||||
|
|
@ -12,7 +12,8 @@ if (!fs.existsSync(serverDir)) {
|
|||
const forbidden = [
|
||||
'onnxruntime-node',
|
||||
'@huggingface/tokenizers',
|
||||
'@openreader/compute-core/local-runtime',
|
||||
'@openreader/compute-worker',
|
||||
'/compute-worker/src/compute/',
|
||||
];
|
||||
|
||||
const includeExt = new Set(['.js', '.mjs', '.cjs']);
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ function spawnMainCommand(command, env) {
|
|||
function resolveEmbeddedWorkerLaunch() {
|
||||
const candidateDirs = [
|
||||
path.join(process.cwd(), 'embedded-compute-worker'),
|
||||
path.join(process.cwd(), 'compute', 'worker'),
|
||||
path.join(process.cwd(), 'compute-worker'),
|
||||
];
|
||||
|
||||
for (const workerDir of candidateDirs) {
|
||||
|
|
@ -237,7 +237,7 @@ function resolveEmbeddedWorkerLaunch() {
|
|||
|
||||
throw new Error(
|
||||
'Could not find an embedded compute worker runtime. '
|
||||
+ 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute/worker available locally.',
|
||||
+ 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute-worker available locally.',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { and, eq, inArray } from 'drizzle-orm';
|
|||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker';
|
||||
import { getComputeWorkerClient } from '@/lib/server/compute-worker/client';
|
||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
fetchPdfParseOperation,
|
||||
|
|
@ -45,14 +45,6 @@ async function loadOwnedDocumentRow(input: {
|
|||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
function workerEventsUrl(baseUrl: string, opId: string, sinceEventId: string | null): string {
|
||||
const url = new URL(`${baseUrl}/ops/${encodeURIComponent(opId)}/events`);
|
||||
if (sinceEventId) {
|
||||
url.searchParams.set('sinceEventId', sinceEventId);
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||
const { logger } = createRequestLogger({
|
||||
route: '/api/documents/[id]/parsed/events',
|
||||
|
|
@ -92,17 +84,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
|||
return NextResponse.json({ error: 'Operation not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const cfg = getWorkerClientConfigFromEnv();
|
||||
const lastEventId = req.headers.get('last-event-id');
|
||||
const sinceEventId = req.nextUrl.searchParams.get('sinceEventId') || lastEventId;
|
||||
const upstream = await fetch(workerEventsUrl(cfg.baseUrl, opId, sinceEventId), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${cfg.token}`,
|
||||
Accept: 'text/event-stream',
|
||||
...(lastEventId ? { 'Last-Event-ID': lastEventId } : {}),
|
||||
},
|
||||
cache: 'no-store',
|
||||
const upstream = await getComputeWorkerClient().openOperationEvents(opId, {
|
||||
sinceEventId,
|
||||
lastEventId,
|
||||
signal: req.signal,
|
||||
});
|
||||
|
||||
|
|
|
|||
249
src/lib/server/compute-worker/client.ts
Normal file
249
src/lib/server/compute-worker/client.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
import type {
|
||||
PdfLayoutRequest,
|
||||
PdfLayoutResolution,
|
||||
PdfLayoutResult,
|
||||
WhisperAlignRequest,
|
||||
WhisperAlignResult,
|
||||
WorkerOperation,
|
||||
WorkerOperationEvent,
|
||||
} from './protocol';
|
||||
import { isWorkerOperation } from './protocol';
|
||||
|
||||
class WorkerHttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly retryAfterMs: number | null = null,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'WorkerHttpError';
|
||||
}
|
||||
}
|
||||
|
||||
function readRequiredEnv(name: string): string {
|
||||
const value = process.env[name]?.trim();
|
||||
if (!value) throw new Error(`${name} is required for compute worker client`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeWorkerBaseUrl(raw: string): string {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty');
|
||||
const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed)
|
||||
? trimmed
|
||||
: (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed)
|
||||
? `http://${trimmed}`
|
||||
: `https://${trimmed}`);
|
||||
const parsed = new URL(withScheme);
|
||||
parsed.pathname = parsed.pathname.replace(/\/+$/, '');
|
||||
return parsed.toString().replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
export function getComputeWorkerConfigFromEnv(): { baseUrl: string; token: string } {
|
||||
return {
|
||||
baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')),
|
||||
token: readRequiredEnv('COMPUTE_WORKER_TOKEN'),
|
||||
};
|
||||
}
|
||||
|
||||
export function isComputeWorkerAvailable(): boolean {
|
||||
try {
|
||||
getComputeWorkerConfigFromEnv();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readPositiveIntEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
|
||||
}
|
||||
|
||||
function alignmentWaitTimeoutMs(): number {
|
||||
return Math.max(60_000, readPositiveIntEnv('COMPUTE_WHISPER_TIMEOUT_MS', 30_000) + 15_000);
|
||||
}
|
||||
|
||||
function parseRetryAfterMs(value: string | null): number | null {
|
||||
if (!value) return null;
|
||||
const seconds = Number(value);
|
||||
if (Number.isFinite(seconds)) return Math.max(0, Math.floor(seconds * 1000));
|
||||
const date = Date.parse(value);
|
||||
return Number.isNaN(date) ? null : Math.max(0, date - Date.now());
|
||||
}
|
||||
|
||||
function parseSseFrame(frame: string): { id: number | null; data: string | null } {
|
||||
let id: number | null = null;
|
||||
const data: string[] = [];
|
||||
for (const line of frame.split(/\r?\n/)) {
|
||||
if (line.startsWith('id:')) {
|
||||
const parsed = Number(line.slice(3).trim());
|
||||
if (Number.isFinite(parsed) && parsed >= 0) id = Math.floor(parsed);
|
||||
} else if (line.startsWith('data:')) {
|
||||
data.push(line.slice(5).trimStart());
|
||||
}
|
||||
}
|
||||
return { id, data: data.length > 0 ? data.join('\n') : null };
|
||||
}
|
||||
|
||||
function isTerminal(operation: WorkerOperation): boolean {
|
||||
return operation.status === 'succeeded' || operation.status === 'failed';
|
||||
}
|
||||
|
||||
export class ComputeWorkerClient {
|
||||
private readonly baseUrl: string;
|
||||
private readonly token: string;
|
||||
|
||||
constructor(config = getComputeWorkerConfigFromEnv()) {
|
||||
this.baseUrl = config.baseUrl;
|
||||
this.token = config.token;
|
||||
}
|
||||
|
||||
async alignWords(input: WhisperAlignRequest): Promise<WhisperAlignResult> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
||||
try {
|
||||
const operation = await this.requestJson<WorkerOperation<WhisperAlignResult>>(
|
||||
'POST',
|
||||
'/v1/whisper-align/operations',
|
||||
input,
|
||||
);
|
||||
const final = isTerminal(operation)
|
||||
? operation
|
||||
: await this.waitForOperation<WhisperAlignResult>(operation.opId, alignmentWaitTimeoutMs());
|
||||
if (final.status !== 'succeeded' || !final.result) {
|
||||
throw new Error(final.error?.message || 'Whisper worker operation did not complete');
|
||||
}
|
||||
return final.result;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt >= 2 || !this.shouldRetry(error)) break;
|
||||
const delay = error instanceof WorkerHttpError && error.retryAfterMs !== null
|
||||
? error.retryAfterMs
|
||||
: attempt * 250;
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error('Unknown compute worker failure');
|
||||
}
|
||||
|
||||
createPdfLayoutOperation(input: PdfLayoutRequest): Promise<WorkerOperation<PdfLayoutResult>> {
|
||||
return this.requestJson('POST', '/v1/pdf-layout/operations', input);
|
||||
}
|
||||
|
||||
resolvePdfLayout(input: PdfLayoutRequest): Promise<PdfLayoutResolution> {
|
||||
return this.requestJson('POST', '/v1/pdf-layout/resolve', {
|
||||
documentId: input.documentId,
|
||||
namespace: input.namespace,
|
||||
documentObjectKey: input.documentObjectKey,
|
||||
});
|
||||
}
|
||||
|
||||
async getOperation<Result>(opId: string): Promise<WorkerOperation<Result> | null> {
|
||||
try {
|
||||
return await this.requestJson('GET', `/v1/operations/${encodeURIComponent(opId)}`);
|
||||
} catch (error) {
|
||||
if (error instanceof WorkerHttpError && error.status === 404) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async openOperationEvents(opId: string, input?: {
|
||||
sinceEventId?: string | null;
|
||||
lastEventId?: string | null;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<Response> {
|
||||
const url = new URL(`${this.baseUrl}/v1/operations/${encodeURIComponent(opId)}/events`);
|
||||
if (input?.sinceEventId) url.searchParams.set('sinceEventId', input.sinceEventId);
|
||||
return fetch(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
Accept: 'text/event-stream',
|
||||
...(input?.lastEventId ? { 'Last-Event-ID': input.lastEventId } : {}),
|
||||
},
|
||||
cache: 'no-store',
|
||||
signal: input?.signal,
|
||||
});
|
||||
}
|
||||
|
||||
private async requestJson<T>(method: 'GET' | 'POST', path: string, body?: unknown): Promise<T> {
|
||||
const response = await fetch(`${this.baseUrl}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
Accept: 'application/json',
|
||||
...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}),
|
||||
},
|
||||
...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}),
|
||||
cache: 'no-store',
|
||||
});
|
||||
if (!response.ok) {
|
||||
const detail = await response.text().catch(() => '');
|
||||
throw new WorkerHttpError(
|
||||
`Worker request failed (${method} ${path}): ${response.status}${detail ? ` ${detail}` : ''}`,
|
||||
response.status,
|
||||
parseRetryAfterMs(response.headers.get('retry-after')),
|
||||
);
|
||||
}
|
||||
const parsed = await response.json() as T;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
private shouldRetry(error: unknown): boolean {
|
||||
if (error instanceof WorkerHttpError) {
|
||||
return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504;
|
||||
}
|
||||
return error instanceof Error && /network|timeout|fetch failed/i.test(error.message);
|
||||
}
|
||||
|
||||
private async waitForOperation<Result>(opId: string, timeoutMs: number): Promise<WorkerOperation<Result>> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
let lastEventId: number | null = null;
|
||||
try {
|
||||
while (!controller.signal.aborted) {
|
||||
const response = await this.openOperationEvents(opId, {
|
||||
sinceEventId: lastEventId === null ? null : String(lastEventId),
|
||||
lastEventId: lastEventId === null ? null : String(lastEventId),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok || !response.body) {
|
||||
throw new WorkerHttpError(`Worker event stream failed: ${response.status}`, response.status);
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
while (true) {
|
||||
const end = buffer.indexOf('\n\n');
|
||||
if (end < 0) break;
|
||||
const frame = parseSseFrame(buffer.slice(0, end));
|
||||
buffer = buffer.slice(end + 2);
|
||||
if (frame.id !== null) lastEventId = frame.id;
|
||||
if (!frame.data) continue;
|
||||
const event = JSON.parse(frame.data) as WorkerOperationEvent<Result>;
|
||||
if (!event?.snapshot || !isWorkerOperation(event.snapshot) || event.snapshot.opId !== opId) continue;
|
||||
if (isTerminal(event.snapshot)) return event.snapshot;
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
throw new Error(`Operation stream timed out for ${opId}`);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let client: ComputeWorkerClient | null = null;
|
||||
|
||||
export function getComputeWorkerClient(): ComputeWorkerClient {
|
||||
client ??= new ComputeWorkerClient();
|
||||
return client;
|
||||
}
|
||||
700
src/lib/server/compute-worker/generated.ts
Normal file
700
src/lib/server/compute-worker/generated.ts
Normal file
|
|
@ -0,0 +1,700 @@
|
|||
/**
|
||||
* This file was auto-generated by openapi-typescript.
|
||||
* Do not make direct changes to the file.
|
||||
*/
|
||||
|
||||
export interface paths {
|
||||
"/health/live": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Default Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
ok: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/health/ready": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Default Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
ok: boolean;
|
||||
natsConnected: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/whisper-align/operations": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": {
|
||||
text: string;
|
||||
lang?: string;
|
||||
cacheKey?: string;
|
||||
audioObjectKey: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Default Response */
|
||||
202: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
400: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/pdf-layout/operations": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": {
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
documentObjectKey: string;
|
||||
replaceToken?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Default Response */
|
||||
202: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
400: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/pdf-layout/resolve": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": {
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
documentObjectKey: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Default Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
artifact: {
|
||||
objectKey: string;
|
||||
} | null;
|
||||
operation: {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
} | null;
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
400: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/operations/{opId}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
opId: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Default Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
400: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
404: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/operations/{opId}/events": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
sinceEventId?: string | number;
|
||||
};
|
||||
header?: never;
|
||||
path: {
|
||||
opId: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Server-sent PublicOperationEvent stream */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": string;
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
400: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
/** @description Default Response */
|
||||
404: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
}
|
||||
export type webhooks = Record<string, never>;
|
||||
export interface components {
|
||||
schemas: {
|
||||
ParsedPdfDocument: {
|
||||
/** @enum {number} */
|
||||
schemaVersion: 1;
|
||||
documentId: string;
|
||||
parserVersion: string;
|
||||
parsedAt: number;
|
||||
pages: {
|
||||
pageNumber: number;
|
||||
width: number;
|
||||
height: number;
|
||||
blocks: {
|
||||
id: string;
|
||||
/** @enum {string} */
|
||||
kind: "abstract" | "algorithm" | "aside_text" | "chart" | "content" | "formula" | "doc_title" | "figure_title" | "footer" | "footnote" | "formula_number" | "header" | "image" | "number" | "paragraph_title" | "reference" | "reference_content" | "seal" | "table" | "text" | "vision_footnote";
|
||||
fragments: {
|
||||
page: number;
|
||||
bbox: [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number
|
||||
];
|
||||
text: string;
|
||||
readingOrder: number;
|
||||
modelConfidence?: number;
|
||||
}[];
|
||||
text: string;
|
||||
parentSectionId?: string;
|
||||
}[];
|
||||
}[];
|
||||
};
|
||||
TTSSentenceAlignment: {
|
||||
sentence: string;
|
||||
sentenceIndex: number;
|
||||
words: {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
charStart: number;
|
||||
charEnd: number;
|
||||
}[];
|
||||
};
|
||||
ArtifactReference: {
|
||||
objectKey: string;
|
||||
};
|
||||
ErrorResponse: {
|
||||
error: string;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
OperationError: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
PdfLayoutProgress: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
PublicOperation: {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
};
|
||||
PublicOperationEvent: {
|
||||
eventId: number;
|
||||
snapshot: {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
};
|
||||
};
|
||||
PdfLayoutResolution: {
|
||||
artifact: {
|
||||
objectKey: string;
|
||||
} | null;
|
||||
operation: {
|
||||
opId: string;
|
||||
subject: {
|
||||
/** @enum {string} */
|
||||
kind: "whisper_align";
|
||||
} | {
|
||||
/** @enum {string} */
|
||||
kind: "pdf_layout";
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
queuedAt: number;
|
||||
updatedAt: number;
|
||||
startedAt?: number;
|
||||
result?: unknown;
|
||||
error?: {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
timing?: {
|
||||
queueWaitMs?: number;
|
||||
s3FetchMs?: number;
|
||||
computeMs?: number;
|
||||
};
|
||||
progress?: {
|
||||
totalPages: number;
|
||||
pagesParsed: number;
|
||||
currentPage?: number;
|
||||
/** @enum {string} */
|
||||
phase: "infer" | "merge";
|
||||
};
|
||||
} | null;
|
||||
};
|
||||
};
|
||||
responses: never;
|
||||
parameters: never;
|
||||
requestBodies: never;
|
||||
headers: never;
|
||||
pathItems: never;
|
||||
}
|
||||
export type $defs = Record<string, never>;
|
||||
export type operations = Record<string, never>;
|
||||
57
src/lib/server/compute-worker/protocol.ts
Normal file
57
src/lib/server/compute-worker/protocol.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import type { components, paths } from './generated';
|
||||
|
||||
export type ParsedPdfDocument = components['schemas']['ParsedPdfDocument'];
|
||||
export type ParsedPdfPage = ParsedPdfDocument['pages'][number];
|
||||
export type ParsedPdfBlock = ParsedPdfPage['blocks'][number];
|
||||
export type ParsedPdfBlockKind = ParsedPdfBlock['kind'];
|
||||
export type ParsedPdfBlockFragment = ParsedPdfBlock['fragments'][number];
|
||||
|
||||
export type TTSSentenceAlignment = components['schemas']['TTSSentenceAlignment'];
|
||||
export type TTSSentenceWord = TTSSentenceAlignment['words'][number];
|
||||
|
||||
export type PdfLayoutProgress = NonNullable<components['schemas']['PublicOperation']['progress']>;
|
||||
export type WorkerOperationStatus = components['schemas']['PublicOperation']['status'];
|
||||
export type WorkerOperationSubject = components['schemas']['PublicOperation']['subject'];
|
||||
|
||||
export type WorkerOperation<Result = unknown> =
|
||||
Omit<components['schemas']['PublicOperation'], 'result'>
|
||||
& { result?: Result };
|
||||
|
||||
export type WorkerOperationEvent<Result = unknown> = {
|
||||
eventId: number;
|
||||
snapshot: WorkerOperation<Result>;
|
||||
};
|
||||
|
||||
export type WhisperAlignRequest =
|
||||
paths['/v1/whisper-align/operations']['post']['requestBody']['content']['application/json'];
|
||||
export type PdfLayoutRequest =
|
||||
paths['/v1/pdf-layout/operations']['post']['requestBody']['content']['application/json'];
|
||||
export type PdfLayoutResolveRequest =
|
||||
paths['/v1/pdf-layout/resolve']['post']['requestBody']['content']['application/json'];
|
||||
|
||||
export type WhisperAlignResult = {
|
||||
alignments: TTSSentenceAlignment[];
|
||||
timing?: components['schemas']['PublicOperation']['timing'];
|
||||
};
|
||||
|
||||
export type PdfLayoutResult = {
|
||||
parsedObjectKey: string;
|
||||
timing?: components['schemas']['PublicOperation']['timing'];
|
||||
};
|
||||
|
||||
export type PdfLayoutResolution = {
|
||||
artifact: { objectKey: string } | null;
|
||||
operation: WorkerOperation<PdfLayoutResult> | null;
|
||||
};
|
||||
|
||||
export function isWorkerOperation(value: unknown): value is WorkerOperation {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const record = value as Record<string, unknown>;
|
||||
return typeof record.opId === 'string'
|
||||
&& !!record.subject
|
||||
&& typeof record.subject === 'object'
|
||||
&& typeof (record.subject as Record<string, unknown>).kind === 'string'
|
||||
&& typeof record.status === 'string'
|
||||
&& typeof record.queuedAt === 'number'
|
||||
&& typeof record.updatedAt === 'number';
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
export function isAbortLikeError(error: unknown): boolean {
|
||||
if (!error) return false;
|
||||
if (error instanceof DOMException) return error.name === 'AbortError';
|
||||
if (error instanceof Error) return error.name === 'AbortError' || error.message === 'This operation was aborted';
|
||||
if (typeof error === 'object' && error !== null && 'name' in error) {
|
||||
return (error as { name?: unknown }).name === 'AbortError';
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
import type { ComputeBackend } from '@/lib/server/compute/types';
|
||||
import { isWorkerClientConfigAvailable, WorkerComputeBackend } from '@/lib/server/compute/worker';
|
||||
|
||||
let backendPromise: Promise<ComputeBackend> | null = null;
|
||||
|
||||
export async function getCompute(): Promise<ComputeBackend> {
|
||||
if (!backendPromise) {
|
||||
backendPromise = Promise.resolve(new WorkerComputeBackend()).catch((error) => {
|
||||
backendPromise = null;
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
return backendPromise;
|
||||
}
|
||||
|
||||
export function isComputeAvailable(): boolean {
|
||||
return isWorkerClientConfigAvailable();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue