Refactor PDF parsing and rendering logic; migrate to compute-worker protocol
- Removed deprecated functions related to document parsing and blob storage in blobstore.ts. - Introduced new PDF rendering logic in pdf-preview-renderer.ts and pdf-preview-pdfjs-runtime.ts. - Updated previews-render.ts to utilize the new PDF rendering functions. - Refactored user-whisper-align-job.ts to use the compute-worker client for alignment requests. - Enhanced artifact.ts and operation.ts to validate parsed PDF artifacts and resolve current PDF parses. - Updated snapshot.ts to align with new worker operation types. - Adjusted runtime-config.ts to check for compute-worker availability. - Modified types in parsed-pdf.ts and tts.ts to reflect changes in the compute-worker protocol. - Added unit tests for PDF artifact validation and compute-worker client contract. - Removed obsolete pdf-op-key.vitest.spec.ts test file.
This commit is contained in:
parent
4bfd020125
commit
a56aaa2be9
122 changed files with 4285 additions and 1528 deletions
|
|
@ -1,5 +1,6 @@
|
||||||
.env
|
.env
|
||||||
.env.*
|
.env.*
|
||||||
|
**/*.creds
|
||||||
README.md
|
README.md
|
||||||
.next
|
.next
|
||||||
node_modules
|
node_modules
|
||||||
|
|
|
||||||
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -62,13 +62,13 @@ jobs:
|
||||||
platform: linux/amd64
|
platform: linux/amd64
|
||||||
runner: ubuntu-24.04
|
runner: ubuntu-24.04
|
||||||
context: .
|
context: .
|
||||||
dockerfile: ./compute/worker/Dockerfile
|
dockerfile: ./compute-worker/Dockerfile
|
||||||
- image_target: compute-worker
|
- image_target: compute-worker
|
||||||
arch: arm64
|
arch: arm64
|
||||||
platform: linux/arm64
|
platform: linux/arm64
|
||||||
runner: ubuntu-24.04-arm
|
runner: ubuntu-24.04-arm
|
||||||
context: .
|
context: .
|
||||||
dockerfile: ./compute/worker/Dockerfile
|
dockerfile: ./compute-worker/Dockerfile
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
|
|
|
||||||
2
.github/workflows/vitest.yml
vendored
2
.github/workflows/vitest.yml
vendored
|
|
@ -25,3 +25,5 @@ jobs:
|
||||||
run: pnpm migrate
|
run: pnpm migrate
|
||||||
- name: Run Vitest suites
|
- name: Run Vitest suites
|
||||||
run: pnpm test:unit
|
run: pnpm test:unit
|
||||||
|
- name: Verify compute worker contract and boundary
|
||||||
|
run: pnpm compute:openapi:check && pnpm check:compute-boundary
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,7 @@ WORKDIR /app
|
||||||
|
|
||||||
# Copy workspace manifests needed for dependency installation
|
# Copy workspace manifests needed for dependency installation
|
||||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||||
COPY compute/core/package.json ./compute/core/package.json
|
COPY compute-worker/package.json ./compute-worker/package.json
|
||||||
COPY compute/worker/package.json ./compute/worker/package.json
|
|
||||||
COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json
|
COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json
|
||||||
|
|
||||||
# Install dependencies
|
# Install dependencies
|
||||||
|
|
@ -88,7 +87,7 @@ COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses
|
||||||
# Include SeaweedFS license text for the copied weed binary.
|
# Include SeaweedFS license text for the copied weed binary.
|
||||||
COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt
|
COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt
|
||||||
# Include static model notices for runtime-downloaded assets.
|
# Include static model notices for runtime-downloaded assets.
|
||||||
COPY --from=app-builder /app/compute/core/src/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
|
COPY --from=app-builder /app/compute-worker/src/compute/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
|
||||||
|
|
||||||
# Copy seaweedfs weed binary for optional embedded local S3.
|
# Copy seaweedfs weed binary for optional embedded local S3.
|
||||||
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
|
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
|
||||||
|
|
@ -98,7 +97,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server
|
||||||
RUN chmod +x /usr/local/bin/nats-server
|
RUN chmod +x /usr/local/bin/nats-server
|
||||||
|
|
||||||
# Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts.
|
# Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts.
|
||||||
COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
COPY --from=app-builder /app/compute-worker/src/compute/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
||||||
|
|
||||||
# Match the app's historical container port now that standalone server.js
|
# Match the app's historical container port now that standalone server.js
|
||||||
# is started directly instead of `next start -p 3003`.
|
# is started directly instead of `next start -p 3003`.
|
||||||
|
|
|
||||||
|
|
@ -5,11 +5,8 @@ RUN npm install -g pnpm@11.1.2
|
||||||
WORKDIR /workspace
|
WORKDIR /workspace
|
||||||
|
|
||||||
COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||||
COPY compute/worker/package.json compute/worker/package.json
|
COPY compute-worker/package.json compute-worker/package.json
|
||||||
COPY compute/core/package.json compute/core/package.json
|
COPY compute-worker compute-worker
|
||||||
|
|
||||||
COPY compute/worker compute/worker
|
|
||||||
COPY compute/core compute/core
|
|
||||||
|
|
||||||
RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker
|
RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker
|
||||||
|
|
||||||
|
|
@ -11,13 +11,14 @@ services:
|
||||||
|
|
||||||
compute-worker:
|
compute-worker:
|
||||||
build:
|
build:
|
||||||
context: ../..
|
context: ..
|
||||||
dockerfile: compute/worker/Dockerfile
|
dockerfile: compute-worker/Dockerfile
|
||||||
container_name: openreader-compute-worker
|
container_name: openreader-compute-worker
|
||||||
depends_on:
|
depends_on:
|
||||||
- nats
|
- nats
|
||||||
env_file:
|
env_file:
|
||||||
- ./.env
|
- path: ./.env
|
||||||
|
required: false
|
||||||
environment:
|
environment:
|
||||||
NATS_URL: ${NATS_URL:-nats://nats:4222}
|
NATS_URL: ${NATS_URL:-nats://nats:4222}
|
||||||
COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0}
|
COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0}
|
||||||
|
|
@ -32,16 +33,12 @@ services:
|
||||||
- action: sync+restart
|
- action: sync+restart
|
||||||
path: .
|
path: .
|
||||||
target: /workspace
|
target: /workspace
|
||||||
- action: rebuild
|
|
||||||
path: ../../compute/core
|
|
||||||
- action: rebuild
|
- action: rebuild
|
||||||
path: ./package.json
|
path: ./package.json
|
||||||
- action: rebuild
|
- action: rebuild
|
||||||
path: ../../compute/core/package.json
|
path: ../pnpm-lock.yaml
|
||||||
- action: rebuild
|
- action: rebuild
|
||||||
path: ../../pnpm-lock.yaml
|
path: ../pnpm-workspace.yaml
|
||||||
- action: rebuild
|
|
||||||
path: ../../pnpm-workspace.yaml
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
nats-data:
|
nats-data:
|
||||||
1766
compute-worker/openapi.json
Normal file
1766
compute-worker/openapi.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -5,15 +5,22 @@
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "tsx watch src/server.ts",
|
"dev": "tsx watch src/server.ts",
|
||||||
"start": "tsx src/server.ts"
|
"start": "tsx src/server.ts",
|
||||||
|
"openapi:generate": "tsx scripts/generate-openapi.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.1061.0",
|
"@aws-sdk/client-s3": "^3.1061.0",
|
||||||
|
"@fastify/swagger": "^9.0.0",
|
||||||
|
"@huggingface/tokenizers": "^0.1.3",
|
||||||
|
"@napi-rs/canvas": "^0.1.100",
|
||||||
"@nats-io/jetstream": "^3.4.0",
|
"@nats-io/jetstream": "^3.4.0",
|
||||||
"@nats-io/kv": "^3.4.0",
|
"@nats-io/kv": "^3.4.0",
|
||||||
"@nats-io/transport-node": "^3.4.0",
|
"@nats-io/transport-node": "^3.4.0",
|
||||||
"@openreader/compute-core": "workspace:*",
|
"ffmpeg-static": "^5.3.0",
|
||||||
"fastify": "^5.8.5",
|
"fastify": "^5.8.5",
|
||||||
|
"jszip": "^3.10.1",
|
||||||
|
"onnxruntime-node": "^1.26.0",
|
||||||
|
"pdfjs-dist": "4.8.69",
|
||||||
"pino": "^10.3.1",
|
"pino": "^10.3.1",
|
||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
"tsx": "^4.22.4",
|
"tsx": "^4.22.4",
|
||||||
20
compute-worker/scripts/generate-openapi.ts
Normal file
20
compute-worker/scripts/generate-openapi.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
import { mkdir, writeFile } from 'node:fs/promises';
|
||||||
|
import { dirname, resolve } from 'node:path';
|
||||||
|
import { createComputeWorkerApp } from '../src/runtime';
|
||||||
|
|
||||||
|
process.env.COMPUTE_WORKER_TOKEN ||= 'openapi-generation-token';
|
||||||
|
process.env.NATS_URL ||= 'nats://127.0.0.1:4222';
|
||||||
|
|
||||||
|
const outputPath = resolve(process.cwd(), 'openapi.json');
|
||||||
|
const runtime = await createComputeWorkerApp({
|
||||||
|
workerToken: process.env.COMPUTE_WORKER_TOKEN,
|
||||||
|
disableWorkers: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runtime.app.ready();
|
||||||
|
await mkdir(dirname(outputPath), { recursive: true });
|
||||||
|
await writeFile(outputPath, `${JSON.stringify(runtime.app.swagger(), null, 2)}\n`, 'utf8');
|
||||||
|
} finally {
|
||||||
|
await runtime.close();
|
||||||
|
}
|
||||||
56
compute-worker/src/api/operation-keys.ts
Normal file
56
compute-worker/src/api/operation-keys.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version';
|
||||||
|
|
||||||
|
function sha256Hex(input: string): string {
|
||||||
|
return createHash('sha256').update(input).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildWhisperOperationKey(input: {
|
||||||
|
audioObjectKey: string;
|
||||||
|
text: string;
|
||||||
|
lang?: string;
|
||||||
|
cacheKey?: string;
|
||||||
|
}): string {
|
||||||
|
const cacheKey = input.cacheKey?.trim();
|
||||||
|
if (cacheKey) {
|
||||||
|
return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`;
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
'whisper_align',
|
||||||
|
'v1',
|
||||||
|
input.audioObjectKey,
|
||||||
|
input.lang ?? '',
|
||||||
|
sha256Hex(input.text),
|
||||||
|
].join('|');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildPdfOperationKey(input: {
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
documentObjectKey: string;
|
||||||
|
replaceToken?: string;
|
||||||
|
}, parserVersion = PDF_PARSER_VERSION): string {
|
||||||
|
return [
|
||||||
|
'pdf_layout',
|
||||||
|
'v1',
|
||||||
|
parserVersion,
|
||||||
|
input.documentId,
|
||||||
|
input.namespace ?? '',
|
||||||
|
input.documentObjectKey,
|
||||||
|
input.replaceToken?.trim() || '',
|
||||||
|
].join('|');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pdfSubjectFromOperationKey(opKey: string): {
|
||||||
|
kind: 'pdf_layout';
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
} | null {
|
||||||
|
const [kind, version, , documentId, namespace] = opKey.split('|');
|
||||||
|
if (kind !== 'pdf_layout' || version !== 'v1' || !documentId) return null;
|
||||||
|
return {
|
||||||
|
kind: 'pdf_layout',
|
||||||
|
documentId,
|
||||||
|
namespace: namespace || null,
|
||||||
|
};
|
||||||
|
}
|
||||||
44
compute-worker/src/api/public-operation.ts
Normal file
44
compute-worker/src/api/public-operation.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
import type { WorkerOperationState } from '../compute/api-contracts';
|
||||||
|
import { pdfSubjectFromOperationKey } from './operation-keys';
|
||||||
|
|
||||||
|
export type PublicOperationSubject =
|
||||||
|
| { kind: 'whisper_align' }
|
||||||
|
| { kind: 'pdf_layout'; documentId: string; namespace: string | null };
|
||||||
|
|
||||||
|
export interface PublicOperation<Result = unknown> {
|
||||||
|
opId: string;
|
||||||
|
subject: PublicOperationSubject;
|
||||||
|
status: WorkerOperationState['status'];
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: Result;
|
||||||
|
error?: WorkerOperationState['error'];
|
||||||
|
timing?: WorkerOperationState['timing'];
|
||||||
|
progress?: WorkerOperationState['progress'];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PublicOperationEvent<Result = unknown> {
|
||||||
|
eventId: number;
|
||||||
|
snapshot: PublicOperation<Result>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function toPublicOperation<Result>(
|
||||||
|
state: WorkerOperationState<Result>,
|
||||||
|
): PublicOperation<Result> {
|
||||||
|
const subject = state.kind === 'pdf_layout'
|
||||||
|
? (pdfSubjectFromOperationKey(state.opKey) ?? { kind: 'pdf_layout', documentId: '', namespace: null })
|
||||||
|
: { kind: 'whisper_align' as const };
|
||||||
|
return {
|
||||||
|
opId: state.opId,
|
||||||
|
subject,
|
||||||
|
status: state.status,
|
||||||
|
queuedAt: state.queuedAt,
|
||||||
|
updatedAt: state.updatedAt,
|
||||||
|
...(state.startedAt === undefined ? {} : { startedAt: state.startedAt }),
|
||||||
|
...(state.result === undefined ? {} : { result: state.result }),
|
||||||
|
...(state.error === undefined ? {} : { error: state.error }),
|
||||||
|
...(state.timing === undefined ? {} : { timing: state.timing }),
|
||||||
|
...(state.progress === undefined ? {} : { progress: state.progress }),
|
||||||
|
};
|
||||||
|
}
|
||||||
134
compute-worker/src/api/schemas.ts
Normal file
134
compute-worker/src/api/schemas.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
const bboxSchema = z.tuple([z.number(), z.number(), z.number(), z.number()]);
|
||||||
|
const parsedPdfBlockKindSchema = z.enum([
|
||||||
|
'abstract', 'algorithm', 'aside_text', 'chart', 'content', 'formula',
|
||||||
|
'doc_title', 'figure_title', 'footer', 'footnote', 'formula_number',
|
||||||
|
'header', 'image', 'number', 'paragraph_title', 'reference',
|
||||||
|
'reference_content', 'seal', 'table', 'text', 'vision_footnote',
|
||||||
|
]);
|
||||||
|
const documentIdSchema = z.string().trim().regex(/^[a-f0-9]{64}$/i);
|
||||||
|
const namespaceSchema = z.string().trim().regex(/^[a-zA-Z0-9._-]{1,128}$/).nullable();
|
||||||
|
|
||||||
|
export const parsedPdfDocumentSchema = z.object({
|
||||||
|
schemaVersion: z.literal(1),
|
||||||
|
documentId: z.string(),
|
||||||
|
parserVersion: z.string(),
|
||||||
|
parsedAt: z.number(),
|
||||||
|
pages: z.array(z.object({
|
||||||
|
pageNumber: z.number(),
|
||||||
|
width: z.number(),
|
||||||
|
height: z.number(),
|
||||||
|
blocks: z.array(z.object({
|
||||||
|
id: z.string(),
|
||||||
|
kind: parsedPdfBlockKindSchema,
|
||||||
|
fragments: z.array(z.object({
|
||||||
|
page: z.number(),
|
||||||
|
bbox: bboxSchema,
|
||||||
|
text: z.string(),
|
||||||
|
readingOrder: z.number(),
|
||||||
|
modelConfidence: z.number().optional(),
|
||||||
|
})),
|
||||||
|
text: z.string(),
|
||||||
|
parentSectionId: z.string().optional(),
|
||||||
|
})),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const ttsSentenceAlignmentSchema = z.object({
|
||||||
|
sentence: z.string(),
|
||||||
|
sentenceIndex: z.number(),
|
||||||
|
words: z.array(z.object({
|
||||||
|
text: z.string(),
|
||||||
|
startSec: z.number(),
|
||||||
|
endSec: z.number(),
|
||||||
|
charStart: z.number(),
|
||||||
|
charEnd: z.number(),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const whisperOperationCreateSchema = z.object({
|
||||||
|
text: z.string().trim().min(1),
|
||||||
|
lang: z.string().trim().min(1).max(16).optional(),
|
||||||
|
cacheKey: z.string().trim().min(1).max(256).optional(),
|
||||||
|
audioObjectKey: z.string().trim().min(1).max(2048),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const pdfOperationCreateSchema = z.object({
|
||||||
|
documentId: documentIdSchema,
|
||||||
|
namespace: namespaceSchema,
|
||||||
|
documentObjectKey: z.string().trim().min(1).max(2048),
|
||||||
|
replaceToken: z.string().trim().min(1).max(256).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const pdfResolveSchema = z.object({
|
||||||
|
documentId: documentIdSchema,
|
||||||
|
namespace: namespaceSchema,
|
||||||
|
documentObjectKey: z.string().trim().min(1).max(2048),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const operationParamsSchema = z.object({
|
||||||
|
opId: z.string().trim().min(1),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const operationEventsQuerySchema = z.object({
|
||||||
|
sinceEventId: z.union([z.string(), z.number()]).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const apiErrorResponseSchema = z.object({
|
||||||
|
error: z.string(),
|
||||||
|
}).passthrough();
|
||||||
|
|
||||||
|
export const operationErrorSchema = z.object({
|
||||||
|
message: z.string(),
|
||||||
|
code: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const artifactReferenceSchema = z.object({
|
||||||
|
objectKey: z.string(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const pdfLayoutProgressSchema = z.object({
|
||||||
|
totalPages: z.number(),
|
||||||
|
pagesParsed: z.number(),
|
||||||
|
currentPage: z.number().optional(),
|
||||||
|
phase: z.enum(['infer', 'merge']),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const publicOperationSchema = z.object({
|
||||||
|
opId: z.string(),
|
||||||
|
subject: z.discriminatedUnion('kind', [
|
||||||
|
z.object({ kind: z.literal('whisper_align') }),
|
||||||
|
z.object({
|
||||||
|
kind: z.literal('pdf_layout'),
|
||||||
|
documentId: z.string(),
|
||||||
|
namespace: z.string().nullable(),
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
status: z.enum(['queued', 'running', 'succeeded', 'failed']),
|
||||||
|
queuedAt: z.number(),
|
||||||
|
updatedAt: z.number(),
|
||||||
|
startedAt: z.number().optional(),
|
||||||
|
result: z.unknown().optional(),
|
||||||
|
error: operationErrorSchema.optional(),
|
||||||
|
timing: z.object({
|
||||||
|
queueWaitMs: z.number().optional(),
|
||||||
|
s3FetchMs: z.number().optional(),
|
||||||
|
computeMs: z.number().optional(),
|
||||||
|
}).optional(),
|
||||||
|
progress: pdfLayoutProgressSchema.optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const publicOperationEventSchema = z.object({
|
||||||
|
eventId: z.number(),
|
||||||
|
snapshot: publicOperationSchema,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const pdfLayoutResolutionSchema = z.object({
|
||||||
|
artifact: artifactReferenceSchema.nullable(),
|
||||||
|
operation: publicOperationSchema.nullable(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export function jsonSchema(schema: z.ZodType): Record<string, unknown> {
|
||||||
|
return z.toJSONSchema(schema, { target: 'draft-7' }) as Record<string, unknown>;
|
||||||
|
}
|
||||||
44
compute-worker/src/compute/whisper/alignment-map.ts
Normal file
44
compute-worker/src/compute/whisper/alignment-map.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
||||||
|
|
||||||
|
export interface WhisperWord {
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
word: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||||
|
const lowerSentence = sentence.toLowerCase();
|
||||||
|
let cursor = 0;
|
||||||
|
|
||||||
|
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
||||||
|
const token = w.word.trim();
|
||||||
|
if (!token) {
|
||||||
|
return {
|
||||||
|
text: '',
|
||||||
|
startSec: w.start,
|
||||||
|
endSec: w.end,
|
||||||
|
charStart: cursor,
|
||||||
|
charEnd: cursor,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||||
|
const start = idx >= 0 ? idx : cursor;
|
||||||
|
const end = Math.min(sentence.length, start + token.length);
|
||||||
|
cursor = Math.max(cursor, end);
|
||||||
|
|
||||||
|
return {
|
||||||
|
text: token,
|
||||||
|
startSec: w.start,
|
||||||
|
endSec: w.end,
|
||||||
|
charStart: start,
|
||||||
|
charEnd: end,
|
||||||
|
};
|
||||||
|
}).filter((word) => word.text.length > 0);
|
||||||
|
|
||||||
|
return {
|
||||||
|
sentence,
|
||||||
|
sentenceIndex: 0,
|
||||||
|
words: alignedWords,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -8,12 +8,12 @@ import type {
|
||||||
OperationState,
|
OperationState,
|
||||||
OperationStateStore,
|
OperationStateStore,
|
||||||
QueuedOperation,
|
QueuedOperation,
|
||||||
} from '@openreader/compute-core/control-plane';
|
} from '../compute/control-plane';
|
||||||
import type {
|
import type {
|
||||||
PdfLayoutJobRequest,
|
PdfLayoutJobRequest,
|
||||||
WhisperAlignJobRequest,
|
WhisperAlignJobRequest,
|
||||||
WorkerOperationKind,
|
WorkerOperationKind,
|
||||||
} from '@openreader/compute-core/api-contracts';
|
} from '../compute/api-contracts';
|
||||||
import { createJsonCodec } from './json-codec';
|
import { createJsonCodec } from './json-codec';
|
||||||
|
|
||||||
export interface KvEntryLike {
|
export interface KvEntryLike {
|
||||||
|
|
@ -4,7 +4,7 @@ import type {
|
||||||
WorkerJobTiming,
|
WorkerJobTiming,
|
||||||
WorkerJobState,
|
WorkerJobState,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
} from '@openreader/compute-core/api-contracts';
|
} from './compute/api-contracts';
|
||||||
|
|
||||||
export type StreamedOperationState = WorkerOperationState<WhisperAlignJobResult | PdfLayoutJobResult>;
|
export type StreamedOperationState = WorkerOperationState<WhisperAlignJobResult | PdfLayoutJobResult>;
|
||||||
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import type { PdfLayoutProgress } from '@openreader/compute-core/api-contracts';
|
import type { PdfLayoutProgress } from './compute/api-contracts';
|
||||||
|
|
||||||
export function buildInferProgressForPageStart(input: {
|
export function buildInferProgressForPageStart(input: {
|
||||||
pageNumber: number;
|
pageNumber: number;
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify';
|
||||||
|
import swagger from '@fastify/swagger';
|
||||||
import { z } from 'zod';
|
import { z } from 'zod';
|
||||||
import {
|
import {
|
||||||
connect,
|
connect,
|
||||||
|
|
@ -24,18 +25,16 @@ import {
|
||||||
ensureComputeModels,
|
ensureComputeModels,
|
||||||
runPdfLayoutFromPdfBuffer,
|
runPdfLayoutFromPdfBuffer,
|
||||||
runWhisperAlignmentFromAudioBuffer,
|
runWhisperAlignmentFromAudioBuffer,
|
||||||
} from '@openreader/compute-core/local-runtime';
|
} from './compute/local-runtime';
|
||||||
import {
|
import {
|
||||||
getComputeTimeoutConfig,
|
getComputeTimeoutConfig,
|
||||||
getComputeOpStaleMs,
|
getComputeOpStaleMs,
|
||||||
getAvailableCpuCores,
|
getAvailableCpuCores,
|
||||||
getOnnxThreadsPerJob,
|
getOnnxThreadsPerJob,
|
||||||
PDF_PARSER_VERSION,
|
|
||||||
encodeParserVersion,
|
|
||||||
withIdleTimeoutAndHardCap,
|
withIdleTimeoutAndHardCap,
|
||||||
withTimeout,
|
withTimeout,
|
||||||
} from '@openreader/compute-core';
|
} from './compute';
|
||||||
import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane';
|
import { encodeSseFrame, OperationOrchestrator } from './compute/control-plane';
|
||||||
import type {
|
import type {
|
||||||
PdfLayoutJobRequest,
|
PdfLayoutJobRequest,
|
||||||
PdfLayoutJobResult,
|
PdfLayoutJobResult,
|
||||||
|
|
@ -48,7 +47,7 @@ import type {
|
||||||
WorkerOperationRequest,
|
WorkerOperationRequest,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
PdfLayoutProgress,
|
PdfLayoutProgress,
|
||||||
} from '@openreader/compute-core/api-contracts';
|
} from './compute/api-contracts';
|
||||||
import {
|
import {
|
||||||
DeleteObjectCommand,
|
DeleteObjectCommand,
|
||||||
GetObjectCommand,
|
GetObjectCommand,
|
||||||
|
|
@ -71,6 +70,26 @@ import {
|
||||||
import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress';
|
import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress';
|
||||||
import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy';
|
import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy';
|
||||||
import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence';
|
import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence';
|
||||||
|
import { parsedPdfArtifactKey } from './storage/artifact-addressing';
|
||||||
|
import { buildPdfOperationKey, buildWhisperOperationKey } from './api/operation-keys';
|
||||||
|
import { toPublicOperation, type PublicOperationEvent } from './api/public-operation';
|
||||||
|
import {
|
||||||
|
apiErrorResponseSchema,
|
||||||
|
artifactReferenceSchema,
|
||||||
|
jsonSchema,
|
||||||
|
operationEventsQuerySchema,
|
||||||
|
operationParamsSchema,
|
||||||
|
operationErrorSchema,
|
||||||
|
parsedPdfDocumentSchema,
|
||||||
|
pdfLayoutProgressSchema,
|
||||||
|
pdfLayoutResolutionSchema,
|
||||||
|
pdfOperationCreateSchema,
|
||||||
|
pdfResolveSchema,
|
||||||
|
publicOperationEventSchema,
|
||||||
|
publicOperationSchema,
|
||||||
|
ttsSentenceAlignmentSchema,
|
||||||
|
whisperOperationCreateSchema,
|
||||||
|
} from './api/schemas';
|
||||||
|
|
||||||
const JOBS_STREAM_NAME = 'compute_jobs';
|
const JOBS_STREAM_NAME = 'compute_jobs';
|
||||||
const WHISPER_JOBS_SUBJECT = 'jobs.whisper';
|
const WHISPER_JOBS_SUBJECT = 'jobs.whisper';
|
||||||
|
|
@ -88,8 +107,6 @@ const OP_EVENTS_KEEPALIVE_MS = 15_000;
|
||||||
// client from immediately reconnecting and re-waking the worker; instead it
|
// client from immediately reconnecting and re-waking the worker; instead it
|
||||||
// reconnects on a slow cadence so the container stays asleep most of the time.
|
// reconnects on a slow cadence so the container stays asleep most of the time.
|
||||||
const OP_EVENTS_RECONNECT_HINT_MS = 120_000;
|
const OP_EVENTS_RECONNECT_HINT_MS = 120_000;
|
||||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
|
||||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
|
||||||
const WHISPER_MAX_DELIVER = 1;
|
const WHISPER_MAX_DELIVER = 1;
|
||||||
const NATS_API_TIMEOUT_MS = 60_000;
|
const NATS_API_TIMEOUT_MS = 60_000;
|
||||||
// Disconnect from NATS after this much continuous idle so the worker stops
|
// Disconnect from NATS after this much continuous idle so the worker stops
|
||||||
|
|
@ -182,6 +199,7 @@ export interface ComputeWorkerRouteDeps {
|
||||||
orchestrator: OrchestratorLike;
|
orchestrator: OrchestratorLike;
|
||||||
operationStateStore: OperationStateStoreLike;
|
operationStateStore: OperationStateStoreLike;
|
||||||
operationEventStream: OperationEventStreamLike;
|
operationEventStream: OperationEventStreamLike;
|
||||||
|
artifactExists?: (key: string) => Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CreateComputeWorkerAppOptions {
|
export interface CreateComputeWorkerAppOptions {
|
||||||
|
|
@ -254,20 +272,6 @@ function normalizeS3Prefix(prefix: string | undefined): string {
|
||||||
return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader';
|
return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader';
|
||||||
}
|
}
|
||||||
|
|
||||||
function sanitizeNamespace(namespace: string | null): string | null {
|
|
||||||
if (!namespace) return null;
|
|
||||||
return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function documentParsedKey(id: string, namespace: string | null, prefix: string): string {
|
|
||||||
if (!DOCUMENT_ID_REGEX.test(id)) {
|
|
||||||
throw new Error(`Invalid document id: ${id}`);
|
|
||||||
}
|
|
||||||
const ns = sanitizeNamespace(namespace);
|
|
||||||
const nsSegment = ns ? `ns/${ns}/` : '';
|
|
||||||
return `${prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(PDF_PARSER_VERSION)}.json`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildS3Client(): S3Client {
|
function buildS3Client(): S3Client {
|
||||||
const bucket = requireEnv('S3_BUCKET');
|
const bucket = requireEnv('S3_BUCKET');
|
||||||
const region = requireEnv('S3_REGION');
|
const region = requireEnv('S3_REGION');
|
||||||
|
|
@ -357,7 +361,7 @@ function extractOpId(request: FastifyRequest, path: string): string | null {
|
||||||
if (params && typeof params.opId === 'string' && params.opId.trim()) {
|
if (params && typeof params.opId === 'string' && params.opId.trim()) {
|
||||||
return params.opId.trim();
|
return params.opId.trim();
|
||||||
}
|
}
|
||||||
const match = path.match(/^\/ops\/([^/]+)/);
|
const match = path.match(/^\/v1\/operations\/([^/]+)/);
|
||||||
if (!match?.[1]) return null;
|
if (!match?.[1]) return null;
|
||||||
try {
|
try {
|
||||||
return decodeURIComponent(match[1]);
|
return decodeURIComponent(match[1]);
|
||||||
|
|
@ -428,22 +432,7 @@ const layoutSchema = z.object({
|
||||||
documentObjectKey: z.string().trim().min(1).max(2048),
|
documentObjectKey: z.string().trim().min(1).max(2048),
|
||||||
});
|
});
|
||||||
|
|
||||||
const operationCreateSchema = z.discriminatedUnion('kind', [
|
const errorResponseSchema = jsonSchema(apiErrorResponseSchema);
|
||||||
z.object({
|
|
||||||
kind: z.literal('whisper_align'),
|
|
||||||
opKey: z.string().trim().min(1).max(1024),
|
|
||||||
payload: alignSchema,
|
|
||||||
}),
|
|
||||||
z.object({
|
|
||||||
kind: z.literal('pdf_layout'),
|
|
||||||
opKey: z.string().trim().min(1).max(1024),
|
|
||||||
payload: layoutSchema,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const operationLookupSchema = z.object({
|
|
||||||
opKey: z.string().trim().min(1).max(1024),
|
|
||||||
});
|
|
||||||
|
|
||||||
async function ensureJetStreamResources(
|
async function ensureJetStreamResources(
|
||||||
jsm: JetStreamManager,
|
jsm: JetStreamManager,
|
||||||
|
|
@ -567,7 +556,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
// Lazy NATS connection lifecycle. The worker connects on demand (first request
|
// Lazy NATS connection lifecycle. The worker connects on demand (first request
|
||||||
// that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle
|
// that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle
|
||||||
// so it stops emitting outbound traffic and Railway can sleep it. Reconnect is
|
// so it stops emitting outbound traffic and Railway can sleep it. Reconnect is
|
||||||
// transparent: any inbound /ops request both wakes the container and re-establishes
|
// transparent: any inbound operation request both wakes the container and re-establishes
|
||||||
// the session via ensureConnected().
|
// the session via ensureConnected().
|
||||||
let session: NatsSession | null = null;
|
let session: NatsSession | null = null;
|
||||||
let connecting: Promise<NatsSession> | null = null;
|
let connecting: Promise<NatsSession> | null = null;
|
||||||
|
|
@ -785,7 +774,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
if (!s3) {
|
if (!s3) {
|
||||||
throw new Error('S3 access is disabled for this worker app instance');
|
throw new Error('S3 access is disabled for this worker app instance');
|
||||||
}
|
}
|
||||||
const key = documentParsedKey(documentId, namespace, s3Prefix);
|
const key = parsedPdfArtifactKey({ documentId, namespace, prefix: s3Prefix });
|
||||||
const body = Buffer.from(JSON.stringify(parsed));
|
const body = Buffer.from(JSON.stringify(parsed));
|
||||||
await s3.send(new PutObjectCommand({
|
await s3.send(new PutObjectCommand({
|
||||||
Bucket: s3Bucket,
|
Bucket: s3Bucket,
|
||||||
|
|
@ -805,6 +794,35 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
logger: buildLoggerConfig(),
|
logger: buildLoggerConfig(),
|
||||||
disableRequestLogging: true,
|
disableRequestLogging: true,
|
||||||
});
|
});
|
||||||
|
await app.register(swagger, {
|
||||||
|
openapi: {
|
||||||
|
openapi: '3.0.3',
|
||||||
|
info: {
|
||||||
|
title: 'OpenReader Compute Worker API',
|
||||||
|
version: '1.0.0',
|
||||||
|
},
|
||||||
|
components: {
|
||||||
|
securitySchemes: {
|
||||||
|
bearerAuth: {
|
||||||
|
type: 'http',
|
||||||
|
scheme: 'bearer',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
schemas: {
|
||||||
|
ParsedPdfDocument: jsonSchema(parsedPdfDocumentSchema),
|
||||||
|
TTSSentenceAlignment: jsonSchema(ttsSentenceAlignmentSchema),
|
||||||
|
ArtifactReference: jsonSchema(artifactReferenceSchema),
|
||||||
|
ErrorResponse: jsonSchema(apiErrorResponseSchema),
|
||||||
|
OperationError: jsonSchema(operationErrorSchema),
|
||||||
|
PdfLayoutProgress: jsonSchema(pdfLayoutProgressSchema),
|
||||||
|
PublicOperation: jsonSchema(publicOperationSchema),
|
||||||
|
PublicOperationEvent: jsonSchema(publicOperationEventSchema),
|
||||||
|
PdfLayoutResolution: jsonSchema(pdfLayoutResolutionSchema),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
security: [{ bearerAuth: [] }],
|
||||||
|
},
|
||||||
|
});
|
||||||
app.log.info({
|
app.log.info({
|
||||||
jobConcurrency,
|
jobConcurrency,
|
||||||
whisperTimeoutMs,
|
whisperTimeoutMs,
|
||||||
|
|
@ -956,16 +974,37 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/health/live', async () => ({ ok: true }));
|
app.get('/health/live', {
|
||||||
|
schema: {
|
||||||
|
security: [],
|
||||||
|
response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } },
|
||||||
|
},
|
||||||
|
}, async () => ({ ok: true }));
|
||||||
|
|
||||||
// Reports readiness without forcing a NATS round-trip. Probing NATS here would
|
// Reports readiness without forcing a NATS round-trip. Probing NATS here would
|
||||||
// reconnect (and keep) the connection open, defeating idle sleep, so we only
|
// reconnect (and keep) the connection open, defeating idle sleep, so we only
|
||||||
// report the current connection state. The worker reconnects lazily on the next
|
// report the current connection state. The worker reconnects lazily on the next
|
||||||
// /ops request regardless of what this returns.
|
// operation request regardless of what this returns.
|
||||||
app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null }));
|
app.get('/health/ready', {
|
||||||
|
schema: {
|
||||||
|
security: [],
|
||||||
|
response: {
|
||||||
|
200: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } },
|
||||||
|
required: ['ok', 'natsConnected'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, async () => ({ ok: true, natsConnected: session !== null }));
|
||||||
|
|
||||||
app.post('/ops', async (request, reply) => {
|
app.post('/v1/whisper-align/operations', {
|
||||||
const parsed = operationCreateSchema.safeParse(request.body);
|
schema: {
|
||||||
|
body: jsonSchema(whisperOperationCreateSchema),
|
||||||
|
response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema },
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
const parsed = whisperOperationCreateSchema.safeParse(request.body);
|
||||||
if (!parsed.success) {
|
if (!parsed.success) {
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return {
|
return {
|
||||||
|
|
@ -974,7 +1013,11 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const requestOp = parsed.data as WorkerOperationRequest;
|
const requestOp: WorkerOperationRequest = {
|
||||||
|
kind: 'whisper_align',
|
||||||
|
opKey: buildWhisperOperationKey(parsed.data),
|
||||||
|
payload: parsed.data,
|
||||||
|
};
|
||||||
await ensureOrphanedOpRecovery();
|
await ensureOrphanedOpRecovery();
|
||||||
const op = await orchestrator.enqueueOrReuse(requestOp);
|
const op = await orchestrator.enqueueOrReuse(requestOp);
|
||||||
app.log.info({
|
app.log.info({
|
||||||
|
|
@ -985,11 +1028,16 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16),
|
opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16),
|
||||||
}, 'op.accepted');
|
}, 'op.accepted');
|
||||||
reply.code(202);
|
reply.code(202);
|
||||||
return op;
|
return toPublicOperation(op);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.post('/ops/lookup', async (request, reply) => {
|
app.post('/v1/pdf-layout/operations', {
|
||||||
const parsed = operationLookupSchema.safeParse(request.body);
|
schema: {
|
||||||
|
body: jsonSchema(pdfOperationCreateSchema),
|
||||||
|
response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema },
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
const parsed = pdfOperationCreateSchema.safeParse(request.body);
|
||||||
if (!parsed.success) {
|
if (!parsed.success) {
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return {
|
return {
|
||||||
|
|
@ -998,28 +1046,58 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const requestOp: WorkerOperationRequest = {
|
||||||
|
kind: 'pdf_layout',
|
||||||
|
opKey: buildPdfOperationKey(parsed.data),
|
||||||
|
payload: {
|
||||||
|
documentId: parsed.data.documentId,
|
||||||
|
namespace: parsed.data.namespace,
|
||||||
|
documentObjectKey: parsed.data.documentObjectKey,
|
||||||
|
},
|
||||||
|
};
|
||||||
await ensureOrphanedOpRecovery();
|
await ensureOrphanedOpRecovery();
|
||||||
if (typeof operationStateStore.getOpIndex !== 'function') {
|
const op = await orchestrator.enqueueOrReuse(requestOp);
|
||||||
reply.code(501);
|
reply.code(202);
|
||||||
return { error: 'Operation lookup by opKey is not supported by this state store' };
|
return toPublicOperation(op);
|
||||||
}
|
|
||||||
const indexEntry = await operationStateStore.getOpIndex(parsed.data.opKey);
|
|
||||||
if (!indexEntry?.opId) {
|
|
||||||
reply.code(404);
|
|
||||||
return { error: 'Operation not found' };
|
|
||||||
}
|
|
||||||
|
|
||||||
const state = await operationStateStore.getOpState(indexEntry.opId);
|
|
||||||
if (!state) {
|
|
||||||
reply.code(404);
|
|
||||||
return { error: 'Operation not found' };
|
|
||||||
}
|
|
||||||
|
|
||||||
return state;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/ops/:opId', async (request, reply) => {
|
app.post('/v1/pdf-layout/resolve', {
|
||||||
const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params);
|
schema: {
|
||||||
|
body: jsonSchema(pdfResolveSchema),
|
||||||
|
response: {
|
||||||
|
200: jsonSchema(pdfLayoutResolutionSchema),
|
||||||
|
400: errorResponseSchema,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
const parsed = pdfResolveSchema.safeParse(request.body);
|
||||||
|
if (!parsed.success) {
|
||||||
|
reply.code(400);
|
||||||
|
return { error: 'Invalid request body', issues: parsed.error.issues };
|
||||||
|
}
|
||||||
|
await ensureOrphanedOpRecovery();
|
||||||
|
const artifactKey = parsedPdfArtifactKey({
|
||||||
|
documentId: parsed.data.documentId,
|
||||||
|
namespace: parsed.data.namespace,
|
||||||
|
prefix: s3Prefix,
|
||||||
|
});
|
||||||
|
const hasArtifact = await (options.routeDeps?.artifactExists ?? objectExists)(artifactKey);
|
||||||
|
const opKey = buildPdfOperationKey(parsed.data);
|
||||||
|
const index = await operationStateStore.getOpIndex?.(opKey);
|
||||||
|
const operation = index?.opId ? await operationStateStore.getOpState(index.opId) : null;
|
||||||
|
return {
|
||||||
|
artifact: hasArtifact ? { objectKey: artifactKey } : null,
|
||||||
|
operation: operation ? toPublicOperation(operation) : null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get('/v1/operations/:opId', {
|
||||||
|
schema: {
|
||||||
|
params: jsonSchema(operationParamsSchema),
|
||||||
|
response: { 200: jsonSchema(publicOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema },
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
const params = operationParamsSchema.safeParse(request.params);
|
||||||
if (!params.success) {
|
if (!params.success) {
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return { error: 'Invalid op id' };
|
return { error: 'Invalid op id' };
|
||||||
|
|
@ -1031,11 +1109,21 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
return { error: 'Operation not found' };
|
return { error: 'Operation not found' };
|
||||||
}
|
}
|
||||||
|
|
||||||
return state;
|
return toPublicOperation(state);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/ops/:opId/events', async (request, reply) => {
|
app.get('/v1/operations/:opId/events', {
|
||||||
const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params);
|
schema: {
|
||||||
|
params: jsonSchema(operationParamsSchema),
|
||||||
|
querystring: jsonSchema(operationEventsQuerySchema),
|
||||||
|
response: {
|
||||||
|
200: { type: 'string', description: 'Server-sent PublicOperationEvent stream' },
|
||||||
|
400: errorResponseSchema,
|
||||||
|
404: errorResponseSchema,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
const params = operationParamsSchema.safeParse(request.params);
|
||||||
if (!params.success) {
|
if (!params.success) {
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return { error: 'Invalid op id' };
|
return { error: 'Invalid op id' };
|
||||||
|
|
@ -1080,9 +1168,9 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti
|
||||||
|
|
||||||
const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => {
|
const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => {
|
||||||
if (closed || reply.raw.writableEnded) return;
|
if (closed || reply.raw.writableEnded) return;
|
||||||
const frameEvent: WorkerOperationEvent<WhisperAlignJobResult | PdfLayoutJobResult> = {
|
const frameEvent: PublicOperationEvent<WhisperAlignJobResult | PdfLayoutJobResult> = {
|
||||||
eventId,
|
eventId,
|
||||||
snapshot,
|
snapshot: toPublicOperation(snapshot),
|
||||||
};
|
};
|
||||||
reply.raw.write(encodeSseFrame({
|
reply.raw.write(encodeSseFrame({
|
||||||
id: eventId,
|
id: eventId,
|
||||||
21
compute-worker/src/storage/artifact-addressing.ts
Normal file
21
compute-worker/src/storage/artifact-addressing.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version';
|
||||||
|
import { encodeParserVersion } from '../compute/pdf/parser-version-key';
|
||||||
|
|
||||||
|
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||||
|
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||||
|
|
||||||
|
export function parsedPdfArtifactKey(input: {
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
prefix: string;
|
||||||
|
parserVersion?: string;
|
||||||
|
}): string {
|
||||||
|
if (!DOCUMENT_ID_REGEX.test(input.documentId)) {
|
||||||
|
throw new Error(`Invalid document id: ${input.documentId}`);
|
||||||
|
}
|
||||||
|
const namespace = input.namespace && SAFE_NAMESPACE_REGEX.test(input.namespace)
|
||||||
|
? input.namespace
|
||||||
|
: null;
|
||||||
|
const namespaceSegment = namespace ? `ns/${namespace}/` : '';
|
||||||
|
return `${input.prefix}/documents_v1/parsed_v2/${namespaceSegment}${input.documentId}/${encodeParserVersion(input.parserVersion ?? PDF_PARSER_VERSION)}.json`;
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts';
|
import type { WorkerOperationKind } from './compute/api-contracts';
|
||||||
|
|
||||||
export type RetryAction = 'nak_retry' | 'term_fail';
|
export type RetryAction = 'nak_retry' | 'term_fail';
|
||||||
|
|
||||||
|
|
@ -25,53 +25,51 @@ describe('compute worker API routes', () => {
|
||||||
const live = await runtime.app.inject({ method: 'GET', url: '/health/live' });
|
const live = await runtime.app.inject({ method: 'GET', url: '/health/live' });
|
||||||
expect(live.statusCode).toBe(200);
|
expect(live.statusCode).toBe(200);
|
||||||
|
|
||||||
const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' });
|
const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/v1/operations/op-1' });
|
||||||
expect(protectedRoute.statusCode).toBe(401);
|
expect(protectedRoute.statusCode).toBe(401);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('validates operation creation body and returns 400 for invalid payload', async () => {
|
test('validates operation creation body and returns 400 for invalid payload', async () => {
|
||||||
const response = await runtime.app.inject({
|
const response = await runtime.app.inject({
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
url: '/ops',
|
url: '/v1/pdf-layout/operations',
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
payload: {
|
payload: {
|
||||||
kind: 'pdf_layout',
|
documentId: '',
|
||||||
opKey: '',
|
namespace: null,
|
||||||
payload: {
|
documentObjectKey: 'openreader/doc.pdf',
|
||||||
documentId: 'd1',
|
|
||||||
namespace: null,
|
|
||||||
documentObjectKey: 's3://bucket/doc.pdf',
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(response.statusCode).toBe(400);
|
expect(response.statusCode).toBe(400);
|
||||||
expect(response.json()).toMatchObject({ error: 'Invalid request body' });
|
expect(response.json()).toMatchObject({ error: 'Bad Request' });
|
||||||
});
|
});
|
||||||
|
|
||||||
test('creates operation and fetches state by op id', async () => {
|
test('creates operation and fetches state by op id', async () => {
|
||||||
|
const documentId = 'c'.repeat(64);
|
||||||
const create = await runtime.app.inject({
|
const create = await runtime.app.inject({
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
url: '/ops',
|
url: '/v1/pdf-layout/operations',
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
payload: {
|
payload: {
|
||||||
kind: 'pdf_layout',
|
documentId,
|
||||||
opKey: 'doc-1:layout',
|
namespace: null,
|
||||||
payload: {
|
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||||
documentId: 'doc-1',
|
|
||||||
namespace: null,
|
|
||||||
documentObjectKey: 'openreader/doc-1.pdf',
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(create.statusCode).toBe(202);
|
expect(create.statusCode).toBe(202);
|
||||||
const created = create.json();
|
const created = create.json();
|
||||||
expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' });
|
expect(created).toMatchObject({
|
||||||
|
subject: { kind: 'pdf_layout', documentId, namespace: null },
|
||||||
|
status: 'queued',
|
||||||
|
});
|
||||||
|
expect(created).not.toHaveProperty('opKey');
|
||||||
|
expect(created).not.toHaveProperty('jobId');
|
||||||
|
|
||||||
const fetch = await runtime.app.inject({
|
const fetch = await runtime.app.inject({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
url: `/ops/${created.opId}`,
|
url: `/v1/operations/${created.opId}`,
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -79,17 +77,120 @@ describe('compute worker API routes', () => {
|
||||||
expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' });
|
expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('reuses idempotent PDF requests and replaces them only with an explicit token', async () => {
|
||||||
|
const documentId = 'e'.repeat(64);
|
||||||
|
const payload = {
|
||||||
|
documentId,
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||||
|
};
|
||||||
|
const create = (body: typeof payload & { replaceToken?: string }) => runtime.app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/v1/pdf-layout/operations',
|
||||||
|
headers: AUTH,
|
||||||
|
payload: body,
|
||||||
|
});
|
||||||
|
|
||||||
|
const initial = (await create(payload)).json();
|
||||||
|
const reused = (await create(payload)).json();
|
||||||
|
const replacement = (await create({ ...payload, replaceToken: 'replace-1' })).json();
|
||||||
|
|
||||||
|
expect(reused.opId).toBe(initial.opId);
|
||||||
|
expect(replacement.opId).not.toBe(initial.opId);
|
||||||
|
expect(replacement.subject).toEqual(initial.subject);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('creates whisper alignment operations without exposing internal keys', async () => {
|
||||||
|
const response = await runtime.app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/v1/whisper-align/operations',
|
||||||
|
headers: AUTH,
|
||||||
|
payload: {
|
||||||
|
text: 'Canonical worker text',
|
||||||
|
audioObjectKey: 'openreader/audio.mp3',
|
||||||
|
lang: 'en',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.statusCode).toBe(202);
|
||||||
|
expect(response.json()).toMatchObject({
|
||||||
|
subject: { kind: 'whisper_align' },
|
||||||
|
status: 'queued',
|
||||||
|
});
|
||||||
|
expect(response.json()).not.toHaveProperty('opKey');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('resolves the current PDF artifact and operation without exposing parser identity', async () => {
|
||||||
|
const documentId = 'a'.repeat(64);
|
||||||
|
const create = await runtime.app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/v1/pdf-layout/operations',
|
||||||
|
headers: AUTH,
|
||||||
|
payload: {
|
||||||
|
documentId,
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(create.statusCode).toBe(202);
|
||||||
|
|
||||||
|
const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`;
|
||||||
|
fake.seedArtifact(artifactKey);
|
||||||
|
const resolve = await runtime.app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/v1/pdf-layout/resolve',
|
||||||
|
headers: AUTH,
|
||||||
|
payload: {
|
||||||
|
documentId,
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resolve.statusCode).toBe(200);
|
||||||
|
expect(resolve.json()).toMatchObject({
|
||||||
|
artifact: { objectKey: artifactKey },
|
||||||
|
operation: {
|
||||||
|
subject: { kind: 'pdf_layout', documentId, namespace: null },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(resolve.json().operation).not.toHaveProperty('opKey');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('resolves a current PDF artifact after operation state has expired', async () => {
|
||||||
|
const documentId = 'd'.repeat(64);
|
||||||
|
const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`;
|
||||||
|
fake.seedArtifact(artifactKey);
|
||||||
|
|
||||||
|
const resolve = await runtime.app.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/v1/pdf-layout/resolve',
|
||||||
|
headers: AUTH,
|
||||||
|
payload: {
|
||||||
|
documentId,
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: `openreader/${documentId}.pdf`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resolve.statusCode).toBe(200);
|
||||||
|
expect(resolve.json()).toEqual({
|
||||||
|
artifact: { objectKey: artifactKey },
|
||||||
|
operation: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test('returns not found for unknown operation and event stream lookups', async () => {
|
test('returns not found for unknown operation and event stream lookups', async () => {
|
||||||
const opResponse = await runtime.app.inject({
|
const opResponse = await runtime.app.inject({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
url: '/ops/missing',
|
url: '/v1/operations/missing',
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
});
|
});
|
||||||
expect(opResponse.statusCode).toBe(404);
|
expect(opResponse.statusCode).toBe(404);
|
||||||
|
|
||||||
const eventsResponse = await runtime.app.inject({
|
const eventsResponse = await runtime.app.inject({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
url: '/ops/missing/events',
|
url: '/v1/operations/missing/events',
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
});
|
});
|
||||||
expect(eventsResponse.statusCode).toBe(404);
|
expect(eventsResponse.statusCode).toBe(404);
|
||||||
|
|
@ -109,7 +210,7 @@ describe('compute worker API routes', () => {
|
||||||
|
|
||||||
const stream = await runtime.app.inject({
|
const stream = await runtime.app.inject({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
url: '/ops/op-terminal/events?sinceEventId=7',
|
url: '/v1/operations/op-terminal/events?sinceEventId=7',
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -159,11 +260,11 @@ describe('compute worker API routes', () => {
|
||||||
updatedAt: now - 310_000,
|
updatedAt: now - 310_000,
|
||||||
});
|
});
|
||||||
|
|
||||||
// GET /ops/:opId resolves via getOpState(), which first awaits the shared
|
// GET /v1/operations/:opId resolves via getOpState(), which first awaits the shared
|
||||||
// orphanRecoveryPromise path through ensureOrphanedOpRecovery().
|
// orphanRecoveryPromise path through ensureOrphanedOpRecovery().
|
||||||
const fetch = await runtime.app.inject({
|
const fetch = await runtime.app.inject({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
url: '/ops/op-stale-whisper-running',
|
url: '/v1/operations/op-stale-whisper-running',
|
||||||
headers: AUTH,
|
headers: AUTH,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { mergeTextWithRegions } from '@openreader/compute-core';
|
import { mergeTextWithRegions } from '../../../src/compute';
|
||||||
|
|
||||||
describe('mergeTextWithRegions', () => {
|
describe('mergeTextWithRegions', () => {
|
||||||
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
|
|
||||||
import { normalizeTextItemsForLayout } from '@openreader/compute-core';
|
import { normalizeTextItemsForLayout } from '../../../src/compute';
|
||||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||||
|
|
||||||
function makeTextItem(
|
function makeTextItem(
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { stitchCrossPageBlocks } from '@openreader/compute-core';
|
import { stitchCrossPageBlocks } from '../../../src/compute';
|
||||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf';
|
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/compute/types';
|
||||||
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
||||||
|
|
||||||
function makeBlock(
|
function makeBlock(
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
import type { BaseDocument } from '../../../src/types/documents';
|
||||||
|
import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/compute/types';
|
||||||
|
|
||||||
|
export function makeBaseDocument(overrides: Partial<BaseDocument> = {}): BaseDocument {
|
||||||
|
return {
|
||||||
|
id: 'doc-1',
|
||||||
|
name: 'document.pdf',
|
||||||
|
size: 1_024,
|
||||||
|
lastModified: 1_700_000_000_000,
|
||||||
|
type: 'pdf',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeParsedPdfBlock(input: {
|
||||||
|
id: string;
|
||||||
|
kind: ParsedPdfBlockKind;
|
||||||
|
text: string;
|
||||||
|
page: number;
|
||||||
|
readingOrder: number;
|
||||||
|
}): ParsedPdfBlock {
|
||||||
|
return {
|
||||||
|
id: input.id,
|
||||||
|
kind: input.kind,
|
||||||
|
text: input.text,
|
||||||
|
fragments: [
|
||||||
|
{
|
||||||
|
page: input.page,
|
||||||
|
bbox: [0, 0, 100, 10],
|
||||||
|
text: input.text,
|
||||||
|
readingOrder: input.readingOrder,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage {
|
||||||
|
return {
|
||||||
|
pageNumber,
|
||||||
|
width: 800,
|
||||||
|
height: 1_200,
|
||||||
|
blocks,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument {
|
||||||
|
return {
|
||||||
|
schemaVersion: 1,
|
||||||
|
documentId: 'doc-fixture',
|
||||||
|
parserVersion: 'test',
|
||||||
|
parsedAt: 1_700_000_000_000,
|
||||||
|
pages,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import {
|
import {
|
||||||
mapWordsToSentenceOffsets,
|
mapWordsToSentenceOffsets,
|
||||||
} from '@openreader/compute-core';
|
} from '../../../src/compute';
|
||||||
|
|
||||||
describe('whisper alignment mapping', () => {
|
describe('whisper alignment mapping', () => {
|
||||||
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core';
|
import { buildGoertzelCoefficients, goertzelPower } from '../../../src/compute';
|
||||||
|
|
||||||
function dftPower(samples: Float32Array, k: number): number {
|
function dftPower(samples: Float32Array, k: number): number {
|
||||||
const n = samples.length;
|
const n = samples.length;
|
||||||
|
|
@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node';
|
||||||
import {
|
import {
|
||||||
buildWordsFromTimestampedTokens,
|
buildWordsFromTimestampedTokens,
|
||||||
extractTokenStartTimestamps,
|
extractTokenStartTimestamps,
|
||||||
} from '@openreader/compute-core';
|
} from '../../../src/compute';
|
||||||
|
|
||||||
describe('whisper token timestamp alignment', () => {
|
describe('whisper token timestamp alignment', () => {
|
||||||
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import type { WorkerOperationRequest } from '../../src/api-contracts';
|
import type { WorkerOperationRequest } from '../../../src/compute/api-contracts';
|
||||||
import { OperationOrchestrator } from '../../src/control-plane';
|
import { OperationOrchestrator } from '../../../src/compute/control-plane';
|
||||||
import {
|
import {
|
||||||
InMemoryOperationEventStream,
|
InMemoryOperationEventStream,
|
||||||
InMemoryOperationQueue,
|
InMemoryOperationQueue,
|
||||||
|
|
@ -70,13 +70,13 @@ vi.mock('@napi-rs/canvas', () => {
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
vi.mock('../../src/pdf/model', () => ({
|
vi.mock('../../../src/compute/pdf/model', () => ({
|
||||||
ensureModel: vi.fn(async () => '/tmp/model.onnx'),
|
ensureModel: vi.fn(async () => '/tmp/model.onnx'),
|
||||||
MODEL_CONFIG_PATH: '/tmp/model-config.json',
|
MODEL_CONFIG_PATH: '/tmp/model-config.json',
|
||||||
MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json',
|
MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('../../src/config/cpu-budget', () => ({
|
vi.mock('../../../src/compute/config/cpu-budget', () => ({
|
||||||
getOnnxThreadsPerJob: vi.fn(() => 1),
|
getOnnxThreadsPerJob: vi.fn(() => 1),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
|
@ -107,7 +107,7 @@ describe('runLayoutModel', () => {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const { runLayoutModel } = await import('../../src/pdf/runLayoutModel');
|
const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel');
|
||||||
const regions = await runLayoutModel({
|
const regions = await runLayoutModel({
|
||||||
pageWidth: 100,
|
pageWidth: 100,
|
||||||
pageHeight: 100,
|
pageHeight: 100,
|
||||||
|
|
@ -154,7 +154,7 @@ describe('runLayoutModel', () => {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const { runLayoutModel } = await import('../../src/pdf/runLayoutModel');
|
const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel');
|
||||||
const regions = await runLayoutModel({
|
const regions = await runLayoutModel({
|
||||||
pageWidth: 100,
|
pageWidth: 100,
|
||||||
pageHeight: 100,
|
pageHeight: 100,
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse';
|
import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/compute/control-plane/sse';
|
||||||
|
|
||||||
describe('sse codec', () => {
|
describe('sse codec', () => {
|
||||||
test('encodes event id and payload and decodes both reliably', () => {
|
test('encodes event id and payload and decodes both reliably', () => {
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import type { WorkerOperationState } from '../../src/api-contracts';
|
import type { WorkerOperationState } from '../../../src/compute/api-contracts';
|
||||||
import {
|
import {
|
||||||
explainReplacementReason,
|
explainReplacementReason,
|
||||||
isInflightStatus,
|
isInflightStatus,
|
||||||
isTerminalStatus,
|
isTerminalStatus,
|
||||||
shouldReuseExistingOperation,
|
shouldReuseExistingOperation,
|
||||||
} from '../../src/control-plane/state-machine';
|
} from '../../../src/compute/control-plane/state-machine';
|
||||||
|
|
||||||
function runningState(overrides: Partial<WorkerOperationState> = {}): WorkerOperationState {
|
function runningState(overrides: Partial<WorkerOperationState> = {}): WorkerOperationState {
|
||||||
return {
|
return {
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { EventEmitter } from 'node:events';
|
import { EventEmitter } from 'node:events';
|
||||||
import type { WorkerOperationKind } from '../../src/api-contracts';
|
import type { WorkerOperationKind } from '../../../src/compute/api-contracts';
|
||||||
import type {
|
import type {
|
||||||
OperationEvent,
|
OperationEvent,
|
||||||
OperationEventStream,
|
OperationEventStream,
|
||||||
|
|
@ -8,7 +8,7 @@ import type {
|
||||||
OperationState,
|
OperationState,
|
||||||
OperationStateStore,
|
OperationStateStore,
|
||||||
QueuedOperation,
|
QueuedOperation,
|
||||||
} from '../../src/control-plane/types';
|
} from '../../../src/compute/control-plane/types';
|
||||||
|
|
||||||
function topicFor(opId: string): string {
|
function topicFor(opId: string): string {
|
||||||
return `op.${opId}`;
|
return `op.${opId}`;
|
||||||
|
|
@ -4,7 +4,7 @@ import type {
|
||||||
WorkerOperationEvent,
|
WorkerOperationEvent,
|
||||||
WorkerOperationRequest,
|
WorkerOperationRequest,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
} from '@openreader/compute-core/api-contracts';
|
} from '../../src/compute/api-contracts';
|
||||||
import type { ComputeWorkerRouteDeps } from '../../src/runtime';
|
import type { ComputeWorkerRouteDeps } from '../../src/runtime';
|
||||||
|
|
||||||
type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult;
|
type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult;
|
||||||
|
|
@ -16,6 +16,7 @@ export class FakeControlPlane {
|
||||||
private readonly revisionByOpId = new Map<string, number>();
|
private readonly revisionByOpId = new Map<string, number>();
|
||||||
private readonly opIdByOpKey = new Map<string, string>();
|
private readonly opIdByOpKey = new Map<string, string>();
|
||||||
private readonly eventsByOpId = new Map<string, ComputeEvent[]>();
|
private readonly eventsByOpId = new Map<string, ComputeEvent[]>();
|
||||||
|
private readonly artifactKeys = new Set<string>();
|
||||||
private nextOpId = 1;
|
private nextOpId = 1;
|
||||||
|
|
||||||
readonly deps: ComputeWorkerRouteDeps = {
|
readonly deps: ComputeWorkerRouteDeps = {
|
||||||
|
|
@ -58,6 +59,10 @@ export class FakeControlPlane {
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
listOpStates: async () => Array.from(this.stateByOpId.values()),
|
listOpStates: async () => Array.from(this.stateByOpId.values()),
|
||||||
|
getOpIndex: async (opKey) => {
|
||||||
|
const opId = this.opIdByOpKey.get(opKey);
|
||||||
|
return opId ? { opId } : null;
|
||||||
|
},
|
||||||
},
|
},
|
||||||
operationEventStream: {
|
operationEventStream: {
|
||||||
subscribe: async ({ opId, sinceEventId, onEvent }) => {
|
subscribe: async ({ opId, sinceEventId, onEvent }) => {
|
||||||
|
|
@ -69,6 +74,7 @@ export class FakeControlPlane {
|
||||||
return () => undefined;
|
return () => undefined;
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
artifactExists: async (key) => this.artifactKeys.has(key),
|
||||||
};
|
};
|
||||||
|
|
||||||
seedState(state: ComputeState): void {
|
seedState(state: ComputeState): void {
|
||||||
|
|
@ -83,6 +89,10 @@ export class FakeControlPlane {
|
||||||
this.eventsByOpId.set(opId, list);
|
this.eventsByOpId.set(opId, list);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
seedArtifact(key: string): void {
|
||||||
|
this.artifactKeys.add(key);
|
||||||
|
}
|
||||||
|
|
||||||
getState(opId: string): ComputeState | null {
|
getState(opId: string): ComputeState | null {
|
||||||
return this.stateByOpId.get(opId) ?? null;
|
return this.stateByOpId.get(opId) ?? null;
|
||||||
}
|
}
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { OperationOrchestrator } from '@openreader/compute-core/control-plane';
|
import { OperationOrchestrator } from '../../src/compute/control-plane';
|
||||||
import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts';
|
import type { WorkerOperationRequest } from '../../src/compute/api-contracts';
|
||||||
import {
|
import {
|
||||||
JetStreamOperationEventStream,
|
JetStreamOperationEventStream,
|
||||||
JetStreamOperationQueue,
|
JetStreamOperationQueue,
|
||||||
|
|
@ -4,6 +4,8 @@ import {
|
||||||
buildInferProgressForPageParsed,
|
buildInferProgressForPageParsed,
|
||||||
buildInferProgressForPageStart,
|
buildInferProgressForPageStart,
|
||||||
} from '../../src/pdf-progress';
|
} from '../../src/pdf-progress';
|
||||||
|
import { buildPdfOperationKey } from '../../src/api/operation-keys';
|
||||||
|
import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing';
|
||||||
|
|
||||||
describe('compute worker helpers', () => {
|
describe('compute worker helpers', () => {
|
||||||
test('hash and kv key helpers are stable and deterministic', () => {
|
test('hash and kv key helpers are stable and deterministic', () => {
|
||||||
|
|
@ -34,4 +36,17 @@ describe('compute worker helpers', () => {
|
||||||
phase: 'infer',
|
phase: 'infer',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('parser-version changes rotate worker-owned operation and artifact identities', () => {
|
||||||
|
const request = {
|
||||||
|
documentId: 'a'.repeat(64),
|
||||||
|
namespace: null,
|
||||||
|
documentObjectKey: `openreader/${'a'.repeat(64)}.pdf`,
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(buildPdfOperationKey(request, 'parser-v1'))
|
||||||
|
.not.toBe(buildPdfOperationKey(request, 'parser-v2'));
|
||||||
|
expect(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v1' }))
|
||||||
|
.not.toBe(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v2' }));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"extends": "../../tsconfig.json",
|
"extends": "../tsconfig.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"noEmit": true,
|
"noEmit": true,
|
||||||
},
|
},
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
{
|
|
||||||
"name": "@openreader/compute-core",
|
|
||||||
"version": "0.0.0",
|
|
||||||
"private": true,
|
|
||||||
"type": "module",
|
|
||||||
"dependencies": {
|
|
||||||
"@huggingface/tokenizers": "^0.1.3",
|
|
||||||
"@napi-rs/canvas": "^0.1.100",
|
|
||||||
"ffmpeg-static": "^5.3.0",
|
|
||||||
"jszip": "^3.10.1",
|
|
||||||
"onnxruntime-node": "^1.26.0",
|
|
||||||
"pdfjs-dist": "4.8.69"
|
|
||||||
},
|
|
||||||
"exports": {
|
|
||||||
".": "./src/index.ts",
|
|
||||||
"./local-runtime": "./src/local-runtime.ts",
|
|
||||||
"./api-contracts": "./src/api-contracts/index.ts",
|
|
||||||
"./control-plane": "./src/control-plane/index.ts",
|
|
||||||
"./types": "./src/types/index.ts"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,62 +0,0 @@
|
||||||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
|
||||||
|
|
||||||
// Worker-side mirror of the app's canonical audio-text cleaning rules in
|
|
||||||
// `src/lib/shared/audio-text.ts`. This is a separate build target and cannot
|
|
||||||
// import from `@/lib`, so the rules are duplicated here on purpose. The word
|
|
||||||
// `charStart`/`charEnd` offsets this module emits are consumed against text
|
|
||||||
// normalized by that shared module, so any divergence shifts viewer highlights
|
|
||||||
// off-word — keep this byte-for-byte in sync with `audio-text.ts`.
|
|
||||||
const STRIPPED_GLYPHS = /[*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖]/g;
|
|
||||||
|
|
||||||
function preprocessSentenceForAudio(text: string): string {
|
|
||||||
return text
|
|
||||||
.replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -')
|
|
||||||
.replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2')
|
|
||||||
.replace(STRIPPED_GLYPHS, '')
|
|
||||||
.replace(/\s+/g, ' ')
|
|
||||||
.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface WhisperWord {
|
|
||||||
start: number;
|
|
||||||
end: number;
|
|
||||||
word: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
|
||||||
const normalizedSentence = preprocessSentenceForAudio(sentence);
|
|
||||||
const lowerSentence = normalizedSentence.toLowerCase();
|
|
||||||
let cursor = 0;
|
|
||||||
|
|
||||||
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
|
||||||
const token = w.word.trim();
|
|
||||||
if (!token) {
|
|
||||||
return {
|
|
||||||
text: '',
|
|
||||||
startSec: w.start,
|
|
||||||
endSec: w.end,
|
|
||||||
charStart: cursor,
|
|
||||||
charEnd: cursor,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
|
||||||
const start = idx >= 0 ? idx : cursor;
|
|
||||||
const end = Math.min(normalizedSentence.length, start + token.length);
|
|
||||||
cursor = Math.max(cursor, end);
|
|
||||||
|
|
||||||
return {
|
|
||||||
text: token,
|
|
||||||
startSec: w.start,
|
|
||||||
endSec: w.end,
|
|
||||||
charStart: start,
|
|
||||||
charEnd: end,
|
|
||||||
};
|
|
||||||
}).filter((word) => word.text.length > 0);
|
|
||||||
|
|
||||||
return {
|
|
||||||
sentence,
|
|
||||||
sentenceIndex: 0,
|
|
||||||
words: alignedWords,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
{
|
|
||||||
"extends": "../../tsconfig.json",
|
|
||||||
"compilerOptions": {
|
|
||||||
"noEmit": true
|
|
||||||
},
|
|
||||||
"include": ["src/**/*.ts"]
|
|
||||||
}
|
|
||||||
|
|
@ -11,7 +11,8 @@ Use this guide when OpenReader runs compute as a separate service. For the defau
|
||||||
- Runs PDF layout parsing jobs
|
- Runs PDF layout parsing jobs
|
||||||
- Stores durable job state in NATS JetStream and NATS KV
|
- Stores durable job state in NATS JetStream and NATS KV
|
||||||
|
|
||||||
The app server submits work to `POST /ops` and listens for updates on `GET /ops/:opId/events`.
|
The app server submits resource-specific operations under `/v1` and listens for updates on
|
||||||
|
`GET /v1/operations/:opId/events`.
|
||||||
|
|
||||||
## When to use it
|
## When to use it
|
||||||
|
|
||||||
|
|
@ -37,7 +38,7 @@ S3_SECRET_ACCESS_KEY=...
|
||||||
```
|
```
|
||||||
|
|
||||||
:::important
|
:::important
|
||||||
`compute/worker/.env*` is only for standalone worker deployments.
|
`compute-worker/.env*` is only for standalone worker deployments.
|
||||||
|
|
||||||
- Embedded/local mode: configure the root `.env` only.
|
- Embedded/local mode: configure the root `.env` only.
|
||||||
- External worker mode: set `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` on the app, and worker runtime values on the worker service.
|
- External worker mode: set `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` on the app, and worker runtime values on the worker service.
|
||||||
|
|
|
||||||
|
|
@ -163,17 +163,17 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i
|
||||||
<summary><strong>External compute worker dev stack (optional)</strong></summary>
|
<summary><strong>External compute worker dev stack (optional)</strong></summary>
|
||||||
|
|
||||||
Use this only when you intentionally run compute-worker as a separate service.
|
Use this only when you intentionally run compute-worker as a separate service.
|
||||||
Default local flow does not need `compute/worker/.env`; embedded worker startup reads root `.env`.
|
Default local flow does not need `compute-worker/.env`; embedded worker startup reads root `.env`.
|
||||||
Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker).
|
Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker).
|
||||||
|
|
||||||
Start only NATS + compute-worker via compose watch:
|
Start only NATS + compute-worker via compose watch:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch
|
docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --watch
|
||||||
# or: pnpm compute:dev:watch
|
# or: pnpm compute:dev:compose
|
||||||
```
|
```
|
||||||
|
|
||||||
`compute/worker/.env.example` contains a starter config for standalone worker service deployments.
|
`compute-worker/.env.example` contains a starter config for standalone worker service deployments.
|
||||||
|
|
||||||
Run the main app separately on the host:
|
Run the main app separately on the host:
|
||||||
|
|
||||||
|
|
@ -190,9 +190,9 @@ COMPUTE_WORKER_TOKEN=<same-token-used-by-worker>
|
||||||
|
|
||||||
Ownership in external worker mode:
|
Ownership in external worker mode:
|
||||||
- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS`
|
- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS`
|
||||||
- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
||||||
|
|
||||||
For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute/worker/.env*` is ignored in that mode.
|
For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute-worker/.env*` is ignored in that mode.
|
||||||
|
|
||||||
Worker mode requires worker-reachable shared object storage (S3-compatible endpoint).
|
Worker mode requires worker-reachable shared object storage (S3-compatible endpoint).
|
||||||
For external worker mode, object storage must be shared/reachable by both app and worker services.
|
For external worker mode, object storage must be shared/reachable by both app and worker services.
|
||||||
|
|
@ -240,7 +240,7 @@ COMPUTE_WORKER_TOKEN=<same-token-used-by-worker>
|
||||||
|
|
||||||
Use the same ownership split:
|
Use the same ownership split:
|
||||||
- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale overrides
|
- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale overrides
|
||||||
- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning)
|
||||||
|
|
||||||
Use one of these `.env` mode templates:
|
Use one of these `.env` mode templates:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -45,21 +45,20 @@ title: Stack
|
||||||
|
|
||||||
## External compute worker (optional)
|
## External compute worker (optional)
|
||||||
|
|
||||||
Monorepo packages under `compute/`:
|
Standalone worker package:
|
||||||
|
|
||||||
- **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts
|
- **`@openreader/compute-worker`** — standalone Node.js compute service containing its private inference and queue runtime
|
||||||
- ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers`
|
- ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers`
|
||||||
- Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps
|
- Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps
|
||||||
- PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing
|
- PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing
|
||||||
- PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization
|
- PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization
|
||||||
- Utilities: `jszip`, `ffmpeg-static`
|
- Utilities: `jszip`, `ffmpeg-static`
|
||||||
- **`@openreader/compute-worker`** — standalone Node.js worker service
|
- HTTP server: [Fastify](https://fastify.dev/) v5 with a versioned OpenAPI contract
|
||||||
- HTTP server: [Fastify](https://fastify.dev/) v5
|
|
||||||
- Job queue + state: [NATS](https://nats.io/) JetStream WorkQueue pull consumers + NATS KV (`jobs.whisper`, `jobs.layout`)
|
- Job queue + state: [NATS](https://nats.io/) JetStream WorkQueue pull consumers + NATS KV (`jobs.whisper`, `jobs.layout`)
|
||||||
- Storage: AWS SDK v3 S3 client for reading/writing blobs
|
- Storage: AWS SDK v3 S3 client for reading/writing blobs
|
||||||
- Logging: [Pino](https://getpino.io/)
|
- Logging: [Pino](https://getpino.io/)
|
||||||
- Validation: [Zod](https://zod.dev/)
|
- Validation: [Zod](https://zod.dev/)
|
||||||
- Heavy compute is worker-backed via `COMPUTE_WORKER_URL` + `COMPUTE_WORKER_TOKEN` (remote queue via HTTP + NATS)
|
- The Next.js app communicates with the worker only through the versioned HTTP API generated from OpenAPI.
|
||||||
|
|
||||||
## Tooling and testing
|
## Tooling and testing
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -34,16 +34,14 @@ const UI_ARCHITECTURE_FILES = [
|
||||||
"src/components/player/**/*.{ts,tsx}",
|
"src/components/player/**/*.{ts,tsx}",
|
||||||
"src/components/reader/**/*.{ts,tsx}",
|
"src/components/reader/**/*.{ts,tsx}",
|
||||||
];
|
];
|
||||||
const COMPUTE_CORE_IMPORT_PATTERNS = [
|
const COMPUTE_WORKER_IMPORT_PATTERNS = [
|
||||||
{
|
{
|
||||||
group: [
|
group: [
|
||||||
"@openreader/compute-core/*",
|
"@openreader/compute-worker",
|
||||||
"!@openreader/compute-core/local-runtime",
|
"@openreader/compute-worker/*",
|
||||||
"!@openreader/compute-core/types",
|
|
||||||
"!@openreader/compute-core/api-contracts",
|
|
||||||
],
|
],
|
||||||
message:
|
message:
|
||||||
"Use '@openreader/compute-core' root imports for light APIs. Allowed subpaths are '@openreader/compute-core/local-runtime', '@openreader/compute-core/types', and '@openreader/compute-core/api-contracts'.",
|
"The app must communicate with compute-worker through the app-owned HTTP client and generated protocol types.",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const UI_ARCHITECTURE_IMPORT_PATHS = [
|
const UI_ARCHITECTURE_IMPORT_PATHS = [
|
||||||
|
|
@ -124,7 +122,7 @@ const eslintConfig = [
|
||||||
"no-restricted-imports": [
|
"no-restricted-imports": [
|
||||||
"error",
|
"error",
|
||||||
{
|
{
|
||||||
patterns: COMPUTE_CORE_IMPORT_PATTERNS,
|
patterns: COMPUTE_WORKER_IMPORT_PATTERNS,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|
@ -142,7 +140,7 @@ const eslintConfig = [
|
||||||
"error",
|
"error",
|
||||||
{
|
{
|
||||||
paths: UI_ARCHITECTURE_IMPORT_PATHS,
|
paths: UI_ARCHITECTURE_IMPORT_PATHS,
|
||||||
patterns: COMPUTE_CORE_IMPORT_PATTERNS,
|
patterns: COMPUTE_WORKER_IMPORT_PATTERNS,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,6 @@ const securityHeaders = [
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const bundleWorkerCompute = true;
|
|
||||||
const pdfjsTraceFiles = [
|
const pdfjsTraceFiles = [
|
||||||
'./node_modules/pdfjs-dist/package.json',
|
'./node_modules/pdfjs-dist/package.json',
|
||||||
'./node_modules/pdfjs-dist/legacy/build/pdf.mjs',
|
'./node_modules/pdfjs-dist/legacy/build/pdf.mjs',
|
||||||
|
|
@ -29,7 +28,6 @@ const serverExternalPackages = [
|
||||||
// Keep pdfjs-dist as a real package in node_modules. Server-side preview
|
// Keep pdfjs-dist as a real package in node_modules. Server-side preview
|
||||||
// rendering resolves pdf.js runtime assets from the filesystem at runtime.
|
// rendering resolves pdf.js runtime assets from the filesystem at runtime.
|
||||||
'pdfjs-dist',
|
'pdfjs-dist',
|
||||||
...(!bundleWorkerCompute ? ['onnxruntime-node', '@huggingface/tokenizers'] : []),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
const nextConfig: NextConfig = {
|
const nextConfig: NextConfig = {
|
||||||
|
|
@ -88,14 +86,6 @@ const nextConfig: NextConfig = {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
if (isServer && bundleWorkerCompute) {
|
|
||||||
config.resolve.alias = {
|
|
||||||
...(config.resolve.alias || {}),
|
|
||||||
'@openreader/compute-core/local-runtime$': false,
|
|
||||||
'onnxruntime-node$': false,
|
|
||||||
'@huggingface/tokenizers$': false,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return config;
|
return config;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
|
||||||
10
package.json
10
package.json
|
|
@ -8,13 +8,16 @@
|
||||||
"dev:raw": "next dev --turbopack -p 3003",
|
"dev:raw": "next dev --turbopack -p 3003",
|
||||||
"build": "next build",
|
"build": "next build",
|
||||||
"build:bundle-guard": "node scripts/check-next-server-bundle.mjs",
|
"build:bundle-guard": "node scripts/check-next-server-bundle.mjs",
|
||||||
|
"check:compute-boundary": "node scripts/check-compute-boundary.mjs",
|
||||||
"start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw",
|
"start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw",
|
||||||
"start:raw": "next start -p 3003",
|
"start:raw": "next start -p 3003",
|
||||||
"lint": "next lint",
|
"lint": "next lint",
|
||||||
"test": "playwright test",
|
"test": "playwright test",
|
||||||
"test:e2e": "playwright test",
|
"test:e2e": "playwright test",
|
||||||
"test:unit": "vitest run",
|
"test:unit": "vitest run",
|
||||||
"test:compute": "vitest run --project compute-core --project compute-worker",
|
"test:compute": "vitest run --project compute-worker",
|
||||||
|
"compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts",
|
||||||
|
"compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts",
|
||||||
"migrate": "node drizzle/scripts/migrate.mjs",
|
"migrate": "node drizzle/scripts/migrate.mjs",
|
||||||
"migrate-fs": "node scripts/migrate-fs-v2.mjs",
|
"migrate-fs": "node scripts/migrate-fs-v2.mjs",
|
||||||
"migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true",
|
"migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true",
|
||||||
|
|
@ -27,14 +30,13 @@
|
||||||
"docs:clear": "pnpm --dir docs-site clear",
|
"docs:clear": "pnpm --dir docs-site clear",
|
||||||
"compute:worker:dev": "pnpm --filter @openreader/compute-worker dev",
|
"compute:worker:dev": "pnpm --filter @openreader/compute-worker dev",
|
||||||
"compute:worker:start": "pnpm --filter @openreader/compute-worker start",
|
"compute:worker:start": "pnpm --filter @openreader/compute-worker start",
|
||||||
"compute:dev:compose": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --build",
|
"compute:dev:compose": "docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --build",
|
||||||
"lint:route-errors": "node scripts/check-route-error-responses.mjs"
|
"lint:route-errors": "node scripts/check-route-error-responses.mjs"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.1061.0",
|
"@aws-sdk/client-s3": "^3.1061.0",
|
||||||
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
||||||
"@headlessui/react": "^2.2.10",
|
"@headlessui/react": "^2.2.10",
|
||||||
"@huggingface/tokenizers": "^0.1.3",
|
|
||||||
"@microsoft/antissrf": "^1.0.0",
|
"@microsoft/antissrf": "^1.0.0",
|
||||||
"@mozilla/readability": "^0.6.0",
|
"@mozilla/readability": "^0.6.0",
|
||||||
"@napi-rs/canvas": "^0.1.100",
|
"@napi-rs/canvas": "^0.1.100",
|
||||||
|
|
@ -61,7 +63,6 @@
|
||||||
"linkedom": "^0.18.12",
|
"linkedom": "^0.18.12",
|
||||||
"lru-cache": "^11.5.1",
|
"lru-cache": "^11.5.1",
|
||||||
"next": "^15.5.19",
|
"next": "^15.5.19",
|
||||||
"onnxruntime-node": "^1.26.0",
|
|
||||||
"openai": "^6.41.0",
|
"openai": "^6.41.0",
|
||||||
"pdfjs-dist": "4.8.69",
|
"pdfjs-dist": "4.8.69",
|
||||||
"pg": "^8.21.0",
|
"pg": "^8.21.0",
|
||||||
|
|
@ -98,6 +99,7 @@
|
||||||
"eslint": "^9.39.4",
|
"eslint": "^9.39.4",
|
||||||
"eslint-config-next": "^15.5.19",
|
"eslint-config-next": "^15.5.19",
|
||||||
"postcss": "^8.5.15",
|
"postcss": "^8.5.15",
|
||||||
|
"openapi-typescript": "^7.10.1",
|
||||||
"tailwindcss": "^3.4.19",
|
"tailwindcss": "^3.4.19",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^5.9.3",
|
||||||
"vitest": "^4.1.8"
|
"vitest": "^4.1.8"
|
||||||
|
|
|
||||||
316
pnpm-lock.yaml
316
pnpm-lock.yaml
|
|
@ -20,9 +20,6 @@ importers:
|
||||||
'@headlessui/react':
|
'@headlessui/react':
|
||||||
specifier: ^2.2.10
|
specifier: ^2.2.10
|
||||||
version: 2.2.10(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
|
version: 2.2.10(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
|
||||||
'@huggingface/tokenizers':
|
|
||||||
specifier: ^0.1.3
|
|
||||||
version: 0.1.3
|
|
||||||
'@microsoft/antissrf':
|
'@microsoft/antissrf':
|
||||||
specifier: ^1.0.0
|
specifier: ^1.0.0
|
||||||
version: 1.0.0
|
version: 1.0.0
|
||||||
|
|
@ -49,7 +46,7 @@ importers:
|
||||||
version: 7.0.1
|
version: 7.0.1
|
||||||
better-auth:
|
better-auth:
|
||||||
specifier: ^1.6.14
|
specifier: ^1.6.14
|
||||||
version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)))
|
version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)))
|
||||||
better-sqlite3:
|
better-sqlite3:
|
||||||
specifier: ^12.10.0
|
specifier: ^12.10.0
|
||||||
version: 12.10.0
|
version: 12.10.0
|
||||||
|
|
@ -101,9 +98,6 @@ importers:
|
||||||
next:
|
next:
|
||||||
specifier: ^15.5.19
|
specifier: ^15.5.19
|
||||||
version: 15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
|
version: 15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
|
||||||
onnxruntime-node:
|
|
||||||
specifier: ^1.26.0
|
|
||||||
version: 1.26.0
|
|
||||||
openai:
|
openai:
|
||||||
specifier: ^6.41.0
|
specifier: ^6.41.0
|
||||||
version: 6.41.0(zod@4.4.3)
|
version: 6.41.0(zod@4.4.3)
|
||||||
|
|
@ -170,7 +164,7 @@ importers:
|
||||||
version: 1.60.0
|
version: 1.60.0
|
||||||
'@tailwindcss/typography':
|
'@tailwindcss/typography':
|
||||||
specifier: ^0.5.19
|
specifier: ^0.5.19
|
||||||
version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4))
|
version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))
|
||||||
'@types/better-sqlite3':
|
'@types/better-sqlite3':
|
||||||
specifier: ^7.6.13
|
specifier: ^7.6.13
|
||||||
version: 7.6.13
|
version: 7.6.13
|
||||||
|
|
@ -204,27 +198,48 @@ importers:
|
||||||
eslint-config-next:
|
eslint-config-next:
|
||||||
specifier: ^15.5.19
|
specifier: ^15.5.19
|
||||||
version: 15.5.19(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3)
|
version: 15.5.19(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3)
|
||||||
|
openapi-typescript:
|
||||||
|
specifier: ^7.10.1
|
||||||
|
version: 7.13.0(typescript@5.9.3)
|
||||||
postcss:
|
postcss:
|
||||||
specifier: ^8.5.15
|
specifier: ^8.5.15
|
||||||
version: 8.5.15
|
version: 8.5.15
|
||||||
tailwindcss:
|
tailwindcss:
|
||||||
specifier: ^3.4.19
|
specifier: ^3.4.19
|
||||||
version: 3.4.19(tsx@4.22.4)
|
version: 3.4.19(tsx@4.22.4)(yaml@2.9.0)
|
||||||
typescript:
|
typescript:
|
||||||
specifier: ^5.9.3
|
specifier: ^5.9.3
|
||||||
version: 5.9.3
|
version: 5.9.3
|
||||||
vitest:
|
vitest:
|
||||||
specifier: ^4.1.8
|
specifier: ^4.1.8
|
||||||
version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))
|
version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))
|
||||||
|
|
||||||
compute/core:
|
compute-worker:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
'@aws-sdk/client-s3':
|
||||||
|
specifier: ^3.1061.0
|
||||||
|
version: 3.1061.0
|
||||||
|
'@fastify/swagger':
|
||||||
|
specifier: ^9.0.0
|
||||||
|
version: 9.7.0
|
||||||
'@huggingface/tokenizers':
|
'@huggingface/tokenizers':
|
||||||
specifier: ^0.1.3
|
specifier: ^0.1.3
|
||||||
version: 0.1.3
|
version: 0.1.3
|
||||||
'@napi-rs/canvas':
|
'@napi-rs/canvas':
|
||||||
specifier: ^0.1.100
|
specifier: ^0.1.100
|
||||||
version: 0.1.100
|
version: 0.1.100
|
||||||
|
'@nats-io/jetstream':
|
||||||
|
specifier: ^3.4.0
|
||||||
|
version: 3.4.0
|
||||||
|
'@nats-io/kv':
|
||||||
|
specifier: ^3.4.0
|
||||||
|
version: 3.4.0
|
||||||
|
'@nats-io/transport-node':
|
||||||
|
specifier: ^3.4.0
|
||||||
|
version: 3.4.0
|
||||||
|
fastify:
|
||||||
|
specifier: ^5.8.5
|
||||||
|
version: 5.8.5
|
||||||
ffmpeg-static:
|
ffmpeg-static:
|
||||||
specifier: ^5.3.0
|
specifier: ^5.3.0
|
||||||
version: 5.3.0
|
version: 5.3.0
|
||||||
|
|
@ -237,27 +252,6 @@ importers:
|
||||||
pdfjs-dist:
|
pdfjs-dist:
|
||||||
specifier: 4.8.69
|
specifier: 4.8.69
|
||||||
version: 4.8.69
|
version: 4.8.69
|
||||||
|
|
||||||
compute/worker:
|
|
||||||
dependencies:
|
|
||||||
'@aws-sdk/client-s3':
|
|
||||||
specifier: ^3.1061.0
|
|
||||||
version: 3.1061.0
|
|
||||||
'@nats-io/jetstream':
|
|
||||||
specifier: ^3.4.0
|
|
||||||
version: 3.4.0
|
|
||||||
'@nats-io/kv':
|
|
||||||
specifier: ^3.4.0
|
|
||||||
version: 3.4.0
|
|
||||||
'@nats-io/transport-node':
|
|
||||||
specifier: ^3.4.0
|
|
||||||
version: 3.4.0
|
|
||||||
'@openreader/compute-core':
|
|
||||||
specifier: workspace:*
|
|
||||||
version: link:../core
|
|
||||||
fastify:
|
|
||||||
specifier: ^5.8.5
|
|
||||||
version: 5.8.5
|
|
||||||
pino:
|
pino:
|
||||||
specifier: ^10.3.1
|
specifier: ^10.3.1
|
||||||
version: 10.3.1
|
version: 10.3.1
|
||||||
|
|
@ -424,6 +418,14 @@ packages:
|
||||||
resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==}
|
resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==}
|
||||||
engines: {node: '>=18.0.0'}
|
engines: {node: '>=18.0.0'}
|
||||||
|
|
||||||
|
'@babel/code-frame@7.29.7':
|
||||||
|
resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==}
|
||||||
|
engines: {node: '>=6.9.0'}
|
||||||
|
|
||||||
|
'@babel/helper-validator-identifier@7.29.7':
|
||||||
|
resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==}
|
||||||
|
engines: {node: '>=6.9.0'}
|
||||||
|
|
||||||
'@babel/runtime@7.29.7':
|
'@babel/runtime@7.29.7':
|
||||||
resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==}
|
resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==}
|
||||||
engines: {node: '>=6.9.0'}
|
engines: {node: '>=6.9.0'}
|
||||||
|
|
@ -1031,6 +1033,9 @@ packages:
|
||||||
'@fastify/proxy-addr@5.1.0':
|
'@fastify/proxy-addr@5.1.0':
|
||||||
resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==}
|
resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==}
|
||||||
|
|
||||||
|
'@fastify/swagger@9.7.0':
|
||||||
|
resolution: {integrity: sha512-Vp1SC1GC2Hrkd3faFILv86BzUNyFz5N4/xdExqtCgkGASOzn/x+eMe4qXIGq7cdT6wif/P/oa6r1Ruqx19paZA==}
|
||||||
|
|
||||||
'@floating-ui/core@1.7.5':
|
'@floating-ui/core@1.7.5':
|
||||||
resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==}
|
resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==}
|
||||||
|
|
||||||
|
|
@ -1508,6 +1513,16 @@ packages:
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1
|
react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1
|
||||||
|
|
||||||
|
'@redocly/ajv@8.11.2':
|
||||||
|
resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==}
|
||||||
|
|
||||||
|
'@redocly/config@0.22.0':
|
||||||
|
resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==}
|
||||||
|
|
||||||
|
'@redocly/openapi-core@1.34.15':
|
||||||
|
resolution: {integrity: sha512-HAwCnNyKcs5XGQqms+9t7OdAPM/5TDstmhF+0i7tdCFato2QKuYIlyWETwkXd8c5zbltr1oB+6y9NTeQLr2d6Q==}
|
||||||
|
engines: {node: '>=18.17.0', npm: '>=9.5.0'}
|
||||||
|
|
||||||
'@rolldown/binding-android-arm64@1.0.2':
|
'@rolldown/binding-android-arm64@1.0.2':
|
||||||
resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==}
|
resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==}
|
||||||
engines: {node: ^20.19.0 || >=22.12.0}
|
engines: {node: ^20.19.0 || >=22.12.0}
|
||||||
|
|
@ -2036,6 +2051,10 @@ packages:
|
||||||
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
|
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
|
||||||
engines: {node: '>= 6.0.0'}
|
engines: {node: '>= 6.0.0'}
|
||||||
|
|
||||||
|
agent-base@7.1.4:
|
||||||
|
resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==}
|
||||||
|
engines: {node: '>= 14'}
|
||||||
|
|
||||||
ajv-formats@3.0.1:
|
ajv-formats@3.0.1:
|
||||||
resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==}
|
resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
|
|
@ -2050,6 +2069,10 @@ packages:
|
||||||
ajv@8.20.0:
|
ajv@8.20.0:
|
||||||
resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==}
|
resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==}
|
||||||
|
|
||||||
|
ansi-colors@4.1.3:
|
||||||
|
resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==}
|
||||||
|
engines: {node: '>=6'}
|
||||||
|
|
||||||
ansi-regex@5.0.1:
|
ansi-regex@5.0.1:
|
||||||
resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
|
resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
|
|
@ -2384,6 +2407,9 @@ packages:
|
||||||
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
|
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
|
||||||
|
change-case@5.4.4:
|
||||||
|
resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==}
|
||||||
|
|
||||||
character-entities-html4@2.1.0:
|
character-entities-html4@2.1.0:
|
||||||
resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==}
|
resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==}
|
||||||
|
|
||||||
|
|
@ -2421,6 +2447,9 @@ packages:
|
||||||
color-name@1.1.4:
|
color-name@1.1.4:
|
||||||
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
|
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
|
||||||
|
|
||||||
|
colorette@1.4.0:
|
||||||
|
resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==}
|
||||||
|
|
||||||
colorette@2.0.20:
|
colorette@2.0.20:
|
||||||
resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==}
|
resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==}
|
||||||
|
|
||||||
|
|
@ -3011,6 +3040,9 @@ packages:
|
||||||
resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==}
|
resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
fastify-plugin@5.1.0:
|
||||||
|
resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==}
|
||||||
|
|
||||||
fastify@5.8.5:
|
fastify@5.8.5:
|
||||||
resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==}
|
resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==}
|
||||||
|
|
||||||
|
|
@ -3212,6 +3244,10 @@ packages:
|
||||||
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
|
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
|
||||||
engines: {node: '>= 6'}
|
engines: {node: '>= 6'}
|
||||||
|
|
||||||
|
https-proxy-agent@7.0.6:
|
||||||
|
resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}
|
||||||
|
engines: {node: '>= 14'}
|
||||||
|
|
||||||
ieee754@1.2.1:
|
ieee754@1.2.1:
|
||||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||||
|
|
||||||
|
|
@ -3234,6 +3270,10 @@ packages:
|
||||||
resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==}
|
resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==}
|
||||||
engines: {node: '>=0.8.19'}
|
engines: {node: '>=0.8.19'}
|
||||||
|
|
||||||
|
index-to-position@1.2.0:
|
||||||
|
resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==}
|
||||||
|
engines: {node: '>=18'}
|
||||||
|
|
||||||
inherits@2.0.4:
|
inherits@2.0.4:
|
||||||
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
||||||
|
|
||||||
|
|
@ -3413,9 +3453,17 @@ packages:
|
||||||
resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==}
|
resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
|
||||||
|
js-levenshtein@1.1.6:
|
||||||
|
resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==}
|
||||||
|
engines: {node: '>=0.10.0'}
|
||||||
|
|
||||||
js-tokens@4.0.0:
|
js-tokens@4.0.0:
|
||||||
resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==}
|
resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==}
|
||||||
|
|
||||||
|
js-yaml@4.1.1:
|
||||||
|
resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
js-yaml@4.2.0:
|
js-yaml@4.2.0:
|
||||||
resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==}
|
resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
@ -3426,6 +3474,10 @@ packages:
|
||||||
json-schema-ref-resolver@3.0.0:
|
json-schema-ref-resolver@3.0.0:
|
||||||
resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==}
|
resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==}
|
||||||
|
|
||||||
|
json-schema-resolver@3.0.0:
|
||||||
|
resolution: {integrity: sha512-HqMnbz0tz2DaEJ3ntsqtx3ezzZyDE7G56A/pPY/NGmrPu76UzsWquOpHFRAf5beTNXoH2LU5cQePVvRli1nchA==}
|
||||||
|
engines: {node: '>=20'}
|
||||||
|
|
||||||
json-schema-traverse@0.4.1:
|
json-schema-traverse@0.4.1:
|
||||||
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
|
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
|
||||||
|
|
||||||
|
|
@ -3927,6 +3979,15 @@ packages:
|
||||||
zod:
|
zod:
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
openapi-types@12.1.3:
|
||||||
|
resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==}
|
||||||
|
|
||||||
|
openapi-typescript@7.13.0:
|
||||||
|
resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==}
|
||||||
|
hasBin: true
|
||||||
|
peerDependencies:
|
||||||
|
typescript: ^5.x
|
||||||
|
|
||||||
optionator@0.9.4:
|
optionator@0.9.4:
|
||||||
resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
|
resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
|
||||||
engines: {node: '>= 0.8.0'}
|
engines: {node: '>= 0.8.0'}
|
||||||
|
|
@ -3963,6 +4024,10 @@ packages:
|
||||||
parse-entities@4.0.2:
|
parse-entities@4.0.2:
|
||||||
resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
|
resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
|
||||||
|
|
||||||
|
parse-json@8.3.0:
|
||||||
|
resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==}
|
||||||
|
engines: {node: '>=18'}
|
||||||
|
|
||||||
path-exists@4.0.0:
|
path-exists@4.0.0:
|
||||||
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
|
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
|
|
@ -4073,6 +4138,10 @@ packages:
|
||||||
engines: {node: '>=18'}
|
engines: {node: '>=18'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
pluralize@8.0.0:
|
||||||
|
resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==}
|
||||||
|
engines: {node: '>=4'}
|
||||||
|
|
||||||
possible-typed-array-names@1.1.0:
|
possible-typed-array-names@1.1.0:
|
||||||
resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==}
|
resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==}
|
||||||
engines: {node: '>= 0.4'}
|
engines: {node: '>= 0.4'}
|
||||||
|
|
@ -4615,6 +4684,10 @@ packages:
|
||||||
suffix-thumb@5.0.2:
|
suffix-thumb@5.0.2:
|
||||||
resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==}
|
resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==}
|
||||||
|
|
||||||
|
supports-color@10.2.2:
|
||||||
|
resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==}
|
||||||
|
engines: {node: '>=18'}
|
||||||
|
|
||||||
supports-color@7.2.0:
|
supports-color@7.2.0:
|
||||||
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
|
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
|
|
@ -4728,6 +4801,10 @@ packages:
|
||||||
resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==}
|
resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
|
||||||
|
type-fest@4.41.0:
|
||||||
|
resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==}
|
||||||
|
engines: {node: '>=16'}
|
||||||
|
|
||||||
type@2.7.3:
|
type@2.7.3:
|
||||||
resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==}
|
resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==}
|
||||||
|
|
||||||
|
|
@ -4786,6 +4863,9 @@ packages:
|
||||||
unrs-resolver@1.12.2:
|
unrs-resolver@1.12.2:
|
||||||
resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==}
|
resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==}
|
||||||
|
|
||||||
|
uri-js-replace@1.0.1:
|
||||||
|
resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==}
|
||||||
|
|
||||||
uri-js@4.4.1:
|
uri-js@4.4.1:
|
||||||
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
|
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
|
||||||
|
|
||||||
|
|
@ -4943,6 +5023,18 @@ packages:
|
||||||
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
|
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
|
||||||
engines: {node: '>=0.4'}
|
engines: {node: '>=0.4'}
|
||||||
|
|
||||||
|
yaml-ast-parser@0.0.43:
|
||||||
|
resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==}
|
||||||
|
|
||||||
|
yaml@2.9.0:
|
||||||
|
resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==}
|
||||||
|
engines: {node: '>= 14.6'}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
|
yargs-parser@21.1.1:
|
||||||
|
resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==}
|
||||||
|
engines: {node: '>=12'}
|
||||||
|
|
||||||
yocto-queue@0.1.0:
|
yocto-queue@0.1.0:
|
||||||
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
|
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
|
@ -5224,6 +5316,14 @@ snapshots:
|
||||||
|
|
||||||
'@aws/lambda-invoke-store@0.2.4': {}
|
'@aws/lambda-invoke-store@0.2.4': {}
|
||||||
|
|
||||||
|
'@babel/code-frame@7.29.7':
|
||||||
|
dependencies:
|
||||||
|
'@babel/helper-validator-identifier': 7.29.7
|
||||||
|
js-tokens: 4.0.0
|
||||||
|
picocolors: 1.1.1
|
||||||
|
|
||||||
|
'@babel/helper-validator-identifier@7.29.7': {}
|
||||||
|
|
||||||
'@babel/runtime@7.29.7': {}
|
'@babel/runtime@7.29.7': {}
|
||||||
|
|
||||||
'@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)':
|
'@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)':
|
||||||
|
|
@ -5546,7 +5646,7 @@ snapshots:
|
||||||
'@eslint/config-array@0.21.2':
|
'@eslint/config-array@0.21.2':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@eslint/object-schema': 2.1.7
|
'@eslint/object-schema': 2.1.7
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
minimatch: 3.1.5
|
minimatch: 3.1.5
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- supports-color
|
- supports-color
|
||||||
|
|
@ -5562,7 +5662,7 @@ snapshots:
|
||||||
'@eslint/eslintrc@3.3.5':
|
'@eslint/eslintrc@3.3.5':
|
||||||
dependencies:
|
dependencies:
|
||||||
ajv: 6.15.0
|
ajv: 6.15.0
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
espree: 10.4.0
|
espree: 10.4.0
|
||||||
globals: 14.0.0
|
globals: 14.0.0
|
||||||
ignore: 5.3.2
|
ignore: 5.3.2
|
||||||
|
|
@ -5605,6 +5705,16 @@ snapshots:
|
||||||
'@fastify/forwarded': 3.0.1
|
'@fastify/forwarded': 3.0.1
|
||||||
ipaddr.js: 2.4.0
|
ipaddr.js: 2.4.0
|
||||||
|
|
||||||
|
'@fastify/swagger@9.7.0':
|
||||||
|
dependencies:
|
||||||
|
fastify-plugin: 5.1.0
|
||||||
|
json-schema-resolver: 3.0.0
|
||||||
|
openapi-types: 12.1.3
|
||||||
|
rfdc: 1.4.1
|
||||||
|
yaml: 2.9.0
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
'@floating-ui/core@1.7.5':
|
'@floating-ui/core@1.7.5':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@floating-ui/utils': 0.2.11
|
'@floating-ui/utils': 0.2.11
|
||||||
|
|
@ -5968,6 +6078,29 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
react: 19.2.7
|
react: 19.2.7
|
||||||
|
|
||||||
|
'@redocly/ajv@8.11.2':
|
||||||
|
dependencies:
|
||||||
|
fast-deep-equal: 3.1.3
|
||||||
|
json-schema-traverse: 1.0.0
|
||||||
|
require-from-string: 2.0.2
|
||||||
|
uri-js-replace: 1.0.1
|
||||||
|
|
||||||
|
'@redocly/config@0.22.0': {}
|
||||||
|
|
||||||
|
'@redocly/openapi-core@1.34.15(supports-color@10.2.2)':
|
||||||
|
dependencies:
|
||||||
|
'@redocly/ajv': 8.11.2
|
||||||
|
'@redocly/config': 0.22.0
|
||||||
|
colorette: 1.4.0
|
||||||
|
https-proxy-agent: 7.0.6(supports-color@10.2.2)
|
||||||
|
js-levenshtein: 1.1.6
|
||||||
|
js-yaml: 4.1.1
|
||||||
|
minimatch: 5.1.9
|
||||||
|
pluralize: 8.0.0
|
||||||
|
yaml-ast-parser: 0.0.43
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
'@rolldown/binding-android-arm64@1.0.2':
|
'@rolldown/binding-android-arm64@1.0.2':
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
|
@ -6088,10 +6221,10 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
tslib: 2.8.1
|
tslib: 2.8.1
|
||||||
|
|
||||||
'@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4))':
|
'@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))':
|
||||||
dependencies:
|
dependencies:
|
||||||
postcss-selector-parser: 6.0.10
|
postcss-selector-parser: 6.0.10
|
||||||
tailwindcss: 3.4.19(tsx@4.22.4)
|
tailwindcss: 3.4.19(tsx@4.22.4)(yaml@2.9.0)
|
||||||
|
|
||||||
'@tanstack/query-core@5.101.0': {}
|
'@tanstack/query-core@5.101.0': {}
|
||||||
|
|
||||||
|
|
@ -6220,7 +6353,7 @@ snapshots:
|
||||||
'@typescript-eslint/types': 8.60.1
|
'@typescript-eslint/types': 8.60.1
|
||||||
'@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3)
|
'@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3)
|
||||||
'@typescript-eslint/visitor-keys': 8.60.1
|
'@typescript-eslint/visitor-keys': 8.60.1
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
eslint: 9.39.4(jiti@1.21.7)
|
eslint: 9.39.4(jiti@1.21.7)
|
||||||
typescript: 5.9.3
|
typescript: 5.9.3
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
|
|
@ -6230,7 +6363,7 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3)
|
'@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3)
|
||||||
'@typescript-eslint/types': 8.60.1
|
'@typescript-eslint/types': 8.60.1
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
typescript: 5.9.3
|
typescript: 5.9.3
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- supports-color
|
- supports-color
|
||||||
|
|
@ -6249,7 +6382,7 @@ snapshots:
|
||||||
'@typescript-eslint/types': 8.60.1
|
'@typescript-eslint/types': 8.60.1
|
||||||
'@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3)
|
'@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3)
|
||||||
'@typescript-eslint/utils': 8.60.1(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3)
|
'@typescript-eslint/utils': 8.60.1(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3)
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
eslint: 9.39.4(jiti@1.21.7)
|
eslint: 9.39.4(jiti@1.21.7)
|
||||||
ts-api-utils: 2.5.0(typescript@5.9.3)
|
ts-api-utils: 2.5.0(typescript@5.9.3)
|
||||||
typescript: 5.9.3
|
typescript: 5.9.3
|
||||||
|
|
@ -6264,7 +6397,7 @@ snapshots:
|
||||||
'@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3)
|
'@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3)
|
||||||
'@typescript-eslint/types': 8.60.1
|
'@typescript-eslint/types': 8.60.1
|
||||||
'@typescript-eslint/visitor-keys': 8.60.1
|
'@typescript-eslint/visitor-keys': 8.60.1
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
minimatch: 10.2.5
|
minimatch: 10.2.5
|
||||||
semver: 7.8.1
|
semver: 7.8.1
|
||||||
tinyglobby: 0.2.17
|
tinyglobby: 0.2.17
|
||||||
|
|
@ -6375,13 +6508,13 @@ snapshots:
|
||||||
chai: 6.2.2
|
chai: 6.2.2
|
||||||
tinyrainbow: 3.1.0
|
tinyrainbow: 3.1.0
|
||||||
|
|
||||||
'@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))':
|
'@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@vitest/spy': 4.1.8
|
'@vitest/spy': 4.1.8
|
||||||
estree-walker: 3.0.3
|
estree-walker: 3.0.3
|
||||||
magic-string: 0.30.21
|
magic-string: 0.30.21
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)
|
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)
|
||||||
|
|
||||||
'@vitest/pretty-format@4.1.8':
|
'@vitest/pretty-format@4.1.8':
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|
@ -6425,10 +6558,12 @@ snapshots:
|
||||||
|
|
||||||
agent-base@6.0.2:
|
agent-base@6.0.2:
|
||||||
dependencies:
|
dependencies:
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- supports-color
|
- supports-color
|
||||||
|
|
||||||
|
agent-base@7.1.4: {}
|
||||||
|
|
||||||
ajv-formats@3.0.1(ajv@8.20.0):
|
ajv-formats@3.0.1(ajv@8.20.0):
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
ajv: 8.20.0
|
ajv: 8.20.0
|
||||||
|
|
@ -6447,6 +6582,8 @@ snapshots:
|
||||||
json-schema-traverse: 1.0.0
|
json-schema-traverse: 1.0.0
|
||||||
require-from-string: 2.0.2
|
require-from-string: 2.0.2
|
||||||
|
|
||||||
|
ansi-colors@4.1.3: {}
|
||||||
|
|
||||||
ansi-regex@5.0.1: {}
|
ansi-regex@5.0.1: {}
|
||||||
|
|
||||||
ansi-regex@6.2.2: {}
|
ansi-regex@6.2.2: {}
|
||||||
|
|
@ -6632,7 +6769,7 @@ snapshots:
|
||||||
|
|
||||||
base64-js@1.5.1: {}
|
base64-js@1.5.1: {}
|
||||||
|
|
||||||
better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))):
|
better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))):
|
||||||
dependencies:
|
dependencies:
|
||||||
'@better-auth/core': 1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)
|
'@better-auth/core': 1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)
|
||||||
'@better-auth/drizzle-adapter': 1.6.14(@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.1)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))
|
'@better-auth/drizzle-adapter': 1.6.14(@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.1)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))
|
||||||
|
|
@ -6659,7 +6796,7 @@ snapshots:
|
||||||
pg: 8.21.0
|
pg: 8.21.0
|
||||||
react: 19.2.7
|
react: 19.2.7
|
||||||
react-dom: 19.2.7(react@19.2.7)
|
react-dom: 19.2.7(react@19.2.7)
|
||||||
vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))
|
vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- '@cloudflare/workers-types'
|
- '@cloudflare/workers-types'
|
||||||
- '@opentelemetry/api'
|
- '@opentelemetry/api'
|
||||||
|
|
@ -6765,6 +6902,8 @@ snapshots:
|
||||||
ansi-styles: 4.3.0
|
ansi-styles: 4.3.0
|
||||||
supports-color: 7.2.0
|
supports-color: 7.2.0
|
||||||
|
|
||||||
|
change-case@5.4.4: {}
|
||||||
|
|
||||||
character-entities-html4@2.1.0: {}
|
character-entities-html4@2.1.0: {}
|
||||||
|
|
||||||
character-entities-legacy@3.0.0: {}
|
character-entities-legacy@3.0.0: {}
|
||||||
|
|
@ -6799,6 +6938,8 @@ snapshots:
|
||||||
|
|
||||||
color-name@1.1.4: {}
|
color-name@1.1.4: {}
|
||||||
|
|
||||||
|
colorette@1.4.0: {}
|
||||||
|
|
||||||
colorette@2.0.20: {}
|
colorette@2.0.20: {}
|
||||||
|
|
||||||
comma-separated-tokens@2.0.3: {}
|
comma-separated-tokens@2.0.3: {}
|
||||||
|
|
@ -6896,9 +7037,11 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
ms: 2.1.3
|
ms: 2.1.3
|
||||||
|
|
||||||
debug@4.4.3:
|
debug@4.4.3(supports-color@10.2.2):
|
||||||
dependencies:
|
dependencies:
|
||||||
ms: 2.1.3
|
ms: 2.1.3
|
||||||
|
optionalDependencies:
|
||||||
|
supports-color: 10.2.2
|
||||||
|
|
||||||
decode-named-character-reference@1.3.0:
|
decode-named-character-reference@1.3.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|
@ -7265,7 +7408,7 @@ snapshots:
|
||||||
eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@1.21.7)):
|
eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@1.21.7)):
|
||||||
dependencies:
|
dependencies:
|
||||||
'@nolyfill/is-core-module': 1.0.39
|
'@nolyfill/is-core-module': 1.0.39
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
eslint: 9.39.4(jiti@1.21.7)
|
eslint: 9.39.4(jiti@1.21.7)
|
||||||
get-tsconfig: 4.14.0
|
get-tsconfig: 4.14.0
|
||||||
is-bun-module: 2.0.0
|
is-bun-module: 2.0.0
|
||||||
|
|
@ -7390,7 +7533,7 @@ snapshots:
|
||||||
ajv: 6.15.0
|
ajv: 6.15.0
|
||||||
chalk: 4.1.2
|
chalk: 4.1.2
|
||||||
cross-spawn: 7.0.6
|
cross-spawn: 7.0.6
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
escape-string-regexp: 4.0.0
|
escape-string-regexp: 4.0.0
|
||||||
eslint-scope: 8.4.0
|
eslint-scope: 8.4.0
|
||||||
eslint-visitor-keys: 4.2.1
|
eslint-visitor-keys: 4.2.1
|
||||||
|
|
@ -7535,6 +7678,8 @@ snapshots:
|
||||||
strnum: 2.3.0
|
strnum: 2.3.0
|
||||||
xml-naming: 0.1.0
|
xml-naming: 0.1.0
|
||||||
|
|
||||||
|
fastify-plugin@5.1.0: {}
|
||||||
|
|
||||||
fastify@5.8.5:
|
fastify@5.8.5:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@fastify/ajv-compiler': 4.0.5
|
'@fastify/ajv-compiler': 4.0.5
|
||||||
|
|
@ -7777,7 +7922,14 @@ snapshots:
|
||||||
https-proxy-agent@5.0.1:
|
https-proxy-agent@5.0.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
agent-base: 6.0.2
|
agent-base: 6.0.2
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
|
https-proxy-agent@7.0.6(supports-color@10.2.2):
|
||||||
|
dependencies:
|
||||||
|
agent-base: 7.1.4
|
||||||
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- supports-color
|
- supports-color
|
||||||
|
|
||||||
|
|
@ -7796,6 +7948,8 @@ snapshots:
|
||||||
|
|
||||||
imurmurhash@0.1.4: {}
|
imurmurhash@0.1.4: {}
|
||||||
|
|
||||||
|
index-to-position@1.2.0: {}
|
||||||
|
|
||||||
inherits@2.0.4: {}
|
inherits@2.0.4: {}
|
||||||
|
|
||||||
ini@1.3.8: {}
|
ini@1.3.8: {}
|
||||||
|
|
@ -7972,8 +8126,14 @@ snapshots:
|
||||||
|
|
||||||
joycon@3.1.1: {}
|
joycon@3.1.1: {}
|
||||||
|
|
||||||
|
js-levenshtein@1.1.6: {}
|
||||||
|
|
||||||
js-tokens@4.0.0: {}
|
js-tokens@4.0.0: {}
|
||||||
|
|
||||||
|
js-yaml@4.1.1:
|
||||||
|
dependencies:
|
||||||
|
argparse: 2.0.1
|
||||||
|
|
||||||
js-yaml@4.2.0:
|
js-yaml@4.2.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
argparse: 2.0.1
|
argparse: 2.0.1
|
||||||
|
|
@ -7984,6 +8144,14 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
dequal: 2.0.3
|
dequal: 2.0.3
|
||||||
|
|
||||||
|
json-schema-resolver@3.0.0:
|
||||||
|
dependencies:
|
||||||
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
|
fast-uri: 3.1.2
|
||||||
|
rfdc: 1.4.1
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
json-schema-traverse@0.4.1: {}
|
json-schema-traverse@0.4.1: {}
|
||||||
|
|
||||||
json-schema-traverse@1.0.0: {}
|
json-schema-traverse@1.0.0: {}
|
||||||
|
|
@ -8482,7 +8650,7 @@ snapshots:
|
||||||
micromark@4.0.2:
|
micromark@4.0.2:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/debug': 4.1.13
|
'@types/debug': 4.1.13
|
||||||
debug: 4.4.3
|
debug: 4.4.3(supports-color@10.2.2)
|
||||||
decode-named-character-reference: 1.3.0
|
decode-named-character-reference: 1.3.0
|
||||||
devlop: 1.1.0
|
devlop: 1.1.0
|
||||||
micromark-core-commonmark: 2.0.3
|
micromark-core-commonmark: 2.0.3
|
||||||
|
|
@ -8658,6 +8826,18 @@ snapshots:
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
zod: 4.4.3
|
zod: 4.4.3
|
||||||
|
|
||||||
|
openapi-types@12.1.3: {}
|
||||||
|
|
||||||
|
openapi-typescript@7.13.0(typescript@5.9.3):
|
||||||
|
dependencies:
|
||||||
|
'@redocly/openapi-core': 1.34.15(supports-color@10.2.2)
|
||||||
|
ansi-colors: 4.1.3
|
||||||
|
change-case: 5.4.4
|
||||||
|
parse-json: 8.3.0
|
||||||
|
supports-color: 10.2.2
|
||||||
|
typescript: 5.9.3
|
||||||
|
yargs-parser: 21.1.1
|
||||||
|
|
||||||
optionator@0.9.4:
|
optionator@0.9.4:
|
||||||
dependencies:
|
dependencies:
|
||||||
deep-is: 0.1.4
|
deep-is: 0.1.4
|
||||||
|
|
@ -8705,6 +8885,12 @@ snapshots:
|
||||||
is-decimal: 2.0.1
|
is-decimal: 2.0.1
|
||||||
is-hexadecimal: 2.0.1
|
is-hexadecimal: 2.0.1
|
||||||
|
|
||||||
|
parse-json@8.3.0:
|
||||||
|
dependencies:
|
||||||
|
'@babel/code-frame': 7.29.7
|
||||||
|
index-to-position: 1.2.0
|
||||||
|
type-fest: 4.41.0
|
||||||
|
|
||||||
path-exists@4.0.0: {}
|
path-exists@4.0.0: {}
|
||||||
|
|
||||||
path-expression-matcher@1.5.0: {}
|
path-expression-matcher@1.5.0: {}
|
||||||
|
|
@ -8819,6 +9005,8 @@ snapshots:
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
fsevents: 2.3.2
|
fsevents: 2.3.2
|
||||||
|
|
||||||
|
pluralize@8.0.0: {}
|
||||||
|
|
||||||
possible-typed-array-names@1.1.0: {}
|
possible-typed-array-names@1.1.0: {}
|
||||||
|
|
||||||
postcss-import@15.1.0(postcss@8.5.15):
|
postcss-import@15.1.0(postcss@8.5.15):
|
||||||
|
|
@ -8833,13 +9021,14 @@ snapshots:
|
||||||
camelcase-css: 2.0.1
|
camelcase-css: 2.0.1
|
||||||
postcss: 8.5.15
|
postcss: 8.5.15
|
||||||
|
|
||||||
postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4):
|
postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0):
|
||||||
dependencies:
|
dependencies:
|
||||||
lilconfig: 3.1.3
|
lilconfig: 3.1.3
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
jiti: 1.21.7
|
jiti: 1.21.7
|
||||||
postcss: 8.5.15
|
postcss: 8.5.15
|
||||||
tsx: 4.22.4
|
tsx: 4.22.4
|
||||||
|
yaml: 2.9.0
|
||||||
|
|
||||||
postcss-nested@6.2.0(postcss@8.5.15):
|
postcss-nested@6.2.0(postcss@8.5.15):
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|
@ -9504,6 +9693,8 @@ snapshots:
|
||||||
|
|
||||||
suffix-thumb@5.0.2: {}
|
suffix-thumb@5.0.2: {}
|
||||||
|
|
||||||
|
supports-color@10.2.2: {}
|
||||||
|
|
||||||
supports-color@7.2.0:
|
supports-color@7.2.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
has-flag: 4.0.0
|
has-flag: 4.0.0
|
||||||
|
|
@ -9512,7 +9703,7 @@ snapshots:
|
||||||
|
|
||||||
tabbable@6.4.0: {}
|
tabbable@6.4.0: {}
|
||||||
|
|
||||||
tailwindcss@3.4.19(tsx@4.22.4):
|
tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0):
|
||||||
dependencies:
|
dependencies:
|
||||||
'@alloc/quick-lru': 5.2.0
|
'@alloc/quick-lru': 5.2.0
|
||||||
arg: 5.0.2
|
arg: 5.0.2
|
||||||
|
|
@ -9531,7 +9722,7 @@ snapshots:
|
||||||
postcss: 8.5.15
|
postcss: 8.5.15
|
||||||
postcss-import: 15.1.0(postcss@8.5.15)
|
postcss-import: 15.1.0(postcss@8.5.15)
|
||||||
postcss-js: 4.1.0(postcss@8.5.15)
|
postcss-js: 4.1.0(postcss@8.5.15)
|
||||||
postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)
|
postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0)
|
||||||
postcss-nested: 6.2.0(postcss@8.5.15)
|
postcss-nested: 6.2.0(postcss@8.5.15)
|
||||||
postcss-selector-parser: 6.1.2
|
postcss-selector-parser: 6.1.2
|
||||||
resolve: 1.22.12
|
resolve: 1.22.12
|
||||||
|
|
@ -9651,6 +9842,8 @@ snapshots:
|
||||||
|
|
||||||
type-fest@0.20.2: {}
|
type-fest@0.20.2: {}
|
||||||
|
|
||||||
|
type-fest@4.41.0: {}
|
||||||
|
|
||||||
type@2.7.3: {}
|
type@2.7.3: {}
|
||||||
|
|
||||||
typed-array-buffer@1.0.3:
|
typed-array-buffer@1.0.3:
|
||||||
|
|
@ -9761,6 +9954,8 @@ snapshots:
|
||||||
'@unrs/resolver-binding-win32-ia32-msvc': 1.12.2
|
'@unrs/resolver-binding-win32-ia32-msvc': 1.12.2
|
||||||
'@unrs/resolver-binding-win32-x64-msvc': 1.12.2
|
'@unrs/resolver-binding-win32-x64-msvc': 1.12.2
|
||||||
|
|
||||||
|
uri-js-replace@1.0.1: {}
|
||||||
|
|
||||||
uri-js@4.4.1:
|
uri-js@4.4.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
punycode: 2.3.1
|
punycode: 2.3.1
|
||||||
|
|
@ -9783,7 +9978,7 @@ snapshots:
|
||||||
'@types/unist': 3.0.3
|
'@types/unist': 3.0.3
|
||||||
vfile-message: 4.0.3
|
vfile-message: 4.0.3
|
||||||
|
|
||||||
vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4):
|
vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0):
|
||||||
dependencies:
|
dependencies:
|
||||||
lightningcss: 1.32.0
|
lightningcss: 1.32.0
|
||||||
picomatch: 4.0.4
|
picomatch: 4.0.4
|
||||||
|
|
@ -9796,11 +9991,12 @@ snapshots:
|
||||||
fsevents: 2.3.3
|
fsevents: 2.3.3
|
||||||
jiti: 1.21.7
|
jiti: 1.21.7
|
||||||
tsx: 4.22.4
|
tsx: 4.22.4
|
||||||
|
yaml: 2.9.0
|
||||||
|
|
||||||
vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)):
|
vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)):
|
||||||
dependencies:
|
dependencies:
|
||||||
'@vitest/expect': 4.1.8
|
'@vitest/expect': 4.1.8
|
||||||
'@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))
|
'@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))
|
||||||
'@vitest/pretty-format': 4.1.8
|
'@vitest/pretty-format': 4.1.8
|
||||||
'@vitest/runner': 4.1.8
|
'@vitest/runner': 4.1.8
|
||||||
'@vitest/snapshot': 4.1.8
|
'@vitest/snapshot': 4.1.8
|
||||||
|
|
@ -9817,7 +10013,7 @@ snapshots:
|
||||||
tinyexec: 1.2.4
|
tinyexec: 1.2.4
|
||||||
tinyglobby: 0.2.17
|
tinyglobby: 0.2.17
|
||||||
tinyrainbow: 3.1.0
|
tinyrainbow: 3.1.0
|
||||||
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)
|
vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)
|
||||||
why-is-node-running: 2.3.0
|
why-is-node-running: 2.3.0
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
'@types/node': 20.19.41
|
'@types/node': 20.19.41
|
||||||
|
|
@ -9898,6 +10094,12 @@ snapshots:
|
||||||
|
|
||||||
xtend@4.0.2: {}
|
xtend@4.0.2: {}
|
||||||
|
|
||||||
|
yaml-ast-parser@0.0.43: {}
|
||||||
|
|
||||||
|
yaml@2.9.0: {}
|
||||||
|
|
||||||
|
yargs-parser@21.1.1: {}
|
||||||
|
|
||||||
yocto-queue@0.1.0: {}
|
yocto-queue@0.1.0: {}
|
||||||
|
|
||||||
zip-stream@6.0.1:
|
zip-stream@6.0.1:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
packages:
|
packages:
|
||||||
- .
|
- .
|
||||||
- compute/*
|
- compute-worker
|
||||||
- docker/*
|
- docker/*
|
||||||
|
|
||||||
allowBuilds:
|
allowBuilds:
|
||||||
|
|
|
||||||
39
scripts/check-compute-boundary.mjs
Normal file
39
scripts/check-compute-boundary.mjs
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
const root = process.cwd();
|
||||||
|
const forbidden = [
|
||||||
|
'@openreader/compute-core',
|
||||||
|
'@openreader/compute-worker',
|
||||||
|
'compute/core',
|
||||||
|
'compute/worker',
|
||||||
|
'compute-worker/src',
|
||||||
|
];
|
||||||
|
const failures = [];
|
||||||
|
|
||||||
|
function walk(dir) {
|
||||||
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||||
|
const fullPath = path.join(dir, entry.name);
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
walk(fullPath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!entry.isFile() || !/\.(ts|tsx|js|mjs|cjs)$/.test(entry.name)) continue;
|
||||||
|
const text = fs.readFileSync(fullPath, 'utf8');
|
||||||
|
for (const pattern of forbidden) {
|
||||||
|
if (text.includes(pattern)) failures.push({ file: fullPath, pattern });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
walk(path.join(root, 'src'));
|
||||||
|
|
||||||
|
if (failures.length > 0) {
|
||||||
|
console.error('[compute-boundary] Forbidden app-to-worker source coupling detected:');
|
||||||
|
for (const failure of failures) {
|
||||||
|
console.error(`- ${failure.pattern} in ${path.relative(root, failure.file)}`);
|
||||||
|
}
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.info('[compute-boundary] OK: app communicates with compute worker through protocol/client only');
|
||||||
|
|
@ -12,7 +12,8 @@ if (!fs.existsSync(serverDir)) {
|
||||||
const forbidden = [
|
const forbidden = [
|
||||||
'onnxruntime-node',
|
'onnxruntime-node',
|
||||||
'@huggingface/tokenizers',
|
'@huggingface/tokenizers',
|
||||||
'@openreader/compute-core/local-runtime',
|
'@openreader/compute-worker',
|
||||||
|
'/compute-worker/src/compute/',
|
||||||
];
|
];
|
||||||
|
|
||||||
const includeExt = new Set(['.js', '.mjs', '.cjs']);
|
const includeExt = new Set(['.js', '.mjs', '.cjs']);
|
||||||
|
|
|
||||||
|
|
@ -222,7 +222,7 @@ function spawnMainCommand(command, env) {
|
||||||
function resolveEmbeddedWorkerLaunch() {
|
function resolveEmbeddedWorkerLaunch() {
|
||||||
const candidateDirs = [
|
const candidateDirs = [
|
||||||
path.join(process.cwd(), 'embedded-compute-worker'),
|
path.join(process.cwd(), 'embedded-compute-worker'),
|
||||||
path.join(process.cwd(), 'compute', 'worker'),
|
path.join(process.cwd(), 'compute-worker'),
|
||||||
];
|
];
|
||||||
|
|
||||||
for (const workerDir of candidateDirs) {
|
for (const workerDir of candidateDirs) {
|
||||||
|
|
@ -237,7 +237,7 @@ function resolveEmbeddedWorkerLaunch() {
|
||||||
|
|
||||||
throw new Error(
|
throw new Error(
|
||||||
'Could not find an embedded compute worker runtime. '
|
'Could not find an embedded compute worker runtime. '
|
||||||
+ 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute/worker available locally.',
|
+ 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute-worker available locally.',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import { and, eq, inArray } from 'drizzle-orm';
|
||||||
import { db } from '@/db';
|
import { db } from '@/db';
|
||||||
import { documents } from '@/db/schema';
|
import { documents } from '@/db/schema';
|
||||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker';
|
import { getComputeWorkerClient } from '@/lib/server/compute-worker/client';
|
||||||
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
import { isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||||
import {
|
import {
|
||||||
fetchPdfParseOperation,
|
fetchPdfParseOperation,
|
||||||
|
|
@ -45,14 +45,6 @@ async function loadOwnedDocumentRow(input: {
|
||||||
return rows[0] ?? null;
|
return rows[0] ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function workerEventsUrl(baseUrl: string, opId: string, sinceEventId: string | null): string {
|
|
||||||
const url = new URL(`${baseUrl}/ops/${encodeURIComponent(opId)}/events`);
|
|
||||||
if (sinceEventId) {
|
|
||||||
url.searchParams.set('sinceEventId', sinceEventId);
|
|
||||||
}
|
|
||||||
return url.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
|
||||||
const { logger } = createRequestLogger({
|
const { logger } = createRequestLogger({
|
||||||
route: '/api/documents/[id]/parsed/events',
|
route: '/api/documents/[id]/parsed/events',
|
||||||
|
|
@ -92,17 +84,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string
|
||||||
return NextResponse.json({ error: 'Operation not found' }, { status: 404 });
|
return NextResponse.json({ error: 'Operation not found' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const cfg = getWorkerClientConfigFromEnv();
|
|
||||||
const lastEventId = req.headers.get('last-event-id');
|
const lastEventId = req.headers.get('last-event-id');
|
||||||
const sinceEventId = req.nextUrl.searchParams.get('sinceEventId') || lastEventId;
|
const sinceEventId = req.nextUrl.searchParams.get('sinceEventId') || lastEventId;
|
||||||
const upstream = await fetch(workerEventsUrl(cfg.baseUrl, opId, sinceEventId), {
|
const upstream = await getComputeWorkerClient().openOperationEvents(opId, {
|
||||||
method: 'GET',
|
sinceEventId,
|
||||||
headers: {
|
lastEventId,
|
||||||
Authorization: `Bearer ${cfg.token}`,
|
|
||||||
Accept: 'text/event-stream',
|
|
||||||
...(lastEventId ? { 'Last-Event-ID': lastEventId } : {}),
|
|
||||||
},
|
|
||||||
cache: 'no-store',
|
|
||||||
signal: req.signal,
|
signal: req.signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
249
src/lib/server/compute-worker/client.ts
Normal file
249
src/lib/server/compute-worker/client.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
||||||
|
import type {
|
||||||
|
PdfLayoutRequest,
|
||||||
|
PdfLayoutResolution,
|
||||||
|
PdfLayoutResult,
|
||||||
|
WhisperAlignRequest,
|
||||||
|
WhisperAlignResult,
|
||||||
|
WorkerOperation,
|
||||||
|
WorkerOperationEvent,
|
||||||
|
} from './protocol';
|
||||||
|
import { isWorkerOperation } from './protocol';
|
||||||
|
|
||||||
|
class WorkerHttpError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
readonly status: number,
|
||||||
|
readonly retryAfterMs: number | null = null,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = 'WorkerHttpError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readRequiredEnv(name: string): string {
|
||||||
|
const value = process.env[name]?.trim();
|
||||||
|
if (!value) throw new Error(`${name} is required for compute worker client`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeWorkerBaseUrl(raw: string): string {
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty');
|
||||||
|
const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed)
|
||||||
|
? trimmed
|
||||||
|
: (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed)
|
||||||
|
? `http://${trimmed}`
|
||||||
|
: `https://${trimmed}`);
|
||||||
|
const parsed = new URL(withScheme);
|
||||||
|
parsed.pathname = parsed.pathname.replace(/\/+$/, '');
|
||||||
|
return parsed.toString().replace(/\/+$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getComputeWorkerConfigFromEnv(): { baseUrl: string; token: string } {
|
||||||
|
return {
|
||||||
|
baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')),
|
||||||
|
token: readRequiredEnv('COMPUTE_WORKER_TOKEN'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isComputeWorkerAvailable(): boolean {
|
||||||
|
try {
|
||||||
|
getComputeWorkerConfigFromEnv();
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPositiveIntEnv(name: string, fallback: number): number {
|
||||||
|
const raw = process.env[name]?.trim();
|
||||||
|
if (!raw) return fallback;
|
||||||
|
const parsed = Number(raw);
|
||||||
|
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function alignmentWaitTimeoutMs(): number {
|
||||||
|
return Math.max(60_000, readPositiveIntEnv('COMPUTE_WHISPER_TIMEOUT_MS', 30_000) + 15_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRetryAfterMs(value: string | null): number | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const seconds = Number(value);
|
||||||
|
if (Number.isFinite(seconds)) return Math.max(0, Math.floor(seconds * 1000));
|
||||||
|
const date = Date.parse(value);
|
||||||
|
return Number.isNaN(date) ? null : Math.max(0, date - Date.now());
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSseFrame(frame: string): { id: number | null; data: string | null } {
|
||||||
|
let id: number | null = null;
|
||||||
|
const data: string[] = [];
|
||||||
|
for (const line of frame.split(/\r?\n/)) {
|
||||||
|
if (line.startsWith('id:')) {
|
||||||
|
const parsed = Number(line.slice(3).trim());
|
||||||
|
if (Number.isFinite(parsed) && parsed >= 0) id = Math.floor(parsed);
|
||||||
|
} else if (line.startsWith('data:')) {
|
||||||
|
data.push(line.slice(5).trimStart());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { id, data: data.length > 0 ? data.join('\n') : null };
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTerminal(operation: WorkerOperation): boolean {
|
||||||
|
return operation.status === 'succeeded' || operation.status === 'failed';
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ComputeWorkerClient {
|
||||||
|
private readonly baseUrl: string;
|
||||||
|
private readonly token: string;
|
||||||
|
|
||||||
|
constructor(config = getComputeWorkerConfigFromEnv()) {
|
||||||
|
this.baseUrl = config.baseUrl;
|
||||||
|
this.token = config.token;
|
||||||
|
}
|
||||||
|
|
||||||
|
async alignWords(input: WhisperAlignRequest): Promise<WhisperAlignResult> {
|
||||||
|
let lastError: unknown;
|
||||||
|
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const operation = await this.requestJson<WorkerOperation<WhisperAlignResult>>(
|
||||||
|
'POST',
|
||||||
|
'/v1/whisper-align/operations',
|
||||||
|
input,
|
||||||
|
);
|
||||||
|
const final = isTerminal(operation)
|
||||||
|
? operation
|
||||||
|
: await this.waitForOperation<WhisperAlignResult>(operation.opId, alignmentWaitTimeoutMs());
|
||||||
|
if (final.status !== 'succeeded' || !final.result) {
|
||||||
|
throw new Error(final.error?.message || 'Whisper worker operation did not complete');
|
||||||
|
}
|
||||||
|
return final.result;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error;
|
||||||
|
if (attempt >= 2 || !this.shouldRetry(error)) break;
|
||||||
|
const delay = error instanceof WorkerHttpError && error.retryAfterMs !== null
|
||||||
|
? error.retryAfterMs
|
||||||
|
: attempt * 250;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError instanceof Error ? lastError : new Error('Unknown compute worker failure');
|
||||||
|
}
|
||||||
|
|
||||||
|
createPdfLayoutOperation(input: PdfLayoutRequest): Promise<WorkerOperation<PdfLayoutResult>> {
|
||||||
|
return this.requestJson('POST', '/v1/pdf-layout/operations', input);
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvePdfLayout(input: PdfLayoutRequest): Promise<PdfLayoutResolution> {
|
||||||
|
return this.requestJson('POST', '/v1/pdf-layout/resolve', {
|
||||||
|
documentId: input.documentId,
|
||||||
|
namespace: input.namespace,
|
||||||
|
documentObjectKey: input.documentObjectKey,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async getOperation<Result>(opId: string): Promise<WorkerOperation<Result> | null> {
|
||||||
|
try {
|
||||||
|
return await this.requestJson('GET', `/v1/operations/${encodeURIComponent(opId)}`);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof WorkerHttpError && error.status === 404) return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async openOperationEvents(opId: string, input?: {
|
||||||
|
sinceEventId?: string | null;
|
||||||
|
lastEventId?: string | null;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<Response> {
|
||||||
|
const url = new URL(`${this.baseUrl}/v1/operations/${encodeURIComponent(opId)}/events`);
|
||||||
|
if (input?.sinceEventId) url.searchParams.set('sinceEventId', input.sinceEventId);
|
||||||
|
return fetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${this.token}`,
|
||||||
|
Accept: 'text/event-stream',
|
||||||
|
...(input?.lastEventId ? { 'Last-Event-ID': input.lastEventId } : {}),
|
||||||
|
},
|
||||||
|
cache: 'no-store',
|
||||||
|
signal: input?.signal,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async requestJson<T>(method: 'GET' | 'POST', path: string, body?: unknown): Promise<T> {
|
||||||
|
const response = await fetch(`${this.baseUrl}${path}`, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${this.token}`,
|
||||||
|
Accept: 'application/json',
|
||||||
|
...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}),
|
||||||
|
},
|
||||||
|
...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}),
|
||||||
|
cache: 'no-store',
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const detail = await response.text().catch(() => '');
|
||||||
|
throw new WorkerHttpError(
|
||||||
|
`Worker request failed (${method} ${path}): ${response.status}${detail ? ` ${detail}` : ''}`,
|
||||||
|
response.status,
|
||||||
|
parseRetryAfterMs(response.headers.get('retry-after')),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const parsed = await response.json() as T;
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private shouldRetry(error: unknown): boolean {
|
||||||
|
if (error instanceof WorkerHttpError) {
|
||||||
|
return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504;
|
||||||
|
}
|
||||||
|
return error instanceof Error && /network|timeout|fetch failed/i.test(error.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async waitForOperation<Result>(opId: string, timeoutMs: number): Promise<WorkerOperation<Result>> {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
let lastEventId: number | null = null;
|
||||||
|
try {
|
||||||
|
while (!controller.signal.aborted) {
|
||||||
|
const response = await this.openOperationEvents(opId, {
|
||||||
|
sinceEventId: lastEventId === null ? null : String(lastEventId),
|
||||||
|
lastEventId: lastEventId === null ? null : String(lastEventId),
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
if (!response.ok || !response.body) {
|
||||||
|
throw new WorkerHttpError(`Worker event stream failed: ${response.status}`, response.status);
|
||||||
|
}
|
||||||
|
const reader = response.body.getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let buffer = '';
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
buffer += decoder.decode(value, { stream: true });
|
||||||
|
while (true) {
|
||||||
|
const end = buffer.indexOf('\n\n');
|
||||||
|
if (end < 0) break;
|
||||||
|
const frame = parseSseFrame(buffer.slice(0, end));
|
||||||
|
buffer = buffer.slice(end + 2);
|
||||||
|
if (frame.id !== null) lastEventId = frame.id;
|
||||||
|
if (!frame.data) continue;
|
||||||
|
const event = JSON.parse(frame.data) as WorkerOperationEvent<Result>;
|
||||||
|
if (!event?.snapshot || !isWorkerOperation(event.snapshot) || event.snapshot.opId !== opId) continue;
|
||||||
|
if (isTerminal(event.snapshot)) return event.snapshot;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||||
|
}
|
||||||
|
throw new Error(`Operation stream timed out for ${opId}`);
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let client: ComputeWorkerClient | null = null;
|
||||||
|
|
||||||
|
export function getComputeWorkerClient(): ComputeWorkerClient {
|
||||||
|
client ??= new ComputeWorkerClient();
|
||||||
|
return client;
|
||||||
|
}
|
||||||
700
src/lib/server/compute-worker/generated.ts
Normal file
700
src/lib/server/compute-worker/generated.ts
Normal file
|
|
@ -0,0 +1,700 @@
|
||||||
|
/**
|
||||||
|
* This file was auto-generated by openapi-typescript.
|
||||||
|
* Do not make direct changes to the file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface paths {
|
||||||
|
"/health/live": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Default Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
ok: boolean;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
put?: never;
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/health/ready": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Default Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
ok: boolean;
|
||||||
|
natsConnected: boolean;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
put?: never;
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/v1/whisper-align/operations": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get?: never;
|
||||||
|
put?: never;
|
||||||
|
post: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody: {
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
text: string;
|
||||||
|
lang?: string;
|
||||||
|
cacheKey?: string;
|
||||||
|
audioObjectKey: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
responses: {
|
||||||
|
/** @description Default Response */
|
||||||
|
202: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
400: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/v1/pdf-layout/operations": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get?: never;
|
||||||
|
put?: never;
|
||||||
|
post: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody: {
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
documentObjectKey: string;
|
||||||
|
replaceToken?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
responses: {
|
||||||
|
/** @description Default Response */
|
||||||
|
202: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
400: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/v1/pdf-layout/resolve": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get?: never;
|
||||||
|
put?: never;
|
||||||
|
post: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody: {
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
documentObjectKey: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
responses: {
|
||||||
|
/** @description Default Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
artifact: {
|
||||||
|
objectKey: string;
|
||||||
|
} | null;
|
||||||
|
operation: {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
} | null;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
400: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/v1/operations/{opId}": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
opId: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Default Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
400: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
404: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
put?: never;
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
"/v1/operations/{opId}/events": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get: {
|
||||||
|
parameters: {
|
||||||
|
query?: {
|
||||||
|
sinceEventId?: string | number;
|
||||||
|
};
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
opId: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Server-sent PublicOperationEvent stream */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
400: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Default Response */
|
||||||
|
404: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
put?: never;
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
export type webhooks = Record<string, never>;
|
||||||
|
export interface components {
|
||||||
|
schemas: {
|
||||||
|
ParsedPdfDocument: {
|
||||||
|
/** @enum {number} */
|
||||||
|
schemaVersion: 1;
|
||||||
|
documentId: string;
|
||||||
|
parserVersion: string;
|
||||||
|
parsedAt: number;
|
||||||
|
pages: {
|
||||||
|
pageNumber: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
blocks: {
|
||||||
|
id: string;
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "abstract" | "algorithm" | "aside_text" | "chart" | "content" | "formula" | "doc_title" | "figure_title" | "footer" | "footnote" | "formula_number" | "header" | "image" | "number" | "paragraph_title" | "reference" | "reference_content" | "seal" | "table" | "text" | "vision_footnote";
|
||||||
|
fragments: {
|
||||||
|
page: number;
|
||||||
|
bbox: [
|
||||||
|
number,
|
||||||
|
number,
|
||||||
|
number,
|
||||||
|
number
|
||||||
|
];
|
||||||
|
text: string;
|
||||||
|
readingOrder: number;
|
||||||
|
modelConfidence?: number;
|
||||||
|
}[];
|
||||||
|
text: string;
|
||||||
|
parentSectionId?: string;
|
||||||
|
}[];
|
||||||
|
}[];
|
||||||
|
};
|
||||||
|
TTSSentenceAlignment: {
|
||||||
|
sentence: string;
|
||||||
|
sentenceIndex: number;
|
||||||
|
words: {
|
||||||
|
text: string;
|
||||||
|
startSec: number;
|
||||||
|
endSec: number;
|
||||||
|
charStart: number;
|
||||||
|
charEnd: number;
|
||||||
|
}[];
|
||||||
|
};
|
||||||
|
ArtifactReference: {
|
||||||
|
objectKey: string;
|
||||||
|
};
|
||||||
|
ErrorResponse: {
|
||||||
|
error: string;
|
||||||
|
} & {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
OperationError: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
PdfLayoutProgress: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
PublicOperation: {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
PublicOperationEvent: {
|
||||||
|
eventId: number;
|
||||||
|
snapshot: {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
PdfLayoutResolution: {
|
||||||
|
artifact: {
|
||||||
|
objectKey: string;
|
||||||
|
} | null;
|
||||||
|
operation: {
|
||||||
|
opId: string;
|
||||||
|
subject: {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "whisper_align";
|
||||||
|
} | {
|
||||||
|
/** @enum {string} */
|
||||||
|
kind: "pdf_layout";
|
||||||
|
documentId: string;
|
||||||
|
namespace: string | null;
|
||||||
|
};
|
||||||
|
/** @enum {string} */
|
||||||
|
status: "queued" | "running" | "succeeded" | "failed";
|
||||||
|
queuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
startedAt?: number;
|
||||||
|
result?: unknown;
|
||||||
|
error?: {
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
};
|
||||||
|
timing?: {
|
||||||
|
queueWaitMs?: number;
|
||||||
|
s3FetchMs?: number;
|
||||||
|
computeMs?: number;
|
||||||
|
};
|
||||||
|
progress?: {
|
||||||
|
totalPages: number;
|
||||||
|
pagesParsed: number;
|
||||||
|
currentPage?: number;
|
||||||
|
/** @enum {string} */
|
||||||
|
phase: "infer" | "merge";
|
||||||
|
};
|
||||||
|
} | null;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
responses: never;
|
||||||
|
parameters: never;
|
||||||
|
requestBodies: never;
|
||||||
|
headers: never;
|
||||||
|
pathItems: never;
|
||||||
|
}
|
||||||
|
export type $defs = Record<string, never>;
|
||||||
|
export type operations = Record<string, never>;
|
||||||
57
src/lib/server/compute-worker/protocol.ts
Normal file
57
src/lib/server/compute-worker/protocol.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
import type { components, paths } from './generated';
|
||||||
|
|
||||||
|
export type ParsedPdfDocument = components['schemas']['ParsedPdfDocument'];
|
||||||
|
export type ParsedPdfPage = ParsedPdfDocument['pages'][number];
|
||||||
|
export type ParsedPdfBlock = ParsedPdfPage['blocks'][number];
|
||||||
|
export type ParsedPdfBlockKind = ParsedPdfBlock['kind'];
|
||||||
|
export type ParsedPdfBlockFragment = ParsedPdfBlock['fragments'][number];
|
||||||
|
|
||||||
|
export type TTSSentenceAlignment = components['schemas']['TTSSentenceAlignment'];
|
||||||
|
export type TTSSentenceWord = TTSSentenceAlignment['words'][number];
|
||||||
|
|
||||||
|
export type PdfLayoutProgress = NonNullable<components['schemas']['PublicOperation']['progress']>;
|
||||||
|
export type WorkerOperationStatus = components['schemas']['PublicOperation']['status'];
|
||||||
|
export type WorkerOperationSubject = components['schemas']['PublicOperation']['subject'];
|
||||||
|
|
||||||
|
export type WorkerOperation<Result = unknown> =
|
||||||
|
Omit<components['schemas']['PublicOperation'], 'result'>
|
||||||
|
& { result?: Result };
|
||||||
|
|
||||||
|
export type WorkerOperationEvent<Result = unknown> = {
|
||||||
|
eventId: number;
|
||||||
|
snapshot: WorkerOperation<Result>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type WhisperAlignRequest =
|
||||||
|
paths['/v1/whisper-align/operations']['post']['requestBody']['content']['application/json'];
|
||||||
|
export type PdfLayoutRequest =
|
||||||
|
paths['/v1/pdf-layout/operations']['post']['requestBody']['content']['application/json'];
|
||||||
|
export type PdfLayoutResolveRequest =
|
||||||
|
paths['/v1/pdf-layout/resolve']['post']['requestBody']['content']['application/json'];
|
||||||
|
|
||||||
|
export type WhisperAlignResult = {
|
||||||
|
alignments: TTSSentenceAlignment[];
|
||||||
|
timing?: components['schemas']['PublicOperation']['timing'];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type PdfLayoutResult = {
|
||||||
|
parsedObjectKey: string;
|
||||||
|
timing?: components['schemas']['PublicOperation']['timing'];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type PdfLayoutResolution = {
|
||||||
|
artifact: { objectKey: string } | null;
|
||||||
|
operation: WorkerOperation<PdfLayoutResult> | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isWorkerOperation(value: unknown): value is WorkerOperation {
|
||||||
|
if (!value || typeof value !== 'object') return false;
|
||||||
|
const record = value as Record<string, unknown>;
|
||||||
|
return typeof record.opId === 'string'
|
||||||
|
&& !!record.subject
|
||||||
|
&& typeof record.subject === 'object'
|
||||||
|
&& typeof (record.subject as Record<string, unknown>).kind === 'string'
|
||||||
|
&& typeof record.status === 'string'
|
||||||
|
&& typeof record.queuedAt === 'number'
|
||||||
|
&& typeof record.updatedAt === 'number';
|
||||||
|
}
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
export function isAbortLikeError(error: unknown): boolean {
|
|
||||||
if (!error) return false;
|
|
||||||
if (error instanceof DOMException) return error.name === 'AbortError';
|
|
||||||
if (error instanceof Error) return error.name === 'AbortError' || error.message === 'This operation was aborted';
|
|
||||||
if (typeof error === 'object' && error !== null && 'name' in error) {
|
|
||||||
return (error as { name?: unknown }).name === 'AbortError';
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
import type { ComputeBackend } from '@/lib/server/compute/types';
|
|
||||||
import { isWorkerClientConfigAvailable, WorkerComputeBackend } from '@/lib/server/compute/worker';
|
|
||||||
|
|
||||||
let backendPromise: Promise<ComputeBackend> | null = null;
|
|
||||||
|
|
||||||
export async function getCompute(): Promise<ComputeBackend> {
|
|
||||||
if (!backendPromise) {
|
|
||||||
backendPromise = Promise.resolve(new WorkerComputeBackend()).catch((error) => {
|
|
||||||
backendPromise = null;
|
|
||||||
throw error;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return backendPromise;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isComputeAvailable(): boolean {
|
|
||||||
return isWorkerClientConfigAvailable();
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue