diff --git a/.dockerignore b/.dockerignore index 6c309d7..51d3428 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .env .env.* +**/*.creds README.md .next node_modules diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2c50173..fedd0d3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -62,13 +62,13 @@ jobs: platform: linux/amd64 runner: ubuntu-24.04 context: . - dockerfile: ./compute/worker/Dockerfile + dockerfile: ./packages/compute-worker/Dockerfile - image_target: compute-worker arch: arm64 platform: linux/arm64 runner: ubuntu-24.04-arm context: . - dockerfile: ./compute/worker/Dockerfile + dockerfile: ./packages/compute-worker/Dockerfile steps: - name: Checkout repository diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 27c2102..c659070 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -25,3 +25,5 @@ jobs: run: pnpm migrate - name: Run Vitest suites run: pnpm test:unit + - name: Verify compute worker contract and boundary + run: pnpm compute:openapi:check && pnpm check:compute-boundary diff --git a/.gitignore b/.gitignore index 55b2755..466d577 100644 --- a/.gitignore +++ b/.gitignore @@ -56,5 +56,7 @@ node_modules/ # vscode .vscode -# .agents +# Agents .agents +.codex +.claude diff --git a/Dockerfile b/Dockerfile index c6b5ac6..f3396d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ WORKDIR /app # Copy workspace manifests needed for dependency installation COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute/core/package.json ./compute/core/package.json -COPY compute/worker/package.json ./compute/worker/package.json -COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json +COPY packages/bootstrap/package.json ./packages/bootstrap/package.json +COPY packages/compute-worker/package.json ./packages/compute-worker/package.json +COPY packages/database/package.json ./packages/database/package.json # Install dependencies RUN pnpm install --frozen-lockfile @@ -33,8 +33,9 @@ COPY . . # Build the Next.js application RUN pnpm exec next telemetry disable RUN AUTH_SECRET=build-placeholder-secret-value-32chars!! BASE_URL=http://localhost:3003 pnpm build -RUN pnpm --config.inject-workspace-packages=true --filter @openreader/entrypoint-migration-tools deploy /opt/entrypoint-migration-tools -RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/embedded-compute-worker +RUN pnpm --config.inject-workspace-packages=true --filter @openreader/bootstrap deploy /opt/openreader/bootstrap +RUN pnpm --dir /opt/openreader/bootstrap rebuild better-sqlite3 ffmpeg-static +RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/openreader/embedded-compute-worker # Generate third-party dependency license report plus copied license files. RUN mkdir -p /app/THIRD_PARTY_LICENSES && \ pnpm dlx license-checker-rseidelsohn@4.3.0 \ @@ -63,32 +64,15 @@ COPY --from=app-builder /app/.next/standalone ./ COPY --from=app-builder /app/.next/static ./.next/static COPY --from=app-builder /app/public ./public -# Copy the entrypoint and migration/runtime helper files it invokes directly. -COPY --from=app-builder /app/scripts/openreader-entrypoint.mjs ./scripts/openreader-entrypoint.mjs -COPY --from=app-builder /app/scripts/migrate-fs-v2.mjs ./scripts/migrate-fs-v2.mjs -COPY --from=app-builder /app/drizzle ./drizzle -COPY --from=app-builder /app/drizzle.config.pg.ts ./drizzle.config.pg.ts -COPY --from=app-builder /app/drizzle.config.sqlite.ts ./drizzle.config.sqlite.ts -COPY --from=app-builder /app/src/db ./src/db - -# Merge in the dependency subset needed by the entrypoint migration scripts. -COPY --from=app-builder /opt/entrypoint-migration-tools/node_modules /tmp/runtime-tools-node_modules -RUN mkdir -p /app/node_modules && \ - rm -rf /tmp/runtime-tools-node_modules/@aws-sdk \ - /tmp/runtime-tools-node_modules/better-sqlite3 \ - /tmp/runtime-tools-node_modules/ffmpeg-static \ - /tmp/runtime-tools-node_modules/pg && \ - cp -an /tmp/runtime-tools-node_modules/. /app/node_modules/ && \ - rm -rf /tmp/runtime-tools-node_modules - -# Ship the embedded compute worker as a separate deployed bundle. -COPY --from=app-builder /opt/embedded-compute-worker ./embedded-compute-worker +# Ship startup orchestration and the embedded worker as independent deployed bundles. +COPY --from=app-builder /opt/openreader/bootstrap /opt/openreader/bootstrap +COPY --from=app-builder /opt/openreader/embedded-compute-worker /opt/openreader/embedded-compute-worker # Include third-party license report and copied license texts at a stable path in the image. COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses # Include SeaweedFS license text for the copied weed binary. COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt # Include static model notices for runtime-downloaded assets. -COPY --from=app-builder /app/compute/core/src/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt +COPY --from=app-builder /app/packages/compute-worker/src/inference/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt # Copy seaweedfs weed binary for optional embedded local S3. COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed @@ -98,7 +82,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server RUN chmod +x /usr/local/bin/nats-server # Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts. -COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt +COPY --from=app-builder /app/packages/compute-worker/src/inference/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt # Match the app's historical container port now that standalone server.js # is started directly instead of `next start -p 3003`. @@ -108,5 +92,5 @@ ENV PORT=3003 EXPOSE 3003 # Start the application -ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"] +ENTRYPOINT ["node", "/opt/openreader/bootstrap/src/cli.mjs", "--"] CMD ["node", "server.js"] diff --git a/compute/core/package.json b/compute/core/package.json deleted file mode 100644 index 4f27980..0000000 --- a/compute/core/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "@openreader/compute-core", - "version": "0.0.0", - "private": true, - "type": "module", - "dependencies": { - "@huggingface/tokenizers": "^0.1.3", - "@napi-rs/canvas": "^0.1.100", - "ffmpeg-static": "^5.3.0", - "jszip": "^3.10.1", - "onnxruntime-node": "^1.26.0", - "pdfjs-dist": "4.8.69" - }, - "exports": { - ".": "./src/index.ts", - "./local-runtime": "./src/local-runtime.ts", - "./api-contracts": "./src/api-contracts/index.ts", - "./control-plane": "./src/control-plane/index.ts", - "./types": "./src/types/index.ts" - } -} diff --git a/compute/core/src/config/cpu-budget.ts b/compute/core/src/config/cpu-budget.ts deleted file mode 100644 index 50d161b..0000000 --- a/compute/core/src/config/cpu-budget.ts +++ /dev/null @@ -1,28 +0,0 @@ -import os from 'node:os'; - -function readPositiveInt(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -export function getComputeJobConcurrency(): number { - return readPositiveInt('COMPUTE_JOB_CONCURRENCY', 1); -} - -export function getAvailableCpuCores(): number { - if (typeof os.availableParallelism === 'function') { - const value = os.availableParallelism(); - if (Number.isFinite(value) && value >= 1) return Math.floor(value); - } - const fallback = os.cpus().length; - return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1; -} - -export function getOnnxThreadsPerJob(): number { - const concurrency = getComputeJobConcurrency(); - const usableCores = Math.max(1, getAvailableCpuCores() - 1); - return Math.max(1, Math.floor(usableCores / concurrency)); -} diff --git a/compute/core/src/index.ts b/compute/core/src/index.ts deleted file mode 100644 index ea7b118..0000000 --- a/compute/core/src/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -export * from './api-contracts'; -export { - getComputeJobConcurrency, - getAvailableCpuCores, - getOnnxThreadsPerJob, -} from './config/cpu-budget'; -export { - getComputeTimeoutConfig, - getComputeOpStaleMs, - getWorkerClientWaitTimeoutMs, - withTimeout, - withIdleTimeoutAndHardCap, - type ComputeTimeoutConfig, - type ComputeOperationKind, - type IdleTimeoutAndHardCapInput, -} from './config/timeout'; -export { renderPage } from './pdf/render'; -export { mergeTextWithRegions } from './pdf/merge'; -export { PDF_PARSER_VERSION } from './pdf/parser-version'; -export { encodeParserVersion } from './pdf/parser-version-key'; -export { stitchCrossPageBlocks } from './pdf/stitch'; -export { normalizeTextItemsForLayout } from './pdf/normalize-text'; -export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map'; -export { buildGoertzelCoefficients, goertzelPower } from './whisper/spectral'; -export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './whisper/token-timestamps'; -export * from './control-plane'; diff --git a/compute/core/src/pdf/parser-version-key.ts b/compute/core/src/pdf/parser-version-key.ts deleted file mode 100644 index f9f82b2..0000000 --- a/compute/core/src/pdf/parser-version-key.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { PDF_PARSER_VERSION } from './parser-version'; - -export function encodeParserVersion( - parserVersion: string, - defaultVersion = PDF_PARSER_VERSION, -): string { - const normalized = parserVersion.trim() || defaultVersion; - return encodeURIComponent(normalized); -} diff --git a/compute/core/src/pdf/parser-version.ts b/compute/core/src/pdf/parser-version.ts deleted file mode 100644 index b5fa7d7..0000000 --- a/compute/core/src/pdf/parser-version.ts +++ /dev/null @@ -1 +0,0 @@ -export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69'; diff --git a/compute/core/src/pdf/types.ts b/compute/core/src/pdf/types.ts deleted file mode 100644 index 38089a6..0000000 --- a/compute/core/src/pdf/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { ParsedPdfBlockKind } from '../types/parsed-pdf'; - -export interface PdfTextItem { - text: string; - x: number; - y: number; - width: number; - height: number; -} - -export interface LayoutRegion { - bbox: [number, number, number, number]; - label: ParsedPdfBlockKind; - confidence?: number; -} diff --git a/compute/core/src/platform/docstore.ts b/compute/core/src/platform/docstore.ts deleted file mode 100644 index 0b62401..0000000 --- a/compute/core/src/platform/docstore.ts +++ /dev/null @@ -1,21 +0,0 @@ -import fs from 'fs'; -import path from 'path'; - -function findMonorepoRoot(startDir: string): string | null { - let current = path.resolve(startDir); - for (;;) { - const marker = path.join(current, 'pnpm-workspace.yaml'); - if (fs.existsSync(marker)) return current; - const parent = path.dirname(current); - if (parent === current) return null; - current = parent; - } -} - -function resolveDocstoreDir(): string { - const repoRoot = findMonorepoRoot(process.cwd()); - if (repoRoot) return path.join(repoRoot, 'docstore'); - return path.join(process.cwd(), 'docstore'); -} - -export const DOCSTORE_DIR = resolveDocstoreDir(); diff --git a/compute/core/src/platform/ffmpeg.ts b/compute/core/src/platform/ffmpeg.ts deleted file mode 100644 index b4ed669..0000000 --- a/compute/core/src/platform/ffmpeg.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { existsSync } from 'fs'; -import ffmpegStatic from 'ffmpeg-static'; - -function normalizePath(value: unknown): string | null { - if (typeof value !== 'string') return null; - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : null; -} - -function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string { - if (envValue) { - if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) { - throw new Error(`${envVarName} points to a missing binary: ${envValue}`); - } - return envValue; - } - - if (!bundledValue) { - throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`); - } - - if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) { - throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`); - } - - return bundledValue; -} - -export function getFFmpegPath(): string { - return resolveBinary( - normalizePath(process.env.FFMPEG_BIN), - normalizePath(ffmpegStatic), - 'FFMPEG_BIN', - 'ffmpeg-static', - ); -} diff --git a/compute/core/src/types/index.ts b/compute/core/src/types/index.ts deleted file mode 100644 index 7408914..0000000 --- a/compute/core/src/types/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export type { - TTSAudioBuffer, - TTSAudioBytes, - TTSSentenceAlignment, - TTSSentenceWord, -} from './tts'; - -export type { - ParsedPdfBlock, - ParsedPdfBlockFragment, - ParsedPdfBlockKind, - ParsedPdfDocument, - ParsedPdfPage, - PdfParsePhase, - PdfParseProgress, - PdfParseStatus, -} from './parsed-pdf'; diff --git a/compute/core/src/types/tts.ts b/compute/core/src/types/tts.ts deleted file mode 100644 index 73db759..0000000 --- a/compute/core/src/types/tts.ts +++ /dev/null @@ -1,16 +0,0 @@ -export type TTSAudioBuffer = ArrayBuffer; -export type TTSAudioBytes = number[]; - -export interface TTSSentenceWord { - text: string; - startSec: number; - endSec: number; - charStart: number; - charEnd: number; -} - -export interface TTSSentenceAlignment { - sentence: string; - sentenceIndex: number; - words: TTSSentenceWord[]; -} diff --git a/compute/core/src/whisper/alignment-map.ts b/compute/core/src/whisper/alignment-map.ts deleted file mode 100644 index a246cec..0000000 --- a/compute/core/src/whisper/alignment-map.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; - -// Worker-side mirror of the app's canonical audio-text cleaning rules in -// `src/lib/shared/audio-text.ts`. This is a separate build target and cannot -// import from `@/lib`, so the rules are duplicated here on purpose. The word -// `charStart`/`charEnd` offsets this module emits are consumed against text -// normalized by that shared module, so any divergence shifts viewer highlights -// off-word — keep this byte-for-byte in sync with `audio-text.ts`. -const STRIPPED_GLYPHS = /[*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖]/g; - -function preprocessSentenceForAudio(text: string): string { - return text - .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') - .replace(STRIPPED_GLYPHS, '') - .replace(/\s+/g, ' ') - .trim(); -} - -export interface WhisperWord { - start: number; - end: number; - word: string; -} - -export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { - const normalizedSentence = preprocessSentenceForAudio(sentence); - const lowerSentence = normalizedSentence.toLowerCase(); - let cursor = 0; - - const alignedWords: TTSSentenceWord[] = words.map((w) => { - const token = w.word.trim(); - if (!token) { - return { - text: '', - startSec: w.start, - endSec: w.end, - charStart: cursor, - charEnd: cursor, - }; - } - - const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); - const start = idx >= 0 ? idx : cursor; - const end = Math.min(normalizedSentence.length, start + token.length); - cursor = Math.max(cursor, end); - - return { - text: token, - startSec: w.start, - endSec: w.end, - charStart: start, - charEnd: end, - }; - }).filter((word) => word.text.length > 0); - - return { - sentence, - sentenceIndex: 0, - words: alignedWords, - }; -} diff --git a/compute/core/src/whisper/spectral.ts b/compute/core/src/whisper/spectral.ts deleted file mode 100644 index b7223cc..0000000 --- a/compute/core/src/whisper/spectral.ts +++ /dev/null @@ -1,21 +0,0 @@ -export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array { - const coeffs = new Float64Array(freqBins); - for (let k = 0; k < freqBins; k += 1) { - coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize); - } - return coeffs; -} - -export function goertzelPower(samples: Float32Array, coeff: number): number { - let s1 = 0; - let s2 = 0; - for (let i = 0; i < samples.length; i += 1) { - const s0 = samples[i] + (coeff * s1) - s2; - s2 = s1; - s1 = s0; - } - - const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2); - if (!Number.isFinite(power) || power < 0) return 0; - return power; -} diff --git a/compute/core/tsconfig.json b/compute/core/tsconfig.json deleted file mode 100644 index 3e68ae5..0000000 --- a/compute/core/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "noEmit": true - }, - "include": ["src/**/*.ts"] -} diff --git a/compute/worker/docker-compose.yml b/compute/worker/docker-compose.yml deleted file mode 100644 index 1bbf550..0000000 --- a/compute/worker/docker-compose.yml +++ /dev/null @@ -1,47 +0,0 @@ -services: - nats: - image: nats:2.14-alpine - container_name: openreader-compute-nats - command: ["-js", "-sd", "/data"] - ports: - - "4222:4222" - - "8222:8222" - volumes: - - nats-data:/data - - compute-worker: - build: - context: ../.. - dockerfile: compute/worker/Dockerfile - container_name: openreader-compute-worker - depends_on: - - nats - env_file: - - ./.env - environment: - NATS_URL: ${NATS_URL:-nats://nats:4222} - COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0} - PORT: ${PORT:-8081} - COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} - S3_PREFIX: ${S3_PREFIX:-openreader} - COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-false} - ports: - - "8081:8081" - develop: - watch: - - action: sync+restart - path: . - target: /workspace - - action: rebuild - path: ../../compute/core - - action: rebuild - path: ./package.json - - action: rebuild - path: ../../compute/core/package.json - - action: rebuild - path: ../../pnpm-lock.yaml - - action: rebuild - path: ../../pnpm-workspace.yaml - -volumes: - nats-data: diff --git a/compute/worker/src/runtime.ts b/compute/worker/src/runtime.ts deleted file mode 100644 index a5099a6..0000000 --- a/compute/worker/src/runtime.ts +++ /dev/null @@ -1,1674 +0,0 @@ -import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; -import { z } from 'zod'; -import { - connect, - nanos, - credsAuthenticator, - type NatsConnection, -} from '@nats-io/transport-node'; -import { - AckPolicy, - DeliverPolicy, - ReplayPolicy, - RetentionPolicy, - StorageType, - jetstream, - jetstreamManager, - type Consumer, - type JetStreamClient, - type JetStreamManager, - type JsMsg, -} from '@nats-io/jetstream'; -import { Kvm } from '@nats-io/kv'; -import { - ensureComputeModels, - runPdfLayoutFromPdfBuffer, - runWhisperAlignmentFromAudioBuffer, -} from '@openreader/compute-core/local-runtime'; -import { - getComputeTimeoutConfig, - getComputeOpStaleMs, - getAvailableCpuCores, - getOnnxThreadsPerJob, - PDF_PARSER_VERSION, - encodeParserVersion, - withIdleTimeoutAndHardCap, - withTimeout, -} from '@openreader/compute-core'; -import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerJobState, - WorkerJobTiming, - WorkerOperationKind, - WorkerOperationRequest, - WorkerOperationState, - PdfLayoutProgress, -} from '@openreader/compute-core/api-contracts'; -import { - DeleteObjectCommand, - GetObjectCommand, - HeadObjectCommand, - PutObjectCommand, - S3Client, -} from '@aws-sdk/client-s3'; -import { - JetStreamOperationEventStream, - OP_EVENTS_SUBJECT_WILDCARD, - JetStreamOperationQueue, - JetStreamOperationStateStore, - hashOpKey, -} from './control-plane/jetstream'; -import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; -import { - recoverOrphanedOperations, - type StreamedOperationState, -} from './orphan-recovery'; -import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; -import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; -import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; - -const JOBS_STREAM_NAME = 'compute_jobs'; -const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; -const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; -const WHISPER_CONSUMER_NAME = 'compute_whisper'; -const LAYOUT_CONSUMER_NAME = 'compute_layout'; -const EVENTS_STREAM_NAME = 'compute_events'; -const COMPUTE_STATE_BUCKET = 'compute_state'; -const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; -const LOOP_ERROR_BACKOFF_MS = 500; -const RUNNING_HEARTBEAT_MS = 5000; -const OP_EVENTS_KEEPALIVE_MS = 15_000; -// Reconnection delay handed to the browser EventSource via the SSE `retry:` -// directive. When a silent stream is torn down for idle sleep, this keeps the -// client from immediately reconnecting and re-waking the worker; instead it -// reconnects on a slow cadence so the container stays asleep most of the time. -const OP_EVENTS_RECONNECT_HINT_MS = 120_000; -const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; -const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; -const WHISPER_MAX_DELIVER = 1; -const NATS_API_TIMEOUT_MS = 60_000; -// Disconnect from NATS after this much continuous idle so the worker stops -// generating outbound traffic (pull polling + keepalive PINGs) and Railway can -// put it to sleep. Reconnect happens lazily on the next inbound request. -const IDLE_DISCONNECT_MS = 120_000; -const IDLE_CHECK_INTERVAL_MS = 5_000; -const IDLE_STATUS_LOG_INTERVAL_MS = 60_000; -const ORPHAN_SWEEP_INTERVAL_MS = 15_000; -// Bounded pull window so consumer loops yield periodically and can be stopped -// cleanly when going idle, instead of blocking on a long-lived pull. -const PULL_EXPIRES_MS = 5_000; -const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); -const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); -const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { - whisper_align: 15_000, - pdf_layout: 120_000, -}; - -interface QueuedJob { - jobId: string; - opId: string; - opKey: string; - kind: WorkerOperationKind; - queuedAt: number; - payload: TPayload; -} - -interface NatsSession { - nc: NatsConnection; - js: JetStreamClient; - jsm: JetStreamManager; - kv: Awaited>; - whisperConsumer: Consumer; - layoutConsumer: Consumer; -} - -interface OperationEventStreamLike { - subscribe(input: { - opId: string; - sinceEventId?: number; - onEvent: (event: WorkerOperationEvent) => void | Promise; - onError?: (error: unknown) => void; - }): Promise<() => void>; -} - -interface OperationStateStoreLike { - getOpState(opId: string): Promise; - getOpStateRecord?(opId: string): Promise<{ state: StreamedOperationState; revision: number } | null>; - getOpIndex?(opKey: string): Promise<{ opId: string } | null>; - listOpStates?(): Promise; -} - -interface OrchestratorLike { - enqueueOrReuse(request: WorkerOperationRequest): Promise; - markRunning(input: { - opId: string; - startedAt?: number; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markProgress(input: { - opId: string; - progress: WorkerOperationState['progress']; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markSucceeded(input: { - opId: string; - result: unknown; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markFailed(input: { - opId: string; - error: { message: string; code?: string } | string; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markFailedIfUnchanged?(input: { - current: StreamedOperationState; - expectedRevision: number; - error: { message: string; code?: string } | string; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; -} - -export interface ComputeWorkerRouteDeps { - orchestrator: OrchestratorLike; - operationStateStore: OperationStateStoreLike; - operationEventStream: OperationEventStreamLike; -} - -export interface CreateComputeWorkerAppOptions { - host?: string; - port?: number; - workerToken?: string; - routeDeps?: ComputeWorkerRouteDeps; - disableWorkers?: boolean; -} - -export interface ComputeWorkerApp { - app: FastifyInstance; - host: string; - port: number; - start(options?: { registerSignalHandlers?: boolean }): Promise; - close(): Promise; -} - -function requireEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} - -function readIntEnv(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -function normalizeNatsReplicas(value: number): number { - if (value === 3 || value === 5) return value; - return 1; -} - -function parseBoolEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const normalized = raw.toLowerCase(); - return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; -} - -function buildLoggerConfig(): boolean | Record { - const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); - const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; - if (format === 'json') { - return { - level, - base: null, - }; - } - return { - level, - base: null, - transport: { - target: 'pino-pretty', - options: { - colorize: true, - translateTime: 'SYS:standard', - ignore: 'pid,hostname', - }, - }, - }; -} - -function normalizeS3Prefix(prefix: string | undefined): string { - const value = (prefix || 'openreader').trim(); - return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; -} - -function sanitizeNamespace(namespace: string | null): string | null { - if (!namespace) return null; - return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; -} - -function documentParsedKey(id: string, namespace: string | null, prefix: string): string { - if (!DOCUMENT_ID_REGEX.test(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(PDF_PARSER_VERSION)}.json`; -} - -function buildS3Client(): S3Client { - const bucket = requireEnv('S3_BUCKET'); - const region = requireEnv('S3_REGION'); - const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); - const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); - const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; - const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); - - void bucket; - - return new S3Client({ - region, - endpoint, - forcePathStyle, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - credentials: { - accessKeyId, - secretAccessKey, - }, - }); -} - -async function bodyToBuffer(body: unknown): Promise { - if (!body) return Buffer.alloc(0); - if (body instanceof Uint8Array) return Buffer.from(body); - if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); - if (body instanceof ArrayBuffer) return Buffer.from(body); - if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { - const maybe = body as { transformToByteArray?: () => Promise }; - if (typeof maybe.transformToByteArray === 'function') { - return Buffer.from(await maybe.transformToByteArray()); - } - } - if (typeof body === 'object' && body !== null && 'on' in body) { - const stream = body as NodeJS.ReadableStream; - const chunks: Buffer[] = []; - for await (const chunk of stream) { - if (Buffer.isBuffer(chunk)) chunks.push(chunk); - else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); - else chunks.push(Buffer.from(chunk as Uint8Array)); - } - return Buffer.concat(chunks); - } - throw new Error('Unsupported S3 response body type'); -} - -function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { - const copy = new Uint8Array(bytes.byteLength); - copy.set(bytes); - return copy.buffer; -} - -function isAuthed(request: FastifyRequest, expectedToken: string): boolean { - const auth = request.headers.authorization; - if (!auth?.startsWith('Bearer ')) return false; - const token = auth.slice('Bearer '.length).trim(); - return token === expectedToken; -} - -function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { - if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; - return Math.max(0, Math.floor((end as number) - (start as number))); -} - -function toErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - return String(error); -} - -function requestPath(request: FastifyRequest): string { - return request.url.split('?')[0] ?? request.url; -} - -function isHealthPath(path: string): boolean { - return path === '/health/live' || path === '/health/ready'; -} - -function extractTraceId(request: FastifyRequest): string | null { - const header = request.headers['x-openreader-trace-id']; - if (Array.isArray(header)) return header[0] ?? null; - return typeof header === 'string' ? header : null; -} - -function extractOpId(request: FastifyRequest, path: string): string | null { - const params = request.params as { opId?: unknown } | undefined; - if (params && typeof params.opId === 'string' && params.opId.trim()) { - return params.opId.trim(); - } - const match = path.match(/^\/ops\/([^/]+)/); - if (!match?.[1]) return null; - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; - } -} - -function isAlreadyExistsError(error: unknown): boolean { - const message = toErrorMessage(error).toLowerCase(); - return message.includes('already in use') || message.includes('already exists'); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -class ConcurrencyGate { - private readonly maxInFlight: number; - private inFlight = 0; - private readonly queue: Array<() => void> = []; - - constructor(limit: number) { - this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; - } - - async acquire(): Promise { - if (this.inFlight < this.maxInFlight) { - this.inFlight += 1; - return; - } - - await new Promise((resolve) => { - this.queue.push(() => { - this.inFlight += 1; - resolve(); - }); - }); - } - - release(): void { - this.inFlight = Math.max(0, this.inFlight - 1); - const next = this.queue.shift(); - if (next) next(); - } -} - -function isTerminalStatus(status: WorkerJobState): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { - if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; - const maybe = result as { parsedObjectKey?: unknown }; - return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; -} - -const alignSchema = z.object({ - text: z.string().trim().min(1), - lang: z.string().trim().min(1).max(16).optional(), - cacheKey: z.string().trim().min(1).max(256).optional(), - audioObjectKey: z.string().trim().min(1).max(2048), -}); - -const layoutSchema = z.object({ - documentId: z.string().trim().min(1), - namespace: z.string().trim().min(1).max(128).nullable(), - documentObjectKey: z.string().trim().min(1).max(2048), -}); - -const operationCreateSchema = z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('whisper_align'), - opKey: z.string().trim().min(1).max(1024), - payload: alignSchema, - }), - z.object({ - kind: z.literal('pdf_layout'), - opKey: z.string().trim().min(1).max(1024), - payload: layoutSchema, - }), -]); - -const operationLookupSchema = z.object({ - opKey: z.string().trim().min(1).max(1024), -}); - -async function ensureJetStreamResources( - jsm: JetStreamManager, - whisperTimeoutMs: number, - pdfTimeoutMs: number, - pdfAttempts: number, - jobsMaxBytes: number, - eventsMaxBytes: number, - natsReplicas: number, -): Promise { - const streamConfig = { - name: JOBS_STREAM_NAME, - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - retention: RetentionPolicy.Workqueue, - storage: StorageType.File, - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(streamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(JOBS_STREAM_NAME, { - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }); - } - - const eventsStreamConfig = { - name: EVENTS_STREAM_NAME, - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - retention: RetentionPolicy.Limits, - storage: StorageType.File, - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(eventsStreamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(EVENTS_STREAM_NAME, { - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }); - } - - const ensureConsumer = async ( - name: string, - subject: string, - ackWaitMs: number, - maxDeliver: number, - ): Promise => { - const config = { - durable_name: name, - ack_policy: AckPolicy.Explicit, - deliver_policy: DeliverPolicy.All, - replay_policy: ReplayPolicy.Instant, - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }; - - try { - await jsm.consumers.add(JOBS_STREAM_NAME, config); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.consumers.update(JOBS_STREAM_NAME, name, { - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }); - } - }; - - await Promise.all([ - ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), - ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), - ]); -} - -export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { - const port = options.port ?? readIntEnv('PORT', 8081); - const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); - const workerToken = options.workerToken ?? requireEnv('COMPUTE_WORKER_TOKEN'); - const disableWorkers = options.disableWorkers ?? false; - const natsUrl = requireEnv('NATS_URL'); - const timeoutConfig = getComputeTimeoutConfig(); - - const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); - const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; - const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; - const pdfHardCapMs = timeoutConfig.pdfHardCapMs; - const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); - const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', false); - const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); - const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); - const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); - const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); - const opStaleMs = getComputeOpStaleMs(); - - const connectOpts: Parameters[0] = { servers: natsUrl }; - const natsCreds = process.env.NATS_CREDS?.trim(); - const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); - - if (natsCreds) { - console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); - connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); - } else if (natsCredsFile) { - console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); - const { readFileSync } = await import('node:fs'); - const credsData = readFileSync(natsCredsFile); - connectOpts.authenticator = credsAuthenticator(credsData); - } - - // Lazy NATS connection lifecycle. The worker connects on demand (first request - // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle - // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound /ops request both wakes the container and re-establishes - // the session via ensureConnected(). - let session: NatsSession | null = null; - let connecting: Promise | null = null; - let workerLoops: Promise[] = []; - let idleTimer: NodeJS.Timeout | null = null; - let orphanSweepTimer: NodeJS.Timeout | null = null; - let stopping = false; - let loopStopRequested = false; - - // Activity accounting feeding the idle detector. The worker is considered idle - // only when no HTTP request is in flight, no SSE stream is open, no job is - // processing, and nothing has happened for IDLE_DISCONNECT_MS. - let inFlightHttp = 0; - let activeSse = 0; - let inFlightJobs = 0; - let lastActivityAt = Date.now(); - let lastActivityReason = 'startup'; - let lastIdleStatusLogAt = 0; - const jobGate = new ConcurrencyGate(jobConcurrency); - - const markActivity = (reason: string): void => { - lastActivityAt = Date.now(); - lastActivityReason = reason; - }; - - function startIdleTimer(): void { - if (idleTimer) return; - idleTimer = setInterval(() => { - if (!session || stopping) return; - const now = Date.now(); - const idleForMs = now - lastActivityAt; - if (now - lastIdleStatusLogAt >= IDLE_STATUS_LOG_INTERVAL_MS) { - lastIdleStatusLogAt = now; - app.log.info({ - activeSse, - idleForMs, - inFlightHttp, - inFlightJobs, - lastActivityReason, - disconnectEligible: inFlightHttp === 0 - && inFlightJobs === 0 - && idleForMs >= IDLE_DISCONNECT_MS, - }, 'nats idle status'); - } - // Hard work in flight always blocks idle. An open SSE no longer blocks just - // by existing — only by delivering events, which refresh lastActivityAt via - // markActivity() in the stream's onEvent. So a silent/stuck-op stream lets - // the idle window elapse and is torn down by disconnect() below. - if (inFlightHttp > 0 || inFlightJobs > 0) return; - if (idleForMs < IDLE_DISCONNECT_MS) return; - void disconnect('idle'); - }, IDLE_CHECK_INTERVAL_MS); - // Don't let the idle checker keep the process alive on its own. - idleTimer.unref?.(); - } - - function startOrphanSweepTimer(): void { - if (orphanSweepTimer) return; - orphanSweepTimer = setInterval(() => { - if (!session || stopping) return; - void runOrphanedOpRecovery({ force: true }).catch((error) => { - app.log.error({ - error: toErrorMessage(error), - }, 'periodic orphaned operation recovery failed'); - }); - }, ORPHAN_SWEEP_INTERVAL_MS); - orphanSweepTimer.unref?.(); - } - - async function disconnect(reason: string): Promise { - const current = session; - if (!current) return; - // Snapshot what's still attached so a dropped connection is visible in logs - // (e.g. SSE streams torn down because we went idle while they were open). - app.log.info({ - reason, - activeSse, - inFlightHttp, - inFlightJobs, - idleForMs: Date.now() - lastActivityAt, - }, 'nats dropping connection'); - // Clear synchronously (before any await) so concurrent requests reconnect a - // fresh session instead of using the connection we're about to close. - session = null; - loopStopRequested = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - if (orphanSweepTimer) { - clearInterval(orphanSweepTimer); - orphanSweepTimer = null; - } - try { - await current.nc.close(); - } catch { - // ignore close errors - } - await Promise.allSettled(workerLoops); - workerLoops = []; - loopStopRequested = false; - app.log.info({ reason }, 'nats disconnected'); - } - - async function ensureConnected(): Promise { - if (session) return session; - if (connecting) return connecting; - connecting = (async () => { - const nc: NatsConnection = await connect(connectOpts); - const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); - const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); - await ensureJetStreamResources( - jsm, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - jobsStreamMaxBytes, - eventsStreamMaxBytes, - natsReplicas, - ); - const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { - replicas: natsReplicas, - history: 1, - ttl: COMPUTE_STATE_TTL_MS, - max_bytes: jobStatesMaxBytes, - }); - const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); - const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); - const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; - session = next; - sessionGeneration += 1; - orphanRecoveryDoneForGeneration = -1; - markActivity('nats_connected'); - startWorkerLoops(next); - startIdleTimer(); - startOrphanSweepTimer(); - // Safety net: if the connection closes for any reason (network drop after - // exhausting reconnects, or our own disconnect), drop the stale session so - // the next request reconnects cleanly. - void nc.closed().then(() => { - if (session?.nc === nc) session = null; - }); - app.log.info('nats connected'); - return next; - })(); - try { - return await connecting; - } finally { - connecting = null; - } - } - - const s3 = disableWorkers ? null : buildS3Client(); - const s3Bucket = disableWorkers ? '' : requireEnv('S3_BUCKET'); - const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); - - const ensureSafeKey = (key: string): string => { - const trimmed = key.trim(); - if (!trimmed.startsWith(`${s3Prefix}/`)) { - throw new Error('Object key prefix mismatch'); - } - return trimmed; - }; - - const readObjectByKey = async (key: string): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - const safeKey = ensureSafeKey(key); - const response = await s3.send(new GetObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - const bytes = await bodyToBuffer(response.Body); - return toArrayBuffer(new Uint8Array(bytes)); - }; - - const objectExists = async (key: string): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - const safeKey = ensureSafeKey(key); - try { - await s3.send(new HeadObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - return true; - } catch (error) { - const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; - if ( - maybe.$metadata?.httpStatusCode === 404 - || maybe.name === 'NotFound' - || maybe.name === 'NoSuchKey' - || maybe.Code === 'NotFound' - || maybe.Code === 'NoSuchKey' - ) { - return false; - } - throw error; - } - }; - - const deleteObjectByKey = async (key: string): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - await s3.send(new DeleteObjectCommand({ - Bucket: s3Bucket, - Key: ensureSafeKey(key), - })); - }; - - const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - const key = documentParsedKey(documentId, namespace, s3Prefix); - const body = Buffer.from(JSON.stringify(parsed)); - await s3.send(new PutObjectCommand({ - Bucket: s3Bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - })); - return key; - }; - - if (prewarmModels && !disableWorkers) { - await ensureComputeModels(); - } - - const app = Fastify({ - logger: buildLoggerConfig(), - disableRequestLogging: true, - }); - app.log.info({ - jobConcurrency, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - opStaleMs, - availableCpuCores: getAvailableCpuCores(), - onnxThreadsPerJob: getOnnxThreadsPerJob(), - natsApiTimeoutMs: NATS_API_TIMEOUT_MS, - natsReplicas, - eventsStreamMaxBytes, - pdfLayoutHardCapMs: pdfHardCapMs, - }, 'compute runtime config'); - - const whisperJobCodec = createJsonCodec>(); - const layoutJobCodec = createJsonCodec>(); - - const defaultOperationStateStore = new JetStreamOperationStateStore({ - getKv: async () => (await ensureConnected()).kv, - }); - - const defaultOperationEventStream = new JetStreamOperationEventStream({ - getJs: async () => (await ensureConnected()).js, - getJsm: async () => (await ensureConnected()).jsm, - eventsStreamName: EVENTS_STREAM_NAME, - }); - - const operationQueue = new JetStreamOperationQueue({ - getJs: async () => (await ensureConnected()).js, - whisperSubject: WHISPER_JOBS_SUBJECT, - layoutSubject: LAYOUT_JOBS_SUBJECT, - }); - - const defaultOrchestrator = new OperationOrchestrator({ - queue: operationQueue, - stateStore: defaultOperationStateStore, - eventStream: defaultOperationEventStream, - config: { - opStaleMs, - maxCasRetries: 10, - }, - }); - - const operationStateStore = options.routeDeps?.operationStateStore ?? defaultOperationStateStore; - const operationEventStream = options.routeDeps?.operationEventStream ?? defaultOperationEventStream; - const orchestrator = options.routeDeps?.orchestrator ?? defaultOrchestrator; - let orphanRecoveryPromise: Promise | null = null; - let orphanRecoveryDoneForGeneration = -1; - let sessionGeneration = options.routeDeps ? 0 : -1; - - const runOrphanedOpRecovery = async (options?: { force?: boolean }): Promise => { - if (typeof operationStateStore.listOpStates !== 'function') return; - if (typeof operationStateStore.getOpStateRecord !== 'function') return; - if (typeof orchestrator.markFailedIfUnchanged !== 'function') return; - if (!options?.force && orphanRecoveryDoneForGeneration === sessionGeneration) return; - if (orphanRecoveryPromise) { - await orphanRecoveryPromise; - return; - } - - orphanRecoveryPromise = (async () => { - const recoveredStates = await recoverOrphanedOperations({ - operationStateStore: operationStateStore as typeof operationStateStore & { - getOpStateRecord(opId: string): Promise<{ state: StreamedOperationState; revision: number } | null>; - listOpStates(): Promise; - }, - orchestrator: orchestrator as typeof orchestrator & { - markFailedIfUnchanged(input: { - current: StreamedOperationState; - expectedRevision: number; - error: { message: string; code?: string } | string; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - }, - whisperTimeoutMs, - pdfTimeoutMs, - opStaleMs, - }); - - if (recoveredStates.length > 0) { - app.log.warn({ - recoveredCount: recoveredStates.length, - ops: recoveredStates.map((state) => ({ - opId: state.opId, - kind: state.kind, - status: state.status, - })), - }, 'recovered stale in-flight operations during reconciliation'); - } - - orphanRecoveryDoneForGeneration = sessionGeneration; - })().finally(() => { - orphanRecoveryPromise = null; - }); - - await orphanRecoveryPromise; - }; - - const ensureOrphanedOpRecovery = async (): Promise => { - await runOrphanedOpRecovery(); - }; - - const getOpState = async (opId: string): Promise => { - await ensureOrphanedOpRecovery(); - return await operationStateStore.getOpState(opId); - }; - - const releaseHttp = (request: FastifyRequest): void => { - const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; - if (!counted[REQUEST_COUNTED_KEY]) return; - counted[REQUEST_COUNTED_KEY] = false; - inFlightHttp = Math.max(0, inFlightHttp - 1); - markActivity('http_completed'); - }; - - app.addHook('onRequest', async (request, reply) => { - const path = requestPath(request); - (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); - // Count every request as in-flight activity so the idle detector never - // disconnects mid-request. Released in onResponse, or manually after hijack - // for SSE streams (where onResponse does not fire). - (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; - inFlightHttp += 1; - markActivity(`http_started:${path}`); - if (isHealthPath(path)) return; - if (!isAuthed(request, workerToken)) { - return reply.code(401).send({ error: 'Unauthorized' }); - } - return; - }); - - app.addHook('onResponse', async (request, reply) => { - releaseHttp(request); - const path = requestPath(request); - if (isHealthPath(path)) return; - if (reply.statusCode >= 500) { - const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; - const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; - app.log.error({ - reqId: request.id, - method: request.method, - path, - statusCode: reply.statusCode, - durationMs, - traceId: extractTraceId(request) ?? null, - opId: extractOpId(request, path), - }, 'http.error'); - } - }); - - app.get('/health/live', async () => ({ ok: true })); - - // Reports readiness without forcing a NATS round-trip. Probing NATS here would - // reconnect (and keep) the connection open, defeating idle sleep, so we only - // report the current connection state. The worker reconnects lazily on the next - // /ops request regardless of what this returns. - app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); - - app.post('/ops', async (request, reply) => { - const parsed = operationCreateSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - const requestOp = parsed.data as WorkerOperationRequest; - await ensureOrphanedOpRecovery(); - const op = await orchestrator.enqueueOrReuse(requestOp); - app.log.info({ - kind: requestOp.kind, - opId: op.opId, - jobId: op.jobId, - status: op.status, - opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), - }, 'op.accepted'); - reply.code(202); - return op; - }); - - app.post('/ops/lookup', async (request, reply) => { - const parsed = operationLookupSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - await ensureOrphanedOpRecovery(); - if (typeof operationStateStore.getOpIndex !== 'function') { - reply.code(501); - return { error: 'Operation lookup by opKey is not supported by this state store' }; - } - const indexEntry = await operationStateStore.getOpIndex(parsed.data.opKey); - if (!indexEntry?.opId) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const state = await operationStateStore.getOpState(indexEntry.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; - }); - - app.get('/ops/:opId', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const state = await getOpState(params.data.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; - }); - - app.get('/ops/:opId/events', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const initial = await getOpState(params.data.opId); - if (!initial) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; - const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); - const lastEventIdHeader = request.headers['last-event-id']; - const cursorFromHeader = Number( - Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), - ); - const sinceEventId = Math.max( - 0, - Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, - Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, - ); - - reply.hijack(); - // onResponse will not fire for a hijacked reply, so release the HTTP in-flight - // count here and track the long-lived stream via activeSse instead. - releaseHttp(request); - activeSse += 1; - markActivity('sse_started'); - reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); - reply.raw.setHeader('Connection', 'keep-alive'); - reply.raw.setHeader('X-Accel-Buffering', 'no'); - // Tell the browser EventSource to back off before reconnecting. If this stream - // is torn down because the worker went idle (NATS dropped), we don't want the - // client to reconnect immediately and re-wake the container. - reply.raw.write(encodeSseFrame({ retry: OP_EVENTS_RECONNECT_HINT_MS })); - - let closed = false; - let unsubscribe: (() => void) | null = null; - let keepalive: NodeJS.Timeout | null = null; - - const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { - if (closed || reply.raw.writableEnded) return; - const frameEvent: WorkerOperationEvent = { - eventId, - snapshot, - }; - reply.raw.write(encodeSseFrame({ - id: eventId, - event: 'snapshot', - data: frameEvent, - })); - }; - - const closeStream = (): void => { - if (closed) return; - closed = true; - if (unsubscribe) { - unsubscribe(); - unsubscribe = null; - } - if (keepalive) { - clearInterval(keepalive); - keepalive = null; - } - activeSse = Math.max(0, activeSse - 1); - markActivity('sse_closed'); - if (!reply.raw.writableEnded) { - reply.raw.end(); - } - }; - - request.raw.on('close', () => { - closeStream(); - }); - - try { - let current = initial; - let signature = JSON.stringify(current); - writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); - if (isTerminalStatus(current.status)) { - return reply; - } - - keepalive = setInterval(() => { - if (closed || reply.raw.writableEnded) return; - reply.raw.write(': keepalive\n\n'); - }, OP_EVENTS_KEEPALIVE_MS); - - unsubscribe = await operationEventStream.subscribe({ - opId: params.data.opId, - sinceEventId, - onEvent: (event) => { - if (closed) return; - if (event.snapshot.opId !== params.data.opId) return; - const nextSignature = JSON.stringify(event.snapshot); - if (nextSignature !== signature) { - current = event.snapshot; - signature = nextSignature; - // A real event is progress: refresh the idle window so an actively - // streaming op keeps the worker awake. A silent stream does not. - markActivity('sse_event'); - writeSnapshot(current, event.eventId); - } - if (isTerminalStatus(event.snapshot.status)) { - closeStream(); - } - }, - onError: (error) => { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - closeStream(); - }, - }); - - await new Promise((resolve) => { - request.raw.once('close', () => resolve()); - }); - } catch (error) { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - } finally { - closeStream(); - } - - return reply; - }); - - const runWhisper = async ( - payload: WhisperAlignJobRequest, - queueWaitMs: number, - ): Promise => { - const parsed = alignSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const audioBuffer = await withTimeout( - readObjectByKey(parsed.audioObjectKey), - whisperTimeoutMs, - 'whisper s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - const computeStartedAt = Date.now(); - const result = await withTimeout( - runWhisperAlignmentFromAudioBuffer({ - audioBuffer, - text: parsed.text, - cacheKey: parsed.cacheKey, - lang: parsed.lang, - }), - whisperTimeoutMs, - 'whisper alignment job', - ); - - const computeMs = Date.now() - computeStartedAt; - return { - ...result, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - const runLayout = async ( - payload: PdfLayoutJobRequest, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ): Promise => { - const parsed = layoutSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const pdfBytes = await withTimeout( - readObjectByKey(parsed.documentObjectKey), - Math.max(pdfTimeoutMs, 1_000), - 'pdf s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - let lastTotalPages = 0; - let lastPagesParsed = 0; - const computeStartedAt = Date.now(); - const result = await withIdleTimeoutAndHardCap({ - idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), - hardCapMs: pdfHardCapMs, - label: 'pdf layout job', - run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ - documentId: parsed.documentId, - pdfBytes, - onPageStarted: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageStart({ - pageNumber, - totalPages, - })); - }, - onPageParsed: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - lastPagesParsed = pageNumber; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageParsed({ - pageNumber, - totalPages, - })); - }, - }), - }); - - const computeMs = Date.now() - computeStartedAt; - if (hooks?.onProgress && lastTotalPages > 0) { - await hooks.onProgress({ - totalPages: lastTotalPages, - pagesParsed: lastPagesParsed, - currentPage: lastPagesParsed || undefined, - phase: 'merge', - }); - } - const parsedObjectKey = await persistParsedPdfWhileSourceExists({ - sourceObjectKey: parsed.documentObjectKey, - sourceExists: objectExists, - putParsedObject: () => putParsedObject(parsed.documentId, parsed.namespace, result.parsed), - deleteParsedObject: deleteObjectByKey, - }); - return { - parsedObjectKey, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - type ProcessMessageInput = { - msg: JsMsg; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }; - - type ProcessMessageContext = { - decoded: QueuedJob; - workerLabel: string; - startedAt: number; - queueWaitTiming?: { queueWaitMs: number }; - latestProgress?: PdfLayoutProgress; - }; - - function extractTiming(result: unknown): WorkerJobTiming | undefined { - if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; - return (result as { timing?: WorkerJobTiming }).timing; - } - - async function markRunning( - context: ProcessMessageContext, - updatedAt: number, - options?: { includeStartedAt?: boolean }, - ): Promise { - if (context.latestProgress) { - await orchestrator.markProgress({ - opId: context.decoded.opId, - progress: context.latestProgress, - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - return; - } - - await orchestrator.markRunning({ - opId: context.decoded.opId, - ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - } - - async function markProgress( - context: ProcessMessageContext, - progress: PdfLayoutProgress, - updatedAt: number, - ): Promise { - context.latestProgress = progress; - await markRunning(context, updatedAt); - } - - async function markTerminal(input: { - context: ProcessMessageContext; - status: 'succeeded' | 'failed'; - result?: TResult; - errorMessage?: string; - timing?: WorkerJobTiming; - updatedAt: number; - }): Promise { - if (input.status === 'succeeded') { - await orchestrator.markSucceeded({ - opId: input.context.decoded.opId, - result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - return; - } - - await orchestrator.markFailed({ - opId: input.context.decoded.opId, - error: { message: input.errorMessage ?? 'unknown worker failure' }, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - } - - async function handleRetry(input: { - context: ProcessMessageContext | null; - msg: JsMsg; - errorMessage: string; - }): Promise { - const deliveryCount = input.msg.info.deliveryCount; - const kind = input.context?.decoded.kind ?? 'pdf_layout'; - const retryAction = decideRetryAction({ - kind, - deliveryCount, - pdfAttempts, - whisperMaxDeliver: WHISPER_MAX_DELIVER, - }); - const hasRetriesLeft = retryAction === 'nak_retry'; - const maxAttempts = kind === 'whisper_align' ? WHISPER_MAX_DELIVER : pdfAttempts; - - if (input.context) { - const now = Date.now(); - const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); - const persistOpUpdate = hasRetriesLeft - ? (input.context.latestProgress - ? orchestrator.markProgress({ - opId: input.context.decoded.opId, - progress: input.context.latestProgress, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }) - : orchestrator.markRunning({ - opId: input.context.decoded.opId, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - })) - : markTerminal({ - context: input.context, - status: 'failed', - errorMessage: input.errorMessage, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }); - - await persistOpUpdate.catch((stateError) => { - app.log.error({ - worker: input.context?.workerLabel, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation state'); - }); - } - - if (hasRetriesLeft) { - input.msg.nak(); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'running', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retryAction: 'nack_retry', - }, 'job.terminal'); - return; - } - - input.msg.term(input.errorMessage); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'failed', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retrySuppressed: kind === 'whisper_align' ? 'whisper_align' : undefined, - retryAction: 'term', - }, 'job.terminal'); - } - - async function processMessage(input: ProcessMessageInput): Promise { - let context: ProcessMessageContext | null = null; - let heartbeat: NodeJS.Timeout | null = null; - try { - const decoded = input.codec.decode(input.msg.data); - const startedAt = Date.now(); - context = { - decoded, - workerLabel: input.workerLabel, - startedAt, - queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), - }; - - await markRunning(context, startedAt); - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, - deliveryCount: input.msg.info.deliveryCount, - }, 'job.started'); - - heartbeat = setInterval(() => { - const now = Date.now(); - void markRunning(context!, now).catch((stateError) => { - app.log.error({ - worker: input.workerLabel, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation heartbeat state'); - }); - }, RUNNING_HEARTBEAT_MS); - - const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { - onProgress: async (progress) => { - try { - input.msg.working(); - } catch (ackError) { - app.log.warn({ - worker: input.workerLabel, - kind: context?.decoded.kind, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(ackError), - }, 'failed to extend JetStream ack wait on progress'); - } - await markProgress(context!, progress, Date.now()); - }, - }); - const resultTiming = extractTiming(result); - const now = Date.now(); - - await markTerminal({ - context, - status: 'succeeded', - result, - timing: resultTiming, - updatedAt: now, - }); - - input.msg.ack(); - const terminalDurationMs = safeDurationMs(context.startedAt, now); - const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; - if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - durationMs: terminalDurationMs ?? null, - timing: resultTiming ?? null, - }, 'job.stage'); - } - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - status: 'succeeded', - durationMs: terminalDurationMs ?? null, - resultRef: extractResultRef(decoded.kind, result), - timing: resultTiming ?? null, - }, 'job.terminal'); - } catch (error) { - await handleRetry({ - context, - msg: input.msg, - errorMessage: toErrorMessage(error), - }); - } finally { - if (heartbeat) clearInterval(heartbeat); - } - } - - async function createWorkerLoop(input: { - owner: NatsSession; - consumer: Consumer; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }): Promise { - // Exit when the loop's connection is no longer the active session (idle - // disconnect, unexpected close, or replaced by a reconnect). - const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; - while (!detached()) { - let msg: JsMsg | null = null; - try { - try { - // Bounded pull so the loop yields periodically and exits promptly when - // the session is torn down for idle (nc.close() rejects the pending pull). - msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); - } catch (error) { - if (detached()) return; - app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); - await sleep(LOOP_ERROR_BACKOFF_MS); - continue; - } - - // An empty pull is not activity; let the idle window advance. - if (!msg) continue; - markActivity(`job_received:${input.workerLabel}`); - inFlightJobs += 1; - await jobGate.acquire(); - if (detached()) { - return; - } - await processMessage({ - msg, - codec: input.codec, - run: input.run, - workerLabel: input.workerLabel, - }); - } finally { - if (msg) { - jobGate.release(); - inFlightJobs = Math.max(0, inFlightJobs - 1); - markActivity(`job_completed:${input.workerLabel}`); - } - } - } - } - - function startWorkerLoops(active: NatsSession): void { - if (disableWorkers) return; - // Always starts a fresh set bound to the new session. Any loops from a prior - // session self-terminate once they observe session !== their owner. - workerLoops = []; - loopStopRequested = false; - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.whisperConsumer, - codec: whisperJobCodec, - run: runWhisper, - workerLabel: `whisper-${i + 1}`, - })); - } - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.layoutConsumer, - codec: layoutJobCodec, - run: runLayout, - workerLabel: `layout-${i + 1}`, - })); - } - } - - const close = async (): Promise => { - if (stopping) return; - stopping = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - if (orphanSweepTimer) { - clearInterval(orphanSweepTimer); - orphanSweepTimer = null; - } - await app.close(); - await Promise.allSettled(workerLoops); - const current = session; - session = null; - if (current) { - try { - await current.nc.drain(); - } catch { - try { - await current.nc.close(); - } catch { - // ignore close errors - } - } - } - }; - - const registerSignalHandlers = (): void => { - process.once('SIGINT', () => { - void close().finally(() => process.exit(0)); - }); - - process.once('SIGTERM', () => { - void close().finally(() => process.exit(0)); - }); - }; - - return { - app, - host, - port, - async start(startOptions) { - if (startOptions?.registerSignalHandlers) { - registerSignalHandlers(); - } - await app.listen({ host, port }); - app.log.info({ host, port }, 'compute worker listening'); - }, - close, - }; -} - -export async function startComputeWorkerFromEnv(): Promise { - const runtime = await createComputeWorkerApp(); - await runtime.start({ registerSignalHandlers: true }); -} diff --git a/docker/entrypoint-migration-tools/package.json b/docker/entrypoint-migration-tools/package.json deleted file mode 100644 index db7e2a3..0000000 --- a/docker/entrypoint-migration-tools/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@openreader/entrypoint-migration-tools", - "version": "0.0.0", - "private": true, - "dependencies": { - "@aws-sdk/client-s3": "^3.1061.0", - "better-sqlite3": "^12.10.0", - "dotenv": "^17.4.2", - "drizzle-kit": "^0.31.10", - "drizzle-orm": "^0.45.2", - "ffmpeg-static": "^5.3.0", - "pg": "^8.21.0" - } -} diff --git a/docker/examples/compose.full.yml b/docker/examples/compose.full.yml new file mode 100644 index 0000000..8776f64 --- /dev/null +++ b/docker/examples/compose.full.yml @@ -0,0 +1,124 @@ +name: openreader-full + +services: + openreader: + image: ghcr.io/richardr1126/openreader:latest + # Keep localhost:8333 valid for both app requests and browser-facing presigned URLs. + network_mode: service:seaweedfs + depends_on: + kokoro-tts: + condition: service_started + postgres: + condition: service_healthy + seaweedfs: + condition: service_healthy + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + POSTGRES_URL: postgres://openreader:openreader@postgres:5432/openreader + COMPUTE_WORKER_URL: http://compute-worker:8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + USE_EMBEDDED_WEED_MINI: "false" + S3_ENDPOINT: ${S3_ENDPOINT:-http://localhost:8333} # Used for internal endpoint and public facing presigned URLs. + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + + seaweedfs: + image: chrislusf/seaweedfs:4.18 + command: ["mini", "-dir=/data"] + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/cluster/status >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + ports: + - "3003:3003" + - "8333:8333" + volumes: + - seaweedfs-data:/data + + nats: + image: nats:2.14-alpine + command: ["-js", "-sd", "/data"] + volumes: + - nats-data:/data + + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: openreader + POSTGRES_PASSWORD: openreader + POSTGRES_DB: openreader + healthcheck: + test: ["CMD-SHELL", "pg_isready -U openreader -d openreader"] + interval: 2s + timeout: 2s + retries: 30 + volumes: + - postgres-data:/var/lib/postgresql/data + + compute-worker: + image: ghcr.io/richardr1126/openreader-compute-worker:latest + depends_on: + nats: + condition: service_started + seaweedfs: + condition: service_healthy + environment: + NATS_URL: nats://nats:4222 + COMPUTE_WORKER_HOST: 0.0.0.0 + PORT: 8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-false} + +volumes: + openreader-docstore: + seaweedfs-data: + nats-data: + postgres-data: diff --git a/docker/examples/compose.local-full.yml b/docker/examples/compose.local-full.yml new file mode 100644 index 0000000..b035247 --- /dev/null +++ b/docker/examples/compose.local-full.yml @@ -0,0 +1,130 @@ +name: openreader-local-full-build + +services: + openreader: + image: openreader:local + build: + context: ../.. + dockerfile: Dockerfile + # Keep localhost:8333 valid for both app requests and browser-facing presigned URLs. + network_mode: service:seaweedfs + depends_on: + kokoro-tts: + condition: service_started + postgres: + condition: service_healthy + seaweedfs: + condition: service_healthy + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + POSTGRES_URL: postgres://openreader:openreader@postgres:5432/openreader + COMPUTE_WORKER_URL: http://compute-worker:8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + USE_EMBEDDED_WEED_MINI: "false" + S3_ENDPOINT: ${S3_ENDPOINT:-http://localhost:8333} # Used for internal endpoint and public facing presigned URLs. + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + + seaweedfs: + image: chrislusf/seaweedfs:4.18 + command: ["mini", "-dir=/data"] + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/cluster/status >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + ports: + - "3003:3003" + - "8333:8333" + volumes: + - seaweedfs-data:/data + + nats: + image: nats:2.14-alpine + command: ["-js", "-sd", "/data"] + volumes: + - nats-data:/data + + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: openreader + POSTGRES_PASSWORD: openreader + POSTGRES_DB: openreader + healthcheck: + test: ["CMD-SHELL", "pg_isready -U openreader -d openreader"] + interval: 2s + timeout: 2s + retries: 30 + volumes: + - postgres-data:/var/lib/postgresql/data + + compute-worker: + image: openreader-compute-worker:local + build: + context: ../.. + dockerfile: packages/compute-worker/Dockerfile + depends_on: + nats: + condition: service_started + seaweedfs: + condition: service_healthy + environment: + NATS_URL: nats://nats:4222 + COMPUTE_WORKER_HOST: 0.0.0.0 + PORT: 8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-false} + +volumes: + openreader-docstore: + seaweedfs-data: + nats-data: + postgres-data: diff --git a/docker/examples/compose.local-slim.yml b/docker/examples/compose.local-slim.yml new file mode 100644 index 0000000..d4795aa --- /dev/null +++ b/docker/examples/compose.local-slim.yml @@ -0,0 +1,52 @@ +name: openreader-local-slim-build + +services: + openreader: + image: openreader:local + build: + context: ../.. + dockerfile: Dockerfile + depends_on: + kokoro-tts: + condition: service_started + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + ports: + - "3003:3003" + - "8333:8333" + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + +volumes: + openreader-docstore: diff --git a/docker/examples/compose.yml b/docker/examples/compose.yml new file mode 100644 index 0000000..e83ab7a --- /dev/null +++ b/docker/examples/compose.yml @@ -0,0 +1,49 @@ +name: openreader-slim + +services: + openreader: + image: ghcr.io/richardr1126/openreader:latest + depends_on: + kokoro-tts: + condition: service_started + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + ports: + - "3003:3003" + - "8333:8333" + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + +volumes: + openreader-docstore: diff --git a/docs-site/docs/configure/migrations.md b/docs-site/docs/configure/migrations.md index 43df512..71a51ed 100644 --- a/docs-site/docs/configure/migrations.md +++ b/docs-site/docs/configure/migrations.md @@ -7,6 +7,18 @@ import TabItem from '@theme/TabItem'; This page covers migration behavior for both database schema and storage data in OpenReader. +## Runtime ownership + +- `@openreader/database` owns database clients, schemas, SQL migration files, and programmatic + migration execution for SQLite and PostgreSQL. +- `@openreader/bootstrap` owns startup orchestration, storage migration, and optional embedded + SeaweedFS, NATS, and compute-worker processes. +- The Next.js app imports `@openreader/database` directly, but does not orchestrate migrations or + child processes. + +Docker deploys bootstrap as an isolated runtime bundle under `/opt/openreader/bootstrap`; it does +not merge migration dependencies into the standalone Next.js app under `/app`. + ## Startup migration behavior By default, the shared entrypoint runs migrations automatically before app startup in: @@ -53,11 +65,6 @@ In most cases, you do not need manual migration commands because startup runs mi - Postgres when `POSTGRES_URL` is set - SQLite when `POSTGRES_URL` is unset -You can always override the target explicitly with `--config`. - - - - ```bash # Run pending migrations for one target: # - Postgres if POSTGRES_URL is set @@ -71,26 +78,15 @@ pnpm migrate-fs pnpm migrate-fs:dry-run ``` - - - -```bash -# Migrate SQLite -pnpm exec drizzle-kit migrate --config drizzle.config.sqlite.ts - -# Migrate Postgres -pnpm exec drizzle-kit migrate --config drizzle.config.pg.ts -``` - - - +`pnpm migrate` uses the programmatic Drizzle migrator from `@openreader/database`. Drizzle Kit is +not a production or startup dependency; it is used only to generate new migration files. ## Generate migrations `pnpm generate` is a two-phase script for contributors and schema changes: 1. **Better Auth schema generation** — runs the Better Auth CLI twice (once for SQLite, once for Postgres) to produce auto-generated Drizzle schema files for auth tables (`user`, `session`, `account`, `verification`). -2. **Drizzle migration generation** — runs `drizzle-kit generate` for both `drizzle.config.sqlite.ts` and `drizzle.config.pg.ts`, producing SQL migration files from all schema files (app + auth). +2. **Drizzle migration generation** — runs `drizzle-kit generate` for both configs in `packages/database`, producing SQL migration files from all schema files (app + auth). :::note Most users do not need to run `pnpm generate`. Use it when contributing or when you have changed Drizzle schema files and need new migration files. @@ -100,22 +96,23 @@ Most users do not need to run `pnpm generate`. Use it when contributing or when Auth tables are owned by Better Auth. Their Drizzle schema definitions are auto-generated and should **not** be hand-edited: -- `src/db/schema_auth_sqlite.ts` -- `src/db/schema_auth_postgres.ts` +- `packages/database/src/schema_auth_sqlite.ts` +- `packages/database/src/schema_auth_postgres.ts` App-specific tables are manually maintained in the standard Drizzle schema files: -- `src/db/schema_sqlite.ts` -- `src/db/schema_postgres.ts` +- `packages/database/src/schema_sqlite.ts` +- `packages/database/src/schema_postgres.ts` -Both sets of schema files are included in the Drizzle configs, so `drizzle-kit generate` and `drizzle-kit migrate` handle all tables together. +Both sets of schema files are included in the Drizzle generation configs. Runtime migration +execution is owned by `@openreader/database`. When app schema changes (for example `tts_segments`), keep these in sync: -- `src/db/schema_sqlite.ts` -- `src/db/schema_postgres.ts` -- `drizzle/sqlite/*.sql` + `drizzle/sqlite/meta/_journal.json` -- `drizzle/postgres/*.sql` + `drizzle/postgres/meta/_journal.json` +- `packages/database/src/schema_sqlite.ts` +- `packages/database/src/schema_postgres.ts` +- `packages/database/migrations/sqlite/*.sql` + `packages/database/migrations/sqlite/meta/_journal.json` +- `packages/database/migrations/postgres/*.sql` + `packages/database/migrations/postgres/meta/_journal.json` @@ -130,10 +127,10 @@ pnpm generate ```bash # Generate SQLite migrations only (skips Better Auth CLI) -pnpm exec drizzle-kit generate --config drizzle.config.sqlite.ts +pnpm exec drizzle-kit generate --config packages/database/drizzle.config.sqlite.ts # Generate Postgres migrations only (skips Better Auth CLI) -pnpm exec drizzle-kit generate --config drizzle.config.pg.ts +pnpm exec drizzle-kit generate --config packages/database/drizzle.config.pg.ts ``` :::warning diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md index cd279db..604c7e1 100644 --- a/docs-site/docs/deploy/compute-worker.md +++ b/docs-site/docs/deploy/compute-worker.md @@ -11,7 +11,8 @@ Use this guide when OpenReader runs compute as a separate service. For the defau - Runs PDF layout parsing jobs - Stores durable job state in NATS JetStream and NATS KV -The app server submits work to `POST /ops` and listens for updates on `GET /ops/:opId/events`. +The app server submits resource-specific operations under `/v1` and listens for updates on +`GET /v1/operations/:opId/events`. ## When to use it @@ -37,7 +38,7 @@ S3_SECRET_ACCESS_KEY=... ``` :::important -`compute/worker/.env*` is only for standalone worker deployments. +`compute-worker/.env*` is only for standalone worker deployments. - Embedded/local mode: configure the root `.env` only. - External worker mode: set `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` on the app, and worker runtime values on the worker service. diff --git a/docs-site/docs/deploy/docker-compose.md b/docs-site/docs/deploy/docker-compose.md new file mode 100644 index 0000000..08964d7 --- /dev/null +++ b/docs-site/docs/deploy/docker-compose.md @@ -0,0 +1,175 @@ +--- +title: Docker Compose +description: Run OpenReader with the slim, full, local-slim, or local-full Docker Compose examples. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Use these examples to run OpenReader with Kokoro-FastAPI and persistent storage. Choose the slim +stack for the simplest deployment, or the full stack when you want PostgreSQL, SeaweedFS, NATS, +and the compute worker as separate containers. Local build variants are also available for both slim +and full stacks to build the application from your current checkout. + +## Prerequisites + +- A recent Docker version with Docker Compose +- A clone of the OpenReader repository + +```bash +git clone https://github.com/richardr1126/openreader.git +cd openreader +``` + +## Choose a stack + + + + +The default slim example runs: + +- OpenReader with embedded SeaweedFS, NATS, compute worker, and SQLite +- Kokoro-FastAPI as a companion container + +```bash +docker compose -f docker/examples/compose.yml up +# Repository convenience command: pnpm compose +``` + +Compose file: [`docker/examples/compose.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.yml) + + + + +The full example runs OpenReader, Kokoro-FastAPI, PostgreSQL, SeaweedFS, NATS, and the compute +worker as separate containers using published images. + +```bash +docker compose -f docker/examples/compose.full.yml up +# Repository convenience command: pnpm compose:full +``` + +Compose file: [`docker/examples/compose.full.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.full.yml) + +For details about running the worker separately, see +[Compute Worker](./compute-worker). + + + + +The local-slim example runs a slim setup (OpenReader and Kokoro-FastAPI), but builds the OpenReader app image from the current checkout. + +```bash +docker compose -f docker/examples/compose.local-slim.yml up --build +# Repository convenience command: pnpm compose:local +``` + +Compose file: [`docker/examples/compose.local-slim.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.local-slim.yml) + + + + +The local-full example uses the full multi-container layout, but builds the OpenReader app and compute-worker images from the current checkout. + +```bash +docker compose -f docker/examples/compose.local-full.yml up --build +# Repository convenience command: pnpm compose:local:full +``` + +Compose file: [`docker/examples/compose.local-full.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.local-full.yml) + + + + +## Included services + +| Service | Slim | Full | Local Slim | Local Full | +| --- | --- | --- | --- | --- | +| OpenReader | Published image | Published image | Local build | Local build | +| Kokoro-FastAPI | Container | Container | Container | Container | +| Database | Embedded SQLite | PostgreSQL container | Embedded SQLite | PostgreSQL container | +| SeaweedFS | Embedded | Container | Embedded | Container | +| NATS | Embedded | Container | Embedded | Container | +| Compute worker | Embedded | Published image | Embedded | Local build | + +On first boot, `RUNTIME_SEED_JSON` creates an enabled Kokoro shared provider and selects it as the +default TTS provider. + +## Endpoints + +- OpenReader: `http://localhost:3003` +- SeaweedFS S3: `http://localhost:8333` +- Kokoro-FastAPI: `http://localhost:8880` + +In the full examples, PostgreSQL, the compute worker, and NATS remain internal to the Compose +network. + +## LAN access + +Set `BASE_URL` and `S3_ENDPOINT` to the Docker host's LAN IP so browser-facing app and presigned +S3 URLs are reachable from other devices: + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.yml up +# Repository convenience command: pnpm compose +``` + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.full.yml up +# Repository convenience command: pnpm compose:full +``` + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.local-slim.yml up --build +# Repository convenience command: pnpm compose:local +``` + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.local-full.yml up --build +# Repository convenience command: pnpm compose:local:full +``` + + + + +Replace `192.168.0.XXX` with your Docker host's LAN IP and allow inbound TCP ports `3003` and +`8333` through its firewall. + +:::info Internal full-stack endpoint +The full and local-full compute workers continue using `http://seaweedfs:8333` internally. +`S3_ENDPOINT` configures the app endpoint and browser-facing presigned URLs. +::: + +## Configuration + +The examples use local-only default credentials. Override existing `${VARIABLE}` values through +your shell environment before using them beyond local development. + +:::warning Protect public deployments +Replace the default `AUTH_SECRET`, PostgreSQL credentials, S3 credentials, and compute-worker +token before exposing a stack outside your trusted local network. +::: + +For the complete configuration reference, see +[Environment Variables](../reference/environment-variables). See [Database](../configure/database) +for PostgreSQL and SQLite behavior. diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index b14a087..7d81811 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -159,45 +159,10 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i -
-External compute worker dev stack (optional) - -Use this only when you intentionally run compute-worker as a separate service. -Default local flow does not need `compute/worker/.env`; embedded worker startup reads root `.env`. -Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker). - -Start only NATS + compute-worker via compose watch: - -```bash -docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch -# or: pnpm compute:dev:watch -``` - -`compute/worker/.env.example` contains a starter config for standalone worker service deployments. - -Run the main app separately on the host: - -```bash -pnpm dev -``` - -For app -> external worker routing, set in root `.env`: - -```env -COMPUTE_WORKER_URL=http://localhost:8081 -COMPUTE_WORKER_TOKEN= -``` - -Ownership in external worker mode: -- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS` -- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) - -For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute/worker/.env*` is ignored in that mode. - -Worker mode requires worker-reachable shared object storage (S3-compatible endpoint). -For external worker mode, object storage must be shared/reachable by both app and worker services. - -
+:::tip Docker Compose +To run OpenReader and Kokoro-FastAPI with Docker Compose, including slim, full, and local-build +options, see [Docker Compose](./docker-compose). +::: ## Steps @@ -240,7 +205,7 @@ COMPUTE_WORKER_TOKEN= Use the same ownership split: - root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale overrides -- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) +- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) Use one of these `.env` mode templates: diff --git a/docs-site/docs/deploy/vercel-deployment.md b/docs-site/docs/deploy/vercel-deployment.md index fc634c8..298851c 100644 --- a/docs-site/docs/deploy/vercel-deployment.md +++ b/docs-site/docs/deploy/vercel-deployment.md @@ -106,7 +106,7 @@ For all variables and defaults, see [Environment Variables](../reference/environ ## 4. Database and data migrations -Vercel deployments do not run `scripts/openreader-entrypoint.mjs`, so automatic startup migrations do not run there. +Vercel deployments do not run the `@openreader/bootstrap` process, so automatic startup migrations do not run there. - Run `pnpm migrate` in a controlled environment to apply Drizzle schema migrations to your Postgres DB. - Run `pnpm migrate-fs` only when migrating legacy local filesystem data (`docstore/documents_v1`, `docstore/audiobooks_v1`) into object storage + DB rows. Fresh Vercel deployments usually do not need this. diff --git a/docs-site/docs/reference/stack.md b/docs-site/docs/reference/stack.md index ea847e6..f260e31 100644 --- a/docs-site/docs/reference/stack.md +++ b/docs-site/docs/reference/stack.md @@ -45,21 +45,20 @@ title: Stack ## External compute worker (optional) -Monorepo packages under `compute/`: +Standalone worker package: -- **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts +- **`@openreader/compute-worker`** — standalone Node.js compute service containing its private inference and queue runtime - ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers` - Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps - PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing - PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization - Utilities: `jszip`, `ffmpeg-static` -- **`@openreader/compute-worker`** — standalone Node.js worker service - - HTTP server: [Fastify](https://fastify.dev/) v5 +- HTTP server: [Fastify](https://fastify.dev/) v5 with a versioned OpenAPI contract - Job queue + state: [NATS](https://nats.io/) JetStream WorkQueue pull consumers + NATS KV (`jobs.whisper`, `jobs.layout`) - Storage: AWS SDK v3 S3 client for reading/writing blobs - Logging: [Pino](https://getpino.io/) - Validation: [Zod](https://zod.dev/) -- Heavy compute is worker-backed via `COMPUTE_WORKER_URL` + `COMPUTE_WORKER_TOKEN` (remote queue via HTTP + NATS) +- The Next.js app communicates with the worker only through the versioned HTTP API generated from OpenAPI. ## Tooling and testing diff --git a/docs-site/sidebars.ts b/docs-site/sidebars.ts index 63041f9..45597dc 100644 --- a/docs-site/sidebars.ts +++ b/docs-site/sidebars.ts @@ -55,7 +55,12 @@ const sidebars: SidebarsConfig = { { type: 'category', label: '🚀 Deploy', - items: ['deploy/local-development', 'deploy/compute-worker', 'deploy/vercel-deployment'], + items: [ + 'deploy/local-development', + 'deploy/docker-compose', + 'deploy/compute-worker', + 'deploy/vercel-deployment', + ], }, { type: 'category', diff --git a/drizzle/scripts/migrate.mjs b/drizzle/scripts/migrate.mjs deleted file mode 100644 index d7c9e8e..0000000 --- a/drizzle/scripts/migrate.mjs +++ /dev/null @@ -1,56 +0,0 @@ -import { spawnSync } from 'node:child_process'; -import path from 'node:path'; -import fs from 'node:fs'; -import * as dotenv from 'dotenv'; - -function loadEnvFiles() { - // Approximate Next.js behavior enough for server-side scripts. - // Load .env first, then .env.local (local overrides). - const cwd = process.cwd(); - const envPath = path.join(cwd, '.env'); - const envLocalPath = path.join(cwd, '.env.local'); - - if (fs.existsSync(envPath)) { - dotenv.config({ path: envPath }); - } - if (fs.existsSync(envLocalPath)) { - dotenv.config({ path: envLocalPath, override: true }); - } -} - -loadEnvFiles(); - -const extraArgs = process.argv.slice(2); - -const hasConfigArg = extraArgs.includes('--config'); -const configFile = process.env.POSTGRES_URL ? 'drizzle.config.pg.ts' : 'drizzle.config.sqlite.ts'; -const configArgs = hasConfigArg ? [] : ['--config', configFile]; - -// Ensure the docstore directory exists for SQLite migrations. -// drizzle-kit opens the database file directly and will fail if the parent -// directory is missing (e.g. in a fresh CI checkout). -if (!process.env.POSTGRES_URL) { - const dbDir = path.join(process.cwd(), 'docstore'); - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }); - } -} - -function resolveDrizzleKitBin() { - const binName = process.platform === 'win32' ? 'drizzle-kit.cmd' : 'drizzle-kit'; - const localBin = path.join(process.cwd(), 'node_modules', '.bin', binName); - if (fs.existsSync(localBin)) return localBin; - return 'drizzle-kit'; -} - -const result = spawnSync(resolveDrizzleKitBin(), ['migrate', ...configArgs, ...extraArgs], { - stdio: 'inherit', - env: process.env, -}); - -if (result.error) { - console.error(result.error.message); - process.exit(1); -} - -process.exit(result.status ?? 1); diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a0310d..e7af21a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -34,16 +34,14 @@ const UI_ARCHITECTURE_FILES = [ "src/components/player/**/*.{ts,tsx}", "src/components/reader/**/*.{ts,tsx}", ]; -const COMPUTE_CORE_IMPORT_PATTERNS = [ +const COMPUTE_WORKER_IMPORT_PATTERNS = [ { group: [ - "@openreader/compute-core/*", - "!@openreader/compute-core/local-runtime", - "!@openreader/compute-core/types", - "!@openreader/compute-core/api-contracts", + "@openreader/compute-worker", + "@openreader/compute-worker/*", ], message: - "Use '@openreader/compute-core' root imports for light APIs. Allowed subpaths are '@openreader/compute-core/local-runtime', '@openreader/compute-core/types', and '@openreader/compute-core/api-contracts'.", + "The app must communicate with compute-worker through the app-owned HTTP client and generated protocol types.", }, ]; const UI_ARCHITECTURE_IMPORT_PATHS = [ @@ -124,7 +122,7 @@ const eslintConfig = [ "no-restricted-imports": [ "error", { - patterns: COMPUTE_CORE_IMPORT_PATTERNS, + patterns: COMPUTE_WORKER_IMPORT_PATTERNS, }, ], }, @@ -142,7 +140,7 @@ const eslintConfig = [ "error", { paths: UI_ARCHITECTURE_IMPORT_PATHS, - patterns: COMPUTE_CORE_IMPORT_PATTERNS, + patterns: COMPUTE_WORKER_IMPORT_PATTERNS, }, ], }, diff --git a/next.config.ts b/next.config.ts index 1cf8c6b..4417b46 100644 --- a/next.config.ts +++ b/next.config.ts @@ -15,7 +15,6 @@ const securityHeaders = [ }, ]; -const bundleWorkerCompute = true; const pdfjsTraceFiles = [ './node_modules/pdfjs-dist/package.json', './node_modules/pdfjs-dist/legacy/build/pdf.mjs', @@ -29,7 +28,6 @@ const serverExternalPackages = [ // Keep pdfjs-dist as a real package in node_modules. Server-side preview // rendering resolves pdf.js runtime assets from the filesystem at runtime. 'pdfjs-dist', - ...(!bundleWorkerCompute ? ['onnxruntime-node', '@huggingface/tokenizers'] : []), ]; const nextConfig: NextConfig = { @@ -48,7 +46,7 @@ const nextConfig: NextConfig = { canvas: '@napi-rs/canvas', }, }, - transpilePackages: [], + transpilePackages: ['@openreader/database'], serverExternalPackages, outputFileTracingIncludes: { '/api/audiobook': [ @@ -88,14 +86,6 @@ const nextConfig: NextConfig = { }, ]; } - if (isServer && bundleWorkerCompute) { - config.resolve.alias = { - ...(config.resolve.alias || {}), - '@openreader/compute-core/local-runtime$': false, - 'onnxruntime-node$': false, - '@huggingface/tokenizers$': false, - }; - } return config; }, }; diff --git a/package.json b/package.json index de098e0..55ec014 100644 --- a/package.json +++ b/package.json @@ -4,54 +4,59 @@ "private": true, "packageManager": "pnpm@10.33.4", "scripts": { - "dev": "node scripts/openreader-entrypoint.mjs -- pnpm dev:raw", + "dev": "pnpm --filter @openreader/bootstrap start pnpm dev:raw", "dev:raw": "next dev --turbopack -p 3003", "build": "next build", "build:bundle-guard": "node scripts/check-next-server-bundle.mjs", - "start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw", + "check:compute-boundary": "node scripts/check-compute-boundary.mjs", + "start": "pnpm --filter @openreader/bootstrap start pnpm start:raw", "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", "test:e2e": "playwright test", "test:unit": "vitest run", - "test:compute": "vitest run --project compute-core --project compute-worker", - "migrate": "node drizzle/scripts/migrate.mjs", - "migrate-fs": "node scripts/migrate-fs-v2.mjs", - "migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true", - "generate": "node drizzle/scripts/generate.mjs", + "test:compute": "vitest run --project compute-worker", + "compose": "docker compose -f docker/examples/compose.yml up", + "compose:full": "docker compose -f docker/examples/compose.full.yml up", + "compose:local": "docker compose -f docker/examples/compose.local-slim.yml up --build", + "compose:local:slim": "docker compose -f docker/examples/compose.local-slim.yml up --build", + "compose:local:full": "docker compose -f docker/examples/compose.local-full.yml up --build", + "migrate": "pnpm --filter @openreader/database migrate", + "migrate-fs": "pnpm --filter @openreader/bootstrap migrate-storage", + "migrate-fs:dry-run": "pnpm --filter @openreader/bootstrap migrate-storage --dry-run true", + "generate": "node packages/database/scripts/generate.mjs", "docs:init": "pnpm --dir docs-site install", "docs:dev": "pnpm --dir docs-site start", "docs:build": "pnpm --dir docs-site build", "docs:serve": "pnpm --dir docs-site serve", "docs:version": "pnpm --dir docs-site docusaurus docs:version", "docs:clear": "pnpm --dir docs-site clear", - "compute:worker:dev": "pnpm --filter @openreader/compute-worker dev", - "compute:worker:start": "pnpm --filter @openreader/compute-worker start", - "compute:dev:compose": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --build", + "compute:dev": "pnpm --filter @openreader/compute-worker dev", + "compute:start": "pnpm --filter @openreader/compute-worker start", + "compute:openapi:generate": "pnpm --dir packages/compute-worker openapi:generate && openapi-typescript packages/compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", + "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- packages/compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", "lint:route-errors": "node scripts/check-route-error-responses.mjs" }, "dependencies": { "@aws-sdk/client-s3": "^3.1061.0", "@aws-sdk/s3-request-presigner": "^3.1061.0", "@headlessui/react": "^2.2.10", - "@huggingface/tokenizers": "^0.1.3", "@microsoft/antissrf": "^1.0.0", "@mozilla/readability": "^0.6.0", "@napi-rs/canvas": "^0.1.100", + "@openreader/database": "workspace:*", "@speech-sdk/core": "^0.15.0", "@tanstack/react-query": "^5.101.0", "@types/archiver": "^7.0.0", "@vercel/analytics": "^1.6.1", "archiver": "^7.0.1", "better-auth": "^1.6.14", - "better-sqlite3": "^12.10.0", "cmpstr": "^3.3.0", "compromise": "^14.15.1", "core-js": "^3.49.0", "dexie": "^4.4.3", "dexie-react-hooks": "^4.4.0", "dotenv": "^17.4.2", - "drizzle-kit": "^0.31.10", "drizzle-orm": "^0.45.2", "epubjs": "^0.3.93", "fast-xml-parser": "^5.8.0", @@ -61,10 +66,8 @@ "linkedom": "^0.18.12", "lru-cache": "^11.5.1", "next": "^15.5.19", - "onnxruntime-node": "^1.26.0", "openai": "^6.41.0", "pdfjs-dist": "4.8.69", - "pg": "^8.21.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "react": "^19.2.7", @@ -95,9 +98,12 @@ "@types/react-dom": "^19.2.3", "@types/turndown": "^5.0.6", "@types/uuid": "^10.0.0", + "better-sqlite3": "^12.10.0", + "drizzle-kit": "^0.31.10", "eslint": "^9.39.4", "eslint-config-next": "^15.5.19", "postcss": "^8.5.15", + "openapi-typescript": "^7.10.1", "tailwindcss": "^3.4.19", "typescript": "^5.9.3", "vitest": "^4.1.8" diff --git a/packages/bootstrap/package.json b/packages/bootstrap/package.json new file mode 100644 index 0000000..0f7a8fa --- /dev/null +++ b/packages/bootstrap/package.json @@ -0,0 +1,25 @@ +{ + "name": "@openreader/bootstrap", + "version": "0.0.0", + "private": true, + "type": "module", + "files": [ + "src" + ], + "bin": { + "openreader": "./src/cli.mjs", + "openreader-migrate-storage": "./src/storage-migration.mjs" + }, + "scripts": { + "start": "node src/cli.mjs --", + "migrate-storage": "node src/storage-migration.mjs" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.1061.0", + "@openreader/database": "workspace:*", + "better-sqlite3": "^12.10.0", + "dotenv": "^17.4.2", + "ffmpeg-static": "^5.3.0", + "pg": "^8.21.0" + } +} diff --git a/scripts/openreader-entrypoint.mjs b/packages/bootstrap/src/cli.mjs similarity index 76% rename from scripts/openreader-entrypoint.mjs rename to packages/bootstrap/src/cli.mjs index 44588e1..21e5f84 100644 --- a/scripts/openreader-entrypoint.mjs +++ b/packages/bootstrap/src/cli.mjs @@ -3,16 +3,44 @@ import { spawn, spawnSync } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { once } from 'node:events'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import process from 'node:process'; import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; import * as dotenv from 'dotenv'; +import { runMigrations } from '@openreader/database/migrate'; +import { hasNatsBinary } from './embedded-nats.mjs'; +import { + detectHostForDefaultEndpoint, + hasWeedBinary, + isRunningInDocker, + loopbackS3Endpoint, + parseS3Endpoint, + parseUrlHost, + waitForEndpoint, +} from './embedded-seaweedfs.mjs'; +import { resolveEmbeddedWorkerLaunch } from './embedded-worker.mjs'; + +function findWorkspaceRoot(startDir = process.cwd()) { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + +const workspaceRoot = findWorkspaceRoot(process.cwd()); function loadEnvFiles() { - const cwd = process.cwd(); - const envPath = path.join(cwd, '.env'); - const envLocalPath = path.join(cwd, '.env.local'); + const envPath = path.join(workspaceRoot, '.env'); + const envLocalPath = path.join(workspaceRoot, '.env.local'); if (fs.existsSync(envPath)) { dotenv.config({ path: envPath }); @@ -49,83 +77,6 @@ function requireAuthEnv(env) { } } -function isPrivateIPv4(address) { - if (!address) return false; - if (address.startsWith('10.')) return true; - if (address.startsWith('192.168.')) return true; - const m = /^172\.(\d+)\./.exec(address); - if (m) { - const second = Number.parseInt(m[1], 10); - if (second >= 16 && second <= 31) return true; - } - return false; -} - -function detectHostForDefaultEndpoint() { - const interfaces = os.networkInterfaces(); - const ipv4 = []; - - for (const entries of Object.values(interfaces)) { - for (const entry of entries || []) { - if (!entry) continue; - const family = typeof entry.family === 'string' ? entry.family : String(entry.family); - if (family !== 'IPv4') continue; - if (entry.internal) continue; - ipv4.push(entry.address); - } - } - - const privateAddr = ipv4.find(isPrivateIPv4); - if (privateAddr) return privateAddr; - if (ipv4[0]) return ipv4[0]; - return '127.0.0.1'; -} - -function parseS3Endpoint(endpoint) { - let url; - try { - url = new URL(endpoint); - } catch { - throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`); - } - - if (!url.hostname) { - throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`); - } - - const port = Number.parseInt(url.port || '8333', 10); - if (!Number.isFinite(port) || port < 1 || port > 65535) { - throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`); - } - - return { - hostname: url.hostname, - port, - normalized: `${url.protocol}//${url.hostname}:${port}`, - }; -} - -function loopbackS3Endpoint(endpoint) { - const parsed = parseS3Endpoint(endpoint); - const url = new URL(parsed.normalized); - return `${url.protocol}//127.0.0.1:${parsed.port}`; -} - -function parseUrlHost(urlValue, fieldName) { - let url; - try { - url = new URL(urlValue); - } catch { - throw new Error(`Invalid ${fieldName}: ${urlValue}`); - } - - if (!url.hostname) { - throw new Error(`Invalid ${fieldName} host: ${urlValue}`); - } - - return url.hostname; -} - function parseCommandFromArgs(argv) { const marker = argv.indexOf('--'); if (marker >= 0) return argv.slice(marker + 1); @@ -143,54 +94,10 @@ function forwardChildStream(stream, target) { }; } -function hasWeedBinary() { - const probe = spawnSync('weed', ['version'], { stdio: 'ignore' }); - if (probe.error) return false; - return true; -} - -function hasNatsBinary() { - const probe = spawnSync('nats-server', ['-v'], { stdio: 'ignore' }); - if (probe.error) return false; - return true; -} - -async function waitForEndpoint(url, timeoutSeconds) { - const waitMs = Math.max(1, timeoutSeconds) * 1000; - const deadline = Date.now() + waitMs; - - while (Date.now() < deadline) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 2000); - - try { - const res = await fetch(url, { method: 'GET', signal: controller.signal }); - if (res) return; - } catch { - // retry - } finally { - clearTimeout(timeoutId); - } - await delay(1000); - } - - throw new Error(`Embedded weed mini did not become ready at ${url} within ${timeoutSeconds}s.`); -} - -function isRunningInDocker() { - if (process.platform !== 'linux') return false; - if (fs.existsSync('/.dockerenv')) return true; - try { - const cgroup = fs.readFileSync('/proc/1/cgroup', 'utf8'); - return /(docker|containerd|kubepods|podman)/i.test(cgroup); - } catch { - return false; - } -} - function spawnMainCommand(command, env) { const [cmd, ...args] = command; const child = spawn(cmd, args, { + cwd: workspaceRoot, env, stdio: 'inherit', shell: process.platform === 'win32', @@ -219,53 +126,13 @@ function spawnMainCommand(command, env) { return { child, exitPromise }; } -function resolveEmbeddedWorkerLaunch() { - const candidateDirs = [ - path.join(process.cwd(), 'embedded-compute-worker'), - path.join(process.cwd(), 'compute', 'worker'), - ]; - - for (const workerDir of candidateDirs) { - const serverEntry = path.join(workerDir, 'src', 'server.ts'); - if (!fs.existsSync(serverEntry)) continue; - return { - cmd: process.execPath, - args: ['--import', 'tsx', 'src/server.ts'], - cwd: workerDir, - }; - } - - throw new Error( - 'Could not find an embedded compute worker runtime. ' - + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute/worker available locally.', - ); -} - -function runDbMigrations(env) { - const migrateScript = path.join(process.cwd(), 'drizzle', 'scripts', 'migrate.mjs'); - if (!fs.existsSync(migrateScript)) { - throw new Error(`Could not find migration script at ${migrateScript}`); - } - +async function runDbMigrations(env) { console.log('Running database migrations...'); - const migration = spawnSync(process.execPath, [migrateScript], { - env, - stdio: 'inherit', - }); - - if (migration.error) { - throw migration.error; - } - if (typeof migration.status === 'number' && migration.status !== 0) { - throw new Error(`Database migrations failed with exit code ${migration.status}.`); - } + await runMigrations({ cwd: workspaceRoot, env }); } function runStorageMigrations(env) { - const migrateScript = path.join(process.cwd(), 'scripts', 'migrate-fs-v2.mjs'); - if (!fs.existsSync(migrateScript)) { - throw new Error(`Could not find storage migration script at ${migrateScript}`); - } + const migrateScript = fileURLToPath(new URL('./storage-migration.mjs', import.meta.url)); console.log('Running storage migrations (v2)...'); const migration = spawnSync(process.execPath, [migrateScript, '--dry-run', 'false', '--delete-local', 'false'], { @@ -335,7 +202,7 @@ async function main() { const command = parseCommandFromArgs(process.argv.slice(2)); if (command.length === 0) { - console.error('Usage: node scripts/openreader-entrypoint.mjs -- [args]'); + console.error('Usage: openreader -- [args]'); process.exit(2); } @@ -424,11 +291,11 @@ async function main() { try { const shouldRunDbMigrations = resolveBooleanEnv(runtimeEnv, 'RUN_DRIZZLE_MIGRATIONS', true); if (shouldRunDbMigrations) { - runDbMigrations(runtimeEnv); + await runDbMigrations(runtimeEnv); } if (useEmbeddedWeed) { - runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, 'docstore/seaweedfs'); + runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, path.join(workspaceRoot, 'docstore/seaweedfs')); runtimeEnv.WEED_MINI_WAIT_SEC = withDefault(runtimeEnv.WEED_MINI_WAIT_SEC, '20'); runtimeEnv.S3_BUCKET = withDefault(runtimeEnv.S3_BUCKET, 'openreader-documents'); runtimeEnv.S3_REGION = withDefault(runtimeEnv.S3_REGION, 'us-east-1'); @@ -482,7 +349,7 @@ async function main() { console.log('Starting embedded SeaweedFS weed mini...'); launchWeed(runtimeEnv.S3_ENDPOINT); const startupEndpoint = parseS3Endpoint(runtimeEnv.S3_ENDPOINT); - await waitForEndpoint(`http://127.0.0.1:${startupEndpoint.port}`, waitTimeout); + await waitForEndpoint(`http://127.0.0.1:${startupEndpoint.port}`, waitTimeout, 'Embedded SeaweedFS'); console.log(`Embedded SeaweedFS is ready at ${runtimeEnv.S3_ENDPOINT}`); } @@ -521,7 +388,7 @@ async function main() { runtimeEnv.COMPUTE_WORKER_HOST = withDefault(runtimeEnv.COMPUTE_WORKER_HOST, '127.0.0.1'); runtimeEnv.COMPUTE_NATS_REPLICAS = withDefault(runtimeEnv.COMPUTE_NATS_REPLICAS, '1'); - const natsStoreDir = withDefault(runtimeEnv.EMBEDDED_NATS_STORE_DIR, 'docstore/nats/jetstream'); + const natsStoreDir = withDefault(runtimeEnv.EMBEDDED_NATS_STORE_DIR, path.join(workspaceRoot, 'docstore/nats/jetstream')); fs.mkdirSync(natsStoreDir, { recursive: true }); console.log(`Starting embedded nats-server on 127.0.0.1:${embeddedNatsPort}...`); @@ -555,7 +422,7 @@ async function main() { console.error(`Embedded nats-server failed to start: ${error instanceof Error ? error.message : String(error)}`); scheduleFatalShutdown('nats-server', null, null, 'failed to start'); }); - await waitForEndpoint(`http://127.0.0.1:${embeddedNatsMonitorPort}/healthz`, 20); + await waitForEndpoint(`http://127.0.0.1:${embeddedNatsMonitorPort}/healthz`, 20, 'Embedded nats-server'); console.log(`Embedded nats-server is ready at nats://127.0.0.1:${embeddedNatsPort}`); console.log(`Starting embedded compute-worker on 127.0.0.1:${embeddedWorkerPort}...`); @@ -589,7 +456,7 @@ async function main() { console.error(`Embedded compute-worker failed to start: ${error instanceof Error ? error.message : String(error)}`); scheduleFatalShutdown('compute-worker', null, null, 'failed to start'); }); - await waitForEndpoint(`http://127.0.0.1:${embeddedWorkerPort}/health/ready`, 30); + await waitForEndpoint(`http://127.0.0.1:${embeddedWorkerPort}/health/ready`, 30, 'Embedded compute-worker'); console.log(`Embedded compute-worker is ready at http://127.0.0.1:${embeddedWorkerPort}`); } else if (!runtimeEnv.COMPUTE_WORKER_URL?.trim() || !runtimeEnv.COMPUTE_WORKER_TOKEN?.trim()) { throw new Error('COMPUTE_WORKER_URL and COMPUTE_WORKER_TOKEN are required when embedded compute worker startup is disabled.'); diff --git a/packages/bootstrap/src/embedded-nats.mjs b/packages/bootstrap/src/embedded-nats.mjs new file mode 100644 index 0000000..d9d1238 --- /dev/null +++ b/packages/bootstrap/src/embedded-nats.mjs @@ -0,0 +1,5 @@ +import { spawnSync } from 'node:child_process'; + +export function hasNatsBinary() { + return !spawnSync('nats-server', ['-v'], { stdio: 'ignore' }).error; +} diff --git a/packages/bootstrap/src/embedded-seaweedfs.mjs b/packages/bootstrap/src/embedded-seaweedfs.mjs new file mode 100644 index 0000000..4fc40d8 --- /dev/null +++ b/packages/bootstrap/src/embedded-seaweedfs.mjs @@ -0,0 +1,95 @@ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import process from 'node:process'; +import { setTimeout as delay } from 'node:timers/promises'; + +function isPrivateIPv4(address) { + if (!address) return false; + if (address.startsWith('10.')) return true; + if (address.startsWith('192.168.')) return true; + const match = /^172\.(\d+)\./.exec(address); + if (!match) return false; + const second = Number.parseInt(match[1], 10); + return second >= 16 && second <= 31; +} + +export function hasWeedBinary() { + return !spawnSync('weed', ['version'], { stdio: 'ignore' }).error; +} + +export function detectHostForDefaultEndpoint() { + const ipv4 = Object.values(os.networkInterfaces()) + .flatMap((entries) => entries || []) + .filter((entry) => entry && !entry.internal && String(entry.family) === 'IPv4') + .map((entry) => entry.address); + + return ipv4.find(isPrivateIPv4) || ipv4[0] || '127.0.0.1'; +} + +export function parseS3Endpoint(endpoint) { + let url; + try { + url = new URL(endpoint); + } catch { + throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`); + } + + if (!url.hostname) throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`); + const port = Number.parseInt(url.port || '8333', 10); + if (!Number.isFinite(port) || port < 1 || port > 65535) { + throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`); + } + + return { + hostname: url.hostname, + port, + normalized: `${url.protocol}//${url.hostname}:${port}`, + }; +} + +export function loopbackS3Endpoint(endpoint) { + const parsed = parseS3Endpoint(endpoint); + const url = new URL(parsed.normalized); + return `${url.protocol}//127.0.0.1:${parsed.port}`; +} + +export function parseUrlHost(urlValue, fieldName) { + let url; + try { + url = new URL(urlValue); + } catch { + throw new Error(`Invalid ${fieldName}: ${urlValue}`); + } + + if (!url.hostname) throw new Error(`Invalid ${fieldName} host: ${urlValue}`); + return url.hostname; +} + +export async function waitForEndpoint(url, timeoutSeconds, serviceName = 'service') { + const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000; + while (Date.now() < deadline) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 2000); + try { + const response = await fetch(url, { method: 'GET', signal: controller.signal }); + if (response) return; + } catch { + // Retry until the startup deadline. + } finally { + clearTimeout(timeoutId); + } + await delay(1000); + } + throw new Error(`${serviceName} did not become ready at ${url} within ${timeoutSeconds}s.`); +} + +export function isRunningInDocker() { + if (process.platform !== 'linux') return false; + if (fs.existsSync('/.dockerenv')) return true; + try { + return /(docker|containerd|kubepods|podman)/i.test(fs.readFileSync('/proc/1/cgroup', 'utf8')); + } catch { + return false; + } +} diff --git a/packages/bootstrap/src/embedded-worker.mjs b/packages/bootstrap/src/embedded-worker.mjs new file mode 100644 index 0000000..f4ea4b6 --- /dev/null +++ b/packages/bootstrap/src/embedded-worker.mjs @@ -0,0 +1,30 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const bootstrapDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +export function resolveEmbeddedWorkerLaunch() { + const candidateDirs = [ + path.resolve(bootstrapDir, '..', 'embedded-compute-worker'), + path.resolve(bootstrapDir, '..', 'compute-worker'), + path.join(process.cwd(), 'embedded-compute-worker'), + path.join(process.cwd(), 'packages', 'compute-worker'), + path.join(process.cwd(), 'compute-worker'), + ]; + + for (const workerDir of candidateDirs) { + if (!fs.existsSync(path.join(workerDir, 'src', 'server.ts'))) continue; + return { + cmd: process.execPath, + args: ['--import', 'tsx', 'src/server.ts'], + cwd: workerDir, + }; + } + + throw new Error( + 'Could not find an embedded compute worker runtime. ' + + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep packages/compute-worker available locally.', + ); +} diff --git a/scripts/migrate-fs-v2.mjs b/packages/bootstrap/src/storage-migration.mjs similarity index 97% rename from scripts/migrate-fs-v2.mjs rename to packages/bootstrap/src/storage-migration.mjs index e2f0c7d..53f159d 100644 --- a/scripts/migrate-fs-v2.mjs +++ b/packages/bootstrap/src/storage-migration.mjs @@ -17,7 +17,23 @@ const { Pool } = require('pg'); const BetterSqlite3 = require('better-sqlite3'); const ffmpegStatic = require('ffmpeg-static'); -const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); +function findWorkspaceRoot(startDir = process.cwd()) { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + +const workspaceRoot = findWorkspaceRoot(process.cwd()); +const DOCSTORE_DIR = path.join(workspaceRoot, 'docstore'); const DOCUMENTS_V1_DIR = path.join(DOCSTORE_DIR, 'documents_v1'); const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1'); const UNCLAIMED_USER_ID = 'unclaimed'; @@ -27,10 +43,15 @@ const SAFE_AUDIOBOOK_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; function loadEnvFiles() { - const envPath = path.join(process.cwd(), '.env'); - const envLocalPath = path.join(process.cwd(), '.env.local'); - if (fs.existsSync(envPath)) dotenv.config({ path: envPath }); - if (fs.existsSync(envLocalPath)) dotenv.config({ path: envLocalPath, override: true }); + const envPath = path.join(workspaceRoot, '.env'); + const envLocalPath = path.join(workspaceRoot, '.env.local'); + + if (fs.existsSync(envPath)) { + dotenv.config({ path: envPath }); + } + if (fs.existsSync(envLocalPath)) { + dotenv.config({ path: envLocalPath, override: true }); + } } function parseBool(value, fallback = false) { @@ -688,7 +709,7 @@ function openDatabase() { }; } - const dbPath = path.join(process.cwd(), 'docstore', 'sqlite3.db'); + const dbPath = path.join(DOCSTORE_DIR, 'sqlite3.db'); fs.mkdirSync(path.dirname(dbPath), { recursive: true }); const sqlite = new BetterSqlite3(dbPath); return { diff --git a/compute/worker/.env.example b/packages/compute-worker/.env.example similarity index 100% rename from compute/worker/.env.example rename to packages/compute-worker/.env.example diff --git a/compute/worker/Dockerfile b/packages/compute-worker/Dockerfile similarity index 62% rename from compute/worker/Dockerfile rename to packages/compute-worker/Dockerfile index 1b0eb3a..bcec9ac 100644 --- a/compute/worker/Dockerfile +++ b/packages/compute-worker/Dockerfile @@ -1,22 +1,17 @@ FROM node:lts AS deploy-stage -RUN npm install -g pnpm@11.1.2 +RUN npm install -g pnpm@10.33.4 WORKDIR /workspace COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute/worker/package.json compute/worker/package.json -COPY compute/core/package.json compute/core/package.json - -COPY compute/worker compute/worker -COPY compute/core compute/core +COPY packages/compute-worker/package.json packages/compute-worker/package.json +COPY packages/compute-worker packages/compute-worker RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker FROM node:lts -RUN npm install -g pnpm@11.1.2 - WORKDIR /workspace COPY --from=deploy-stage /opt/compute-worker ./ diff --git a/packages/compute-worker/openapi.json b/packages/compute-worker/openapi.json new file mode 100644 index 0000000..5b1f63a --- /dev/null +++ b/packages/compute-worker/openapi.json @@ -0,0 +1,1766 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "OpenReader Compute Worker API", + "version": "1.0.0" + }, + "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, + "schemas": { + "ParsedPdfDocument": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "documentId": { + "type": "string" + }, + "parserVersion": { + "type": "string" + }, + "parsedAt": { + "type": "number" + }, + "pages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pageNumber": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + }, + "blocks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "abstract", + "algorithm", + "aside_text", + "chart", + "content", + "formula", + "doc_title", + "figure_title", + "footer", + "footnote", + "formula_number", + "header", + "image", + "number", + "paragraph_title", + "reference", + "reference_content", + "seal", + "table", + "text", + "vision_footnote" + ] + }, + "fragments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "type": "number" + }, + "bbox": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "text": { + "type": "string" + }, + "readingOrder": { + "type": "number" + }, + "modelConfidence": { + "type": "number" + } + }, + "required": [ + "page", + "bbox", + "text", + "readingOrder" + ], + "additionalProperties": false + } + }, + "text": { + "type": "string" + }, + "parentSectionId": { + "type": "string" + } + }, + "required": [ + "id", + "kind", + "fragments", + "text" + ], + "additionalProperties": false + } + } + }, + "required": [ + "pageNumber", + "width", + "height", + "blocks" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "documentId", + "parserVersion", + "parsedAt", + "pages" + ], + "additionalProperties": false + }, + "TTSSentenceAlignment": { + "type": "object", + "properties": { + "sentence": { + "type": "string" + }, + "sentenceIndex": { + "type": "number" + }, + "words": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "startSec": { + "type": "number" + }, + "endSec": { + "type": "number" + }, + "charStart": { + "type": "number" + }, + "charEnd": { + "type": "number" + } + }, + "required": [ + "text", + "startSec", + "endSec", + "charStart", + "charEnd" + ], + "additionalProperties": false + } + } + }, + "required": [ + "sentence", + "sentenceIndex", + "words" + ], + "additionalProperties": false + }, + "ArtifactReference": { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + "ErrorResponse": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + }, + "OperationError": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "PdfLayoutProgress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + }, + "ComputeOperation": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + "ComputeOperationEvent": { + "type": "object", + "properties": { + "eventId": { + "type": "number" + }, + "snapshot": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "eventId", + "snapshot" + ], + "additionalProperties": false + }, + "PdfLayoutResolution": { + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "operation": { + "anyOf": [ + { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "artifact", + "operation" + ], + "additionalProperties": false + } + } + }, + "paths": { + "/health/live": { + "get": { + "security": [], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + } + }, + "required": [ + "ok" + ] + } + } + } + } + } + } + }, + "/health/ready": { + "get": { + "security": [], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "natsConnected": { + "type": "boolean" + } + }, + "required": [ + "ok", + "natsConnected" + ] + } + } + } + } + } + } + }, + "/v1/whisper-align/operations": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "minLength": 1 + }, + "lang": { + "type": "string", + "minLength": 1, + "maxLength": 16 + }, + "cacheKey": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "audioObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "text", + "audioObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/pdf-layout/operations": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "documentId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "namespace": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]{1,128}$" + }, + { + "type": "null" + } + ] + }, + "documentObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "replaceToken": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "required": [ + "documentId", + "namespace", + "documentObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/pdf-layout/resolve": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "documentId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "namespace": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]{1,128}$" + }, + { + "type": "null" + } + ] + }, + "documentObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "documentId", + "namespace", + "documentObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "operation": { + "anyOf": [ + { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "artifact", + "operation" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/operations/{opId}": { + "get": { + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "opId", + "required": true + } + ], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + }, + "404": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/operations/{opId}/events": { + "get": { + "parameters": [ + { + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "in": "query", + "name": "sinceEventId", + "required": false + }, + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "opId", + "required": true + } + ], + "responses": { + "200": { + "description": "Server-sent ComputeOperationEvent stream", + "content": { + "application/json": { + "schema": { + "type": "string", + "description": "Server-sent ComputeOperationEvent stream" + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + }, + "404": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + } + }, + "security": [ + { + "bearerAuth": [] + } + ] +} diff --git a/compute/worker/package.json b/packages/compute-worker/package.json similarity index 58% rename from compute/worker/package.json rename to packages/compute-worker/package.json index e07eded..ad31e23 100644 --- a/compute/worker/package.json +++ b/packages/compute-worker/package.json @@ -5,15 +5,22 @@ "type": "module", "scripts": { "dev": "tsx watch src/server.ts", - "start": "tsx src/server.ts" + "start": "tsx src/server.ts", + "openapi:generate": "tsx scripts/generate-openapi.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1061.0", + "@fastify/swagger": "^9.0.0", + "@huggingface/tokenizers": "^0.1.3", + "@napi-rs/canvas": "^0.1.100", "@nats-io/jetstream": "^3.4.0", "@nats-io/kv": "^3.4.0", "@nats-io/transport-node": "^3.4.0", - "@openreader/compute-core": "workspace:*", + "ffmpeg-static": "^5.3.0", "fastify": "^5.8.5", + "jszip": "^3.10.1", + "onnxruntime-node": "^1.26.0", + "pdfjs-dist": "4.8.69", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tsx": "^4.22.4", diff --git a/packages/compute-worker/scripts/generate-openapi.ts b/packages/compute-worker/scripts/generate-openapi.ts new file mode 100644 index 0000000..0cfb141 --- /dev/null +++ b/packages/compute-worker/scripts/generate-openapi.ts @@ -0,0 +1,20 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { createComputeWorkerApp } from '../src/api/app'; + +process.env.COMPUTE_WORKER_TOKEN ||= 'openapi-generation-token'; +process.env.NATS_URL ||= 'nats://127.0.0.1:4222'; + +const outputPath = resolve(process.cwd(), 'openapi.json'); +const runtime = await createComputeWorkerApp({ + workerToken: process.env.COMPUTE_WORKER_TOKEN, + disableWorkers: true, +}); + +try { + await runtime.app.ready(); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(runtime.app.swagger(), null, 2)}\n`, 'utf8'); +} finally { + await runtime.close(); +} diff --git a/packages/compute-worker/src/api/app.ts b/packages/compute-worker/src/api/app.ts new file mode 100644 index 0000000..7ea327c --- /dev/null +++ b/packages/compute-worker/src/api/app.ts @@ -0,0 +1,372 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import swagger from '@fastify/swagger'; +import { + credsAuthenticator, + type ConnectionOptions, +} from '@nats-io/transport-node'; +import { + ensureComputeModels, +} from '../inference/runtime'; +import { + getComputeTimeoutConfig, + getComputeOpStaleMs, + getAvailableCpuCores, + getOnnxThreadsPerJob, + buildLoggerConfig, + normalizeNatsReplicas, + readBoolEnv, + readPositiveIntEnv, + requireEnv, +} from '../infrastructure/config'; +import { OperationOrchestrator } from '../operations'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + WhisperAlignJobRequest, + WhisperAlignJobResult, +} from '../operations/contracts'; +import { + JetStreamOperationEventStream, + JetStreamOperationQueue, + JetStreamOperationStateStore, +} from '../infrastructure/nats-adapters'; +import { createJsonCodec } from '../infrastructure/json-codec'; +import { createOperationReconciler } from '../operations/reconciliation'; +import { + createArtifactStorage, + createS3ClientFromEnv, + normalizeS3Prefix, + type ArtifactStorage, +} from '../infrastructure/storage'; +import { createJobHandlers } from '../jobs/handlers'; +import { createWorkerLoopController, type QueuedJob } from '../jobs/worker-loop'; +import { createNatsSessionManager } from '../infrastructure/nats-session'; +import { + EVENTS_STREAM_NAME, + LAYOUT_JOBS_SUBJECT, + NATS_API_TIMEOUT_MS, + WHISPER_JOBS_SUBJECT, +} from '../infrastructure/nats'; +import { registerHttpHooks } from './http-hooks'; +import { + registerComputeWorkerRoutes, + type ComputeWorkerRouteDeps, +} from './routes'; +import { + apiErrorResponseSchema, + artifactReferenceSchema, + jsonSchema, + operationErrorSchema, + parsedPdfDocumentSchema, + pdfLayoutProgressSchema, + pdfLayoutResolutionSchema, + computeOperationEventSchema, + computeOperationSchema, + ttsSentenceAlignmentSchema, +} from './schemas'; + +// Disconnect from NATS after this much continuous idle so the worker stops +// generating outbound traffic (pull polling + keepalive PINGs) and Railway can +// put it to sleep. Reconnect happens lazily on the next inbound request. +const IDLE_DISCONNECT_MS = 120_000; +const IDLE_CHECK_INTERVAL_MS = 5_000; +const IDLE_STATUS_LOG_INTERVAL_MS = 60_000; +const ORPHAN_SWEEP_INTERVAL_MS = 15_000; +// Bounded pull window so consumer loops yield periodically and can be stopped +// cleanly when going idle, instead of blocking on a long-lived pull. +const WHISPER_MAX_DELIVER = 1; + +export type { ComputeWorkerRouteDeps } from './routes'; + +export interface CreateComputeWorkerAppOptions { + host?: string; + port?: number; + workerToken?: string; + routeDeps?: ComputeWorkerRouteDeps; + disableWorkers?: boolean; +} + +export interface ComputeWorkerApp { + app: FastifyInstance; + host: string; + port: number; + start(options?: { registerSignalHandlers?: boolean }): Promise; + close(): Promise; +} + +export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { + const port = options.port ?? readPositiveIntEnv('PORT', 8081); + const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); + const workerToken = options.workerToken ?? requireEnv('COMPUTE_WORKER_TOKEN'); + const disableWorkers = options.disableWorkers ?? false; + const natsUrl = requireEnv('NATS_URL'); + const timeoutConfig = getComputeTimeoutConfig(); + + const jobConcurrency = readPositiveIntEnv('COMPUTE_JOB_CONCURRENCY', 1); + const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; + const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; + const pdfHardCapMs = timeoutConfig.pdfHardCapMs; + const pdfAttempts = readPositiveIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); + const prewarmModels = readBoolEnv('COMPUTE_PREWARM_MODELS', false); + const jobsStreamMaxBytes = readPositiveIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); + const eventsStreamMaxBytes = readPositiveIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); + const jobStatesMaxBytes = readPositiveIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); + const natsReplicas = normalizeNatsReplicas(readPositiveIntEnv('COMPUTE_NATS_REPLICAS', 1)); + const opStaleMs = getComputeOpStaleMs(); + + const connectOpts: ConnectionOptions = { servers: natsUrl }; + const natsCreds = process.env.NATS_CREDS?.trim(); + const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); + + if (natsCreds) { + console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); + connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); + } else if (natsCredsFile) { + console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); + const { readFileSync } = await import('node:fs'); + const credsData = readFileSync(natsCredsFile); + connectOpts.authenticator = credsAuthenticator(credsData); + } + + let stopping = false; + let inFlightHttp = 0; + let activeSse = 0; + let inFlightJobs = 0; + let lastActivityAt = Date.now(); + let lastActivityReason = "startup"; + const markActivity = (reason: string): void => { + lastActivityAt = Date.now(); + lastActivityReason = reason; + }; + let sessionManager!: ReturnType; + const ensureConnected = () => sessionManager.ensureConnected(); + + const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); + const storageDisabled = async (): Promise => { + throw new Error('S3 access is disabled for this worker app instance'); + }; + const storage: ArtifactStorage = disableWorkers + ? { + readObject: storageDisabled, + objectExists: storageDisabled, + deleteObject: storageDisabled, + putParsedPdf: storageDisabled, + } + : createArtifactStorage({ + bucket: requireEnv('S3_BUCKET'), + prefix: s3Prefix, + client: createS3ClientFromEnv(requireEnv), + }); + + if (prewarmModels && !disableWorkers) { + await ensureComputeModels(); + } + + const app = Fastify({ + logger: buildLoggerConfig(), + disableRequestLogging: true, + }); + await app.register(swagger, { + openapi: { + openapi: '3.0.3', + info: { + title: 'OpenReader Compute Worker API', + version: '1.0.0', + }, + components: { + securitySchemes: { + bearerAuth: { + type: 'http', + scheme: 'bearer', + }, + }, + schemas: { + ParsedPdfDocument: jsonSchema(parsedPdfDocumentSchema), + TTSSentenceAlignment: jsonSchema(ttsSentenceAlignmentSchema), + ArtifactReference: jsonSchema(artifactReferenceSchema), + ErrorResponse: jsonSchema(apiErrorResponseSchema), + OperationError: jsonSchema(operationErrorSchema), + PdfLayoutProgress: jsonSchema(pdfLayoutProgressSchema), + ComputeOperation: jsonSchema(computeOperationSchema), + ComputeOperationEvent: jsonSchema(computeOperationEventSchema), + PdfLayoutResolution: jsonSchema(pdfLayoutResolutionSchema), + }, + }, + security: [{ bearerAuth: [] }], + }, + }); + app.log.info({ + jobConcurrency, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + opStaleMs, + availableCpuCores: getAvailableCpuCores(), + onnxThreadsPerJob: getOnnxThreadsPerJob(), + natsApiTimeoutMs: NATS_API_TIMEOUT_MS, + natsReplicas, + eventsStreamMaxBytes, + pdfLayoutHardCapMs: pdfHardCapMs, + }, 'compute runtime config'); + + const whisperJobCodec = createJsonCodec>(); + const layoutJobCodec = createJsonCodec>(); + + const defaultOperationStateStore = new JetStreamOperationStateStore({ + getKv: async () => (await ensureConnected()).kv, + }); + + const defaultOperationEventStream = new JetStreamOperationEventStream({ + getJs: async () => (await ensureConnected()).js, + getJsm: async () => (await ensureConnected()).jsm, + eventsStreamName: EVENTS_STREAM_NAME, + }); + + const operationQueue = new JetStreamOperationQueue({ + getJs: async () => (await ensureConnected()).js, + whisperSubject: WHISPER_JOBS_SUBJECT, + layoutSubject: LAYOUT_JOBS_SUBJECT, + }); + + const defaultOrchestrator = new OperationOrchestrator({ + queue: operationQueue, + stateStore: defaultOperationStateStore, + eventStream: defaultOperationEventStream, + config: { + opStaleMs, + maxCasRetries: 10, + }, + }); + + const operationStateStore = options.routeDeps?.operationStateStore ?? defaultOperationStateStore; + const operationEventStream = options.routeDeps?.operationEventStream ?? defaultOperationEventStream; + const orchestrator = options.routeDeps?.orchestrator ?? defaultOrchestrator; + const reconciler = createOperationReconciler({ + stateStore: operationStateStore, + orchestrator, + whisperTimeoutMs, + pdfTimeoutMs, + opStaleMs, + getGeneration: () => options.routeDeps ? 0 : sessionManager.getGeneration(), + logger: app.log, + }); + const ensureOrphanedOpRecovery = () => reconciler.run(); + const getOpState = (opId: string) => reconciler.getOpState(opId); + + const { releaseHttp } = registerHttpHooks({ + app, + workerToken, + markActivity, + onInFlightHttpChanged: (delta) => { + inFlightHttp = Math.max(0, inFlightHttp + delta); + }, + }); + + registerComputeWorkerRoutes({ + app, + deps: { + orchestrator, + operationStateStore, + operationEventStream, + artifactExists: options.routeDeps?.artifactExists ?? storage.objectExists, + }, + s3Prefix, + ensureOrphanedOpRecovery, + getOpState, + getNatsConnected: () => sessionManager.isConnected(), + releaseHttp, + markActivity, + onActiveSseChanged: (delta) => { + activeSse = Math.max(0, activeSse + delta); + }, + }); + + const jobHandlers = createJobHandlers({ + storage, + whisperTimeoutMs, + pdfTimeoutMs, + pdfHardCapMs, + }); + + const workerLoops = createWorkerLoopController({ + orchestrator, + handlers: jobHandlers, + logger: app.log, + jobConcurrency, + pdfAttempts, + whisperCodec: whisperJobCodec, + pdfCodec: layoutJobCodec, + isOwnerActive: (owner) => sessionManager.isOwnerActive(owner), + isStopping: () => stopping, + markActivity, + onInFlightJobsChanged: (delta) => { + inFlightJobs = Math.max(0, inFlightJobs + delta); + }, + }); + + sessionManager = createNatsSessionManager({ + connectOptions: connectOpts, + logger: app.log, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + jobsStreamMaxBytes, + eventsStreamMaxBytes, + jobStatesMaxBytes, + natsReplicas, + isStopping: () => stopping, + getActivity: () => ({ + activeSse, + inFlightHttp, + inFlightJobs, + lastActivityAt, + lastActivityReason, + }), + markActivity, + startWorkers: (session) => { + if (disableWorkers) return; + workerLoops.start(session, { + whisper: session.whisperConsumer, + pdfLayout: session.layoutConsumer, + }); + }, + stopWorkers: () => workerLoops.stop(), + runReconciliation: () => reconciler.run({ force: true }), + }); + + const close = async (): Promise => { + if (stopping) return; + stopping = true; + await app.close(); + await sessionManager.close(); + }; + + const registerSignalHandlers = (): void => { + process.once('SIGINT', () => { + void close().finally(() => process.exit(0)); + }); + + process.once('SIGTERM', () => { + void close().finally(() => process.exit(0)); + }); + }; + + return { + app, + host, + port, + async start(startOptions) { + if (startOptions?.registerSignalHandlers) { + registerSignalHandlers(); + } + await app.listen({ host, port }); + app.log.info({ host, port }, 'compute worker listening'); + }, + close, + }; +} + +export async function startComputeWorkerFromEnv(): Promise { + const runtime = await createComputeWorkerApp(); + await runtime.start({ registerSignalHandlers: true }); +} diff --git a/packages/compute-worker/src/api/compute-operation.ts b/packages/compute-worker/src/api/compute-operation.ts new file mode 100644 index 0000000..03c6360 --- /dev/null +++ b/packages/compute-worker/src/api/compute-operation.ts @@ -0,0 +1,44 @@ +import type { WorkerOperationState } from '../operations/contracts'; +import { pdfSubjectFromOperationKey } from '../operations/keys'; + +export type ComputeOperationSubject = + | { kind: 'whisper_align' } + | { kind: 'pdf_layout'; documentId: string; namespace: string | null }; + +export interface ComputeOperation { + opId: string; + subject: ComputeOperationSubject; + status: WorkerOperationState['status']; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: Result; + error?: WorkerOperationState['error']; + timing?: WorkerOperationState['timing']; + progress?: WorkerOperationState['progress']; +} + +export interface ComputeOperationEvent { + eventId: number; + snapshot: ComputeOperation; +} + +export function toComputeOperation( + state: WorkerOperationState, +): ComputeOperation { + const subject = state.kind === 'pdf_layout' + ? (pdfSubjectFromOperationKey(state.opKey) ?? { kind: 'pdf_layout', documentId: '', namespace: null }) + : { kind: 'whisper_align' as const }; + return { + opId: state.opId, + subject, + status: state.status, + queuedAt: state.queuedAt, + updatedAt: state.updatedAt, + ...(state.startedAt === undefined ? {} : { startedAt: state.startedAt }), + ...(state.result === undefined ? {} : { result: state.result }), + ...(state.error === undefined ? {} : { error: state.error }), + ...(state.timing === undefined ? {} : { timing: state.timing }), + ...(state.progress === undefined ? {} : { progress: state.progress }), + }; +} diff --git a/packages/compute-worker/src/api/http-hooks.ts b/packages/compute-worker/src/api/http-hooks.ts new file mode 100644 index 0000000..d633da3 --- /dev/null +++ b/packages/compute-worker/src/api/http-hooks.ts @@ -0,0 +1,80 @@ +import type { FastifyInstance, FastifyRequest } from 'fastify'; + +const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); +const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); + +function requestPath(request: FastifyRequest): string { + return request.url.split('?')[0] ?? request.url; +} + +function isHealthPath(path: string): boolean { + return path === '/health/live' || path === '/health/ready'; +} + +function isAuthed(request: FastifyRequest, expectedToken: string): boolean { + const auth = request.headers.authorization; + return auth?.startsWith('Bearer ') === true + && auth.slice('Bearer '.length).trim() === expectedToken; +} + +function extractTraceId(request: FastifyRequest): string | null { + const header = request.headers['x-openreader-trace-id']; + if (Array.isArray(header)) return header[0] ?? null; + return typeof header === 'string' ? header : null; +} + +function extractOpId(request: FastifyRequest, path: string): string | null { + const params = request.params as { opId?: unknown } | undefined; + if (typeof params?.opId === 'string' && params.opId.trim()) return params.opId.trim(); + const match = path.match(/^\/v1\/operations\/([^/]+)/); + if (!match?.[1]) return null; + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } +} + +export function registerHttpHooks(input: { + app: FastifyInstance; + workerToken: string; + markActivity: (reason: string) => void; + onInFlightHttpChanged: (delta: number) => void; +}) { + const releaseHttp = (request: FastifyRequest): void => { + const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; + if (!counted[REQUEST_COUNTED_KEY]) return; + counted[REQUEST_COUNTED_KEY] = false; + input.onInFlightHttpChanged(-1); + input.markActivity('http_completed'); + }; + + input.app.addHook('onRequest', async (request, reply) => { + const path = requestPath(request); + (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); + (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; + input.onInFlightHttpChanged(1); + input.markActivity(`http_started:${path}`); + if (!isHealthPath(path) && !isAuthed(request, input.workerToken)) { + return reply.code(401).send({ error: 'Unauthorized' }); + } + }); + + input.app.addHook('onResponse', async (request, reply) => { + releaseHttp(request); + const path = requestPath(request); + if (isHealthPath(path) || reply.statusCode < 500) return; + const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; + input.app.log.error({ + reqId: request.id, + method: request.method, + path, + statusCode: reply.statusCode, + durationMs: Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1, + traceId: extractTraceId(request), + opId: extractOpId(request, path), + }, 'http.error'); + }); + + return { releaseHttp }; +} diff --git a/packages/compute-worker/src/api/routes.ts b/packages/compute-worker/src/api/routes.ts new file mode 100644 index 0000000..481e965 --- /dev/null +++ b/packages/compute-worker/src/api/routes.ts @@ -0,0 +1,356 @@ +import type { FastifyInstance, FastifyRequest } from 'fastify'; +import { encodeSseFrame } from '../operations'; +import type { + PdfLayoutJobResult, + WorkerJobTiming, + WorkerOperationEvent, + WorkerOperationRequest, + WorkerOperationState, + WhisperAlignJobResult, +} from '../operations/contracts'; +import { hashOpKey } from '../infrastructure/nats-adapters'; +import type { StreamedOperationState } from '../operations/recovery'; +import type { ReconciliationStateStore } from '../operations/reconciliation'; +import { parsedPdfArtifactKey } from '../storage/artifact-addressing'; +import { buildPdfOperationKey, buildWhisperOperationKey } from '../operations/keys'; +import { toComputeOperation, type ComputeOperationEvent } from './compute-operation'; +import { + apiErrorResponseSchema, + jsonSchema, + operationEventsQuerySchema, + operationParamsSchema, + pdfLayoutResolutionSchema, + pdfOperationCreateSchema, + pdfResolveSchema, + computeOperationSchema, + whisperOperationCreateSchema, +} from './schemas'; + +const OP_EVENTS_KEEPALIVE_MS = 15_000; +const OP_EVENTS_RECONNECT_HINT_MS = 120_000; +const errorResponseSchema = jsonSchema(apiErrorResponseSchema); + +interface OperationEventStreamLike { + subscribe(input: { + opId: string; + sinceEventId?: number; + onEvent: (event: WorkerOperationEvent) => void | Promise; + onError?: (error: unknown) => void; + }): Promise<() => void>; +} + +interface OperationStateStoreLike extends ReconciliationStateStore {} + +interface OrchestratorLike { + enqueueOrReuse(request: WorkerOperationRequest): Promise; + markRunning(input: { + opId: string; + startedAt?: number; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markProgress(input: { + opId: string; + progress: WorkerOperationState['progress']; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { + opId: string; + result: unknown; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailedIfUnchanged?(input: { + current: StreamedOperationState; + expectedRevision: number; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +export interface ComputeWorkerRouteDeps { + orchestrator: OrchestratorLike; + operationStateStore: OperationStateStoreLike; + operationEventStream: OperationEventStreamLike; + artifactExists?: (key: string) => Promise; +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + return String(error); +} + +function isTerminalStatus(status: import('../operations/contracts').WorkerJobState): boolean { + return status === 'succeeded' || status === 'failed'; +} + +export function registerComputeWorkerRoutes(input: { + app: FastifyInstance; + deps: ComputeWorkerRouteDeps; + s3Prefix: string; + ensureOrphanedOpRecovery: () => Promise; + getOpState: (opId: string) => Promise; + getNatsConnected: () => boolean; + releaseHttp: (request: FastifyRequest) => void; + markActivity: (reason: string) => void; + onActiveSseChanged: (delta: number) => void; +}) { + const { + app, + deps, + s3Prefix, + ensureOrphanedOpRecovery, + getOpState, + getNatsConnected, + releaseHttp, + markActivity, + onActiveSseChanged, + } = input; + + app.get('/health/live', { + schema: { + security: [], + response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } }, + }, + }, async () => ({ ok: true })); + + app.get('/health/ready', { + schema: { + security: [], + response: { + 200: { + type: 'object', + properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } }, + required: ['ok', 'natsConnected'], + }, + }, + }, + }, async () => ({ ok: true, natsConnected: getNatsConnected() })); + + app.post('/v1/whisper-align/operations', { + schema: { + body: jsonSchema(whisperOperationCreateSchema), + response: { 202: jsonSchema(computeOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = whisperOperationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + + const requestOp: WorkerOperationRequest = { + kind: 'whisper_align', + opKey: buildWhisperOperationKey(parsed.data), + payload: parsed.data, + }; + await ensureOrphanedOpRecovery(); + const op = await deps.orchestrator.enqueueOrReuse(requestOp); + app.log.info({ + kind: requestOp.kind, + opId: op.opId, + jobId: op.jobId, + status: op.status, + opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), + }, 'op.accepted'); + reply.code(202); + return toComputeOperation(op); + }); + + app.post('/v1/pdf-layout/operations', { + schema: { + body: jsonSchema(pdfOperationCreateSchema), + response: { 202: jsonSchema(computeOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = pdfOperationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + + const requestOp: WorkerOperationRequest = { + kind: 'pdf_layout', + opKey: buildPdfOperationKey(parsed.data), + payload: { + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + documentObjectKey: parsed.data.documentObjectKey, + }, + }; + await ensureOrphanedOpRecovery(); + const op = await deps.orchestrator.enqueueOrReuse(requestOp); + reply.code(202); + return toComputeOperation(op); + }); + + app.post('/v1/pdf-layout/resolve', { + schema: { + body: jsonSchema(pdfResolveSchema), + response: { 200: jsonSchema(pdfLayoutResolutionSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = pdfResolveSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + await ensureOrphanedOpRecovery(); + const artifactKey = parsedPdfArtifactKey({ + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + prefix: s3Prefix, + }); + const hasArtifact = await deps.artifactExists?.(artifactKey) ?? false; + const opKey = buildPdfOperationKey(parsed.data); + const index = await deps.operationStateStore.getOpIndex?.(opKey); + const operation = index?.opId ? await deps.operationStateStore.getOpState(index.opId) : null; + return { + artifact: hasArtifact ? { objectKey: artifactKey } : null, + operation: operation ? toComputeOperation(operation) : null, + }; + }); + + app.get('/v1/operations/:opId', { + schema: { + params: jsonSchema(operationParamsSchema), + response: { 200: jsonSchema(computeOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const state = await getOpState(params.data.opId); + if (!state) { + reply.code(404); + return { error: 'Operation not found' }; + } + return toComputeOperation(state); + }); + + app.get('/v1/operations/:opId/events', { + schema: { + params: jsonSchema(operationParamsSchema), + querystring: jsonSchema(operationEventsQuerySchema), + response: { + 200: { type: 'string', description: 'Server-sent ComputeOperationEvent stream' }, + 400: errorResponseSchema, + 404: errorResponseSchema, + }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const initial = await getOpState(params.data.opId); + if (!initial) { + reply.code(404); + return { error: 'Operation not found' }; + } + + const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; + const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); + const lastEventIdHeader = request.headers['last-event-id']; + const cursorFromHeader = Number( + Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), + ); + const sinceEventId = Math.max( + 0, + Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, + Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, + ); + + reply.hijack(); + releaseHttp(request); + onActiveSseChanged(1); + markActivity('sse_started'); + reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); + reply.raw.setHeader('Connection', 'keep-alive'); + reply.raw.setHeader('X-Accel-Buffering', 'no'); + reply.raw.write(encodeSseFrame({ retry: OP_EVENTS_RECONNECT_HINT_MS })); + + let closed = false; + let unsubscribe: (() => void) | null = null; + let keepalive: NodeJS.Timeout | null = null; + + const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { + if (closed || reply.raw.writableEnded) return; + const frameEvent: ComputeOperationEvent = { + eventId, + snapshot: toComputeOperation(snapshot), + }; + reply.raw.write(encodeSseFrame({ id: eventId, event: 'snapshot', data: frameEvent })); + }; + + const closeStream = (): void => { + if (closed) return; + closed = true; + if (unsubscribe) { + unsubscribe(); + unsubscribe = null; + } + if (keepalive) { + clearInterval(keepalive); + keepalive = null; + } + onActiveSseChanged(-1); + markActivity('sse_closed'); + if (!reply.raw.writableEnded) reply.raw.end(); + }; + + request.raw.on('close', closeStream); + + try { + let current = initial; + let signature = JSON.stringify(current); + writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); + if (isTerminalStatus(current.status)) return reply; + + keepalive = setInterval(() => { + if (!closed && !reply.raw.writableEnded) reply.raw.write(': keepalive\n\n'); + }, OP_EVENTS_KEEPALIVE_MS); + + unsubscribe = await deps.operationEventStream.subscribe({ + opId: params.data.opId, + sinceEventId, + onEvent: (event) => { + if (closed || event.snapshot.opId !== params.data.opId) return; + const nextSignature = JSON.stringify(event.snapshot); + if (nextSignature !== signature) { + current = event.snapshot; + signature = nextSignature; + markActivity('sse_event'); + writeSnapshot(current, event.eventId); + } + if (isTerminalStatus(event.snapshot.status)) closeStream(); + }, + onError: (error) => { + app.log.warn({ opId: params.data.opId, error: toErrorMessage(error) }, 'op events stream loop error'); + closeStream(); + }, + }); + + await new Promise((resolve) => request.raw.once('close', resolve)); + } catch (error) { + app.log.warn({ opId: params.data.opId, error: toErrorMessage(error) }, 'op events stream loop error'); + } finally { + closeStream(); + } + return reply; + }); +} diff --git a/packages/compute-worker/src/api/schemas.ts b/packages/compute-worker/src/api/schemas.ts new file mode 100644 index 0000000..0175d86 --- /dev/null +++ b/packages/compute-worker/src/api/schemas.ts @@ -0,0 +1,134 @@ +import { z } from 'zod'; + +const bboxSchema = z.tuple([z.number(), z.number(), z.number(), z.number()]); +const parsedPdfBlockKindSchema = z.enum([ + 'abstract', 'algorithm', 'aside_text', 'chart', 'content', 'formula', + 'doc_title', 'figure_title', 'footer', 'footnote', 'formula_number', + 'header', 'image', 'number', 'paragraph_title', 'reference', + 'reference_content', 'seal', 'table', 'text', 'vision_footnote', +]); +const documentIdSchema = z.string().trim().regex(/^[a-f0-9]{64}$/i); +const namespaceSchema = z.string().trim().regex(/^[a-zA-Z0-9._-]{1,128}$/).nullable(); + +export const parsedPdfDocumentSchema = z.object({ + schemaVersion: z.literal(1), + documentId: z.string(), + parserVersion: z.string(), + parsedAt: z.number(), + pages: z.array(z.object({ + pageNumber: z.number(), + width: z.number(), + height: z.number(), + blocks: z.array(z.object({ + id: z.string(), + kind: parsedPdfBlockKindSchema, + fragments: z.array(z.object({ + page: z.number(), + bbox: bboxSchema, + text: z.string(), + readingOrder: z.number(), + modelConfidence: z.number().optional(), + })), + text: z.string(), + parentSectionId: z.string().optional(), + })), + })), +}); + +export const ttsSentenceAlignmentSchema = z.object({ + sentence: z.string(), + sentenceIndex: z.number(), + words: z.array(z.object({ + text: z.string(), + startSec: z.number(), + endSec: z.number(), + charStart: z.number(), + charEnd: z.number(), + })), +}); + +export const whisperOperationCreateSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +export const pdfOperationCreateSchema = z.object({ + documentId: documentIdSchema, + namespace: namespaceSchema, + documentObjectKey: z.string().trim().min(1).max(2048), + replaceToken: z.string().trim().min(1).max(256).optional(), +}); + +export const pdfResolveSchema = z.object({ + documentId: documentIdSchema, + namespace: namespaceSchema, + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +export const operationParamsSchema = z.object({ + opId: z.string().trim().min(1), +}); + +export const operationEventsQuerySchema = z.object({ + sinceEventId: z.union([z.string(), z.number()]).optional(), +}); + +export const apiErrorResponseSchema = z.object({ + error: z.string(), +}).passthrough(); + +export const operationErrorSchema = z.object({ + message: z.string(), + code: z.string().optional(), +}); + +export const artifactReferenceSchema = z.object({ + objectKey: z.string(), +}); + +export const pdfLayoutProgressSchema = z.object({ + totalPages: z.number(), + pagesParsed: z.number(), + currentPage: z.number().optional(), + phase: z.enum(['infer', 'merge']), +}); + +export const computeOperationSchema = z.object({ + opId: z.string(), + subject: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('whisper_align') }), + z.object({ + kind: z.literal('pdf_layout'), + documentId: z.string(), + namespace: z.string().nullable(), + }), + ]), + status: z.enum(['queued', 'running', 'succeeded', 'failed']), + queuedAt: z.number(), + updatedAt: z.number(), + startedAt: z.number().optional(), + result: z.unknown().optional(), + error: operationErrorSchema.optional(), + timing: z.object({ + queueWaitMs: z.number().optional(), + s3FetchMs: z.number().optional(), + computeMs: z.number().optional(), + }).optional(), + progress: pdfLayoutProgressSchema.optional(), +}); + +export const computeOperationEventSchema = z.object({ + eventId: z.number(), + snapshot: computeOperationSchema, +}); + +export const pdfLayoutResolutionSchema = z.object({ + artifact: artifactReferenceSchema.nullable(), + operation: computeOperationSchema.nullable(), +}); + +export function jsonSchema(schema: z.ZodType): Record { + return z.toJSONSchema(schema, { target: 'draft-7' }) as Record; +} diff --git a/compute/core/src/types/parsed-pdf.ts b/packages/compute-worker/src/api/types.ts similarity index 78% rename from compute/core/src/types/parsed-pdf.ts rename to packages/compute-worker/src/api/types.ts index 01d7067..16ad911 100644 --- a/compute/core/src/types/parsed-pdf.ts +++ b/packages/compute-worker/src/api/types.ts @@ -61,3 +61,20 @@ export interface PdfParseProgress { currentPage?: number; phase: PdfParsePhase; } + +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; + +export interface TTSSentenceWord { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; +} + +export interface TTSSentenceAlignment { + sentence: string; + sentenceIndex: number; + words: TTSSentenceWord[]; +} diff --git a/compute/core/src/pdf/assets/LICENSE.txt b/packages/compute-worker/src/inference/pdf/assets/LICENSE.txt similarity index 100% rename from compute/core/src/pdf/assets/LICENSE.txt rename to packages/compute-worker/src/inference/pdf/assets/LICENSE.txt diff --git a/compute/core/src/pdf/assets/manifest.json b/packages/compute-worker/src/inference/pdf/assets/manifest.json similarity index 100% rename from compute/core/src/pdf/assets/manifest.json rename to packages/compute-worker/src/inference/pdf/assets/manifest.json diff --git a/compute/core/src/pdf/merge.ts b/packages/compute-worker/src/inference/pdf/document-layout.ts similarity index 94% rename from compute/core/src/pdf/merge.ts rename to packages/compute-worker/src/inference/pdf/document-layout.ts index e1f71b2..8d48fd9 100644 --- a/compute/core/src/pdf/merge.ts +++ b/packages/compute-worker/src/inference/pdf/document-layout.ts @@ -1,4 +1,18 @@ -import type { LayoutRegion, PdfTextItem } from './types'; +import type { ParsedPdfBlockKind } from '../../api/types'; + +export interface PdfTextItem { + text: string; + x: number; + y: number; + width: number; + height: number; +} + +export interface LayoutRegion { + bbox: [number, number, number, number]; + label: ParsedPdfBlockKind; + confidence?: number; +} const NON_TEXT_REGION_LABELS = new Set(['chart', 'image', 'table', 'seal']); const TEXT_ASSIGNABLE_LABELS = new Set([ diff --git a/compute/core/src/pdf/runLayoutModel.ts b/packages/compute-worker/src/inference/pdf/layout-model.ts similarity index 98% rename from compute/core/src/pdf/runLayoutModel.ts rename to packages/compute-worker/src/inference/pdf/layout-model.ts index 5c726e0..4e1e6e2 100644 --- a/compute/core/src/pdf/runLayoutModel.ts +++ b/packages/compute-worker/src/inference/pdf/layout-model.ts @@ -1,8 +1,8 @@ import * as ort from 'onnxruntime-node'; import { readFile } from 'fs/promises'; -import type { LayoutRegion, PdfTextItem } from './types'; +import type { LayoutRegion, PdfTextItem } from './document-layout'; import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './model'; -import { getOnnxThreadsPerJob } from '../config/cpu-budget'; +import { getOnnxThreadsPerJob } from '../../infrastructure/config'; interface RunLayoutInput { pageWidth: number; diff --git a/compute/core/src/pdf/model.ts b/packages/compute-worker/src/inference/pdf/model.ts similarity index 98% rename from compute/core/src/pdf/model.ts rename to packages/compute-worker/src/inference/pdf/model.ts index ca09540..2d13cda 100644 --- a/compute/core/src/pdf/model.ts +++ b/packages/compute-worker/src/inference/pdf/model.ts @@ -3,7 +3,7 @@ import { fileURLToPath } from 'url'; import { createHash } from 'crypto'; import { readFileSync } from 'fs'; import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises'; -import { DOCSTORE_DIR } from '../platform/docstore'; +import { DOCSTORE_DIR } from '../../infrastructure/platform'; const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main'; const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL'; diff --git a/compute/core/src/pdf/normalize-text.ts b/packages/compute-worker/src/inference/pdf/normalize-text.ts similarity index 98% rename from compute/core/src/pdf/normalize-text.ts rename to packages/compute-worker/src/inference/pdf/normalize-text.ts index 1e91101..f8e5dc0 100644 --- a/compute/core/src/pdf/normalize-text.ts +++ b/packages/compute-worker/src/inference/pdf/normalize-text.ts @@ -1,5 +1,5 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; -import type { PdfTextItem } from './types'; +import type { PdfTextItem } from './document-layout'; interface ViewportLike { height: number; diff --git a/compute/core/src/pdf/parse.ts b/packages/compute-worker/src/inference/pdf/parse.ts similarity index 94% rename from compute/core/src/pdf/parse.ts rename to packages/compute-worker/src/inference/pdf/parse.ts index 69d10d2..4f70f53 100644 --- a/compute/core/src/pdf/parse.ts +++ b/packages/compute-worker/src/inference/pdf/parse.ts @@ -1,10 +1,10 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; -import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf'; +import type { ParsedPdfDocument, ParsedPdfPage } from '../../api/types'; import { ensureModel } from './model'; -import { runLayoutModel } from './runLayoutModel'; -import { mergeTextWithRegions } from './merge'; -import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime'; -import { PDF_PARSER_VERSION } from './parser-version'; +import { runLayoutModel } from './layout-model'; +import { mergeTextWithRegions } from './document-layout'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs'; +import { PDF_PARSER_VERSION } from '../../operations/contracts'; import { stitchCrossPageBlocks } from './stitch'; import { renderPage } from './render'; import { normalizeTextItemsForLayout } from './normalize-text'; diff --git a/compute/core/src/pdf/pdfjs-runtime.ts b/packages/compute-worker/src/inference/pdf/pdfjs.ts similarity index 100% rename from compute/core/src/pdf/pdfjs-runtime.ts rename to packages/compute-worker/src/inference/pdf/pdfjs.ts diff --git a/compute/core/src/pdf/render.ts b/packages/compute-worker/src/inference/pdf/render.ts similarity index 99% rename from compute/core/src/pdf/render.ts rename to packages/compute-worker/src/inference/pdf/render.ts index 33ec1a1..eb23993 100644 --- a/compute/core/src/pdf/render.ts +++ b/packages/compute-worker/src/inference/pdf/render.ts @@ -1,5 +1,5 @@ import type { Canvas } from '@napi-rs/canvas'; -import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs'; type CanvasRuntime = { DOMMatrixCtor: unknown; diff --git a/compute/core/src/pdf/stitch.ts b/packages/compute-worker/src/inference/pdf/stitch.ts similarity index 98% rename from compute/core/src/pdf/stitch.ts rename to packages/compute-worker/src/inference/pdf/stitch.ts index 98a3d77..d3da6f6 100644 --- a/compute/core/src/pdf/stitch.ts +++ b/packages/compute-worker/src/inference/pdf/stitch.ts @@ -1,4 +1,4 @@ -import type { ParsedPdfDocument, ParsedPdfBlock } from '../types/parsed-pdf'; +import type { ParsedPdfDocument, ParsedPdfBlock } from '../../api/types'; const STITCHABLE_KINDS = new Set([ 'text', diff --git a/compute/core/src/local-runtime.ts b/packages/compute-worker/src/inference/runtime.ts similarity index 100% rename from compute/core/src/local-runtime.ts rename to packages/compute-worker/src/inference/runtime.ts diff --git a/compute/core/src/whisper/align.ts b/packages/compute-worker/src/inference/whisper/align.ts similarity index 89% rename from compute/core/src/whisper/align.ts rename to packages/compute-worker/src/inference/whisper/align.ts index d18fa07..a271623 100644 --- a/compute/core/src/whisper/align.ts +++ b/packages/compute-worker/src/inference/whisper/align.ts @@ -7,19 +7,18 @@ import { fileURLToPath } from 'url'; import * as ort from 'onnxruntime-node'; import { Tokenizer } from '@huggingface/tokenizers'; import JSZip from 'jszip'; -import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../types/tts'; -import { getFFmpegPath } from '../platform/ffmpeg'; -import { getOnnxThreadsPerJob } from '../config/cpu-budget'; -import { getComputeTimeoutConfig } from '../config/timeout'; +import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../../api/types'; +import { getFFmpegPath } from '../../infrastructure/platform'; +import { getOnnxThreadsPerJob } from '../../infrastructure/config'; +import { getComputeTimeoutConfig } from '../../infrastructure/config'; import { mapWordsToSentenceOffsets, type WhisperWord, -} from './alignment-map'; -import { buildGoertzelCoefficients, goertzelPower } from './spectral'; +} from './timestamps'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from './token-timestamps'; +} from './timestamps'; import { ensureWhisperModel, WHISPER_CONFIG_PATH, @@ -30,6 +29,31 @@ import { WHISPER_DECODER_MERGED_MODEL_PATH, WHISPER_DECODER_WITH_PAST_MODEL_PATH, } from './model'; +import { + applyTokenSuppression, + applyWhisperTimestampLogitsRules, + argmax, +} from './decoder'; + +export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array { + const coeffs = new Float64Array(freqBins); + for (let k = 0; k < freqBins; k += 1) { + coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize); + } + return coeffs; +} + +export function goertzelPower(samples: Float32Array, coeff: number): number { + let s1 = 0; + let s2 = 0; + for (let i = 0; i < samples.length; i += 1) { + const s0 = samples[i] + (coeff * s1) - s2; + s2 = s1; + s1 = s0; + } + const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2); + return !Number.isFinite(power) || power < 0 ? 0 : power; +} interface WhisperAlignmentOptions { lang?: string; @@ -429,118 +453,6 @@ function buildEmptyPastFeeds() { return state.emptyPastFeedsTemplate; } -function argmax(values: Float32Array): number | null { - let bestIdx = 0; - let bestScore = Number.NEGATIVE_INFINITY; - - for (let i = 0; i < values.length; i += 1) { - const score = values[i]; - if (score > bestScore) { - bestScore = score; - bestIdx = i; - } - } - - return Number.isFinite(bestScore) ? bestIdx : null; -} - -function applyTokenSuppression(logits: Float32Array, tokens: Set) { - for (const tokenId of tokens) { - if (tokenId >= 0 && tokenId < logits.length) { - logits[tokenId] = Number.NEGATIVE_INFINITY; - } - } -} - -function logSoftmax(input: Float32Array): Float32Array { - let max = Number.NEGATIVE_INFINITY; - for (let i = 0; i < input.length; i += 1) { - if (input[i] > max) max = input[i]; - } - if (!Number.isFinite(max)) { - return new Float32Array(input.length).fill(Number.NEGATIVE_INFINITY); - } - - let sum = 0; - for (let i = 0; i < input.length; i += 1) { - sum += Math.exp(input[i] - max); - } - const logSum = Math.log(sum); - - const out = new Float32Array(input.length); - for (let i = 0; i < input.length; i += 1) { - out[i] = input[i] - max - logSum; - } - return out; -} - -function applyWhisperTimestampLogitsRules(input: { - logits: Float32Array; - generated: number[]; - beginIndex: number; - eosTokenId: number; - noTimestampsTokenId: number; - timestampBeginTokenId: number; - maxInitialTimestampIndex: number; -}) { - const { - logits, - generated, - beginIndex, - eosTokenId, - noTimestampsTokenId, - timestampBeginTokenId, - maxInitialTimestampIndex, - } = input; - - if (noTimestampsTokenId >= 0 && noTimestampsTokenId < logits.length) { - logits[noTimestampsTokenId] = Number.NEGATIVE_INFINITY; - } - - if (generated.length === beginIndex) { - const upper = Math.min(timestampBeginTokenId, logits.length); - for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } - - const seq = generated.slice(beginIndex); - const lastWasTimestamp = seq.length >= 1 && seq[seq.length - 1] >= timestampBeginTokenId; - const penultimateWasTimestamp = seq.length < 2 || seq[seq.length - 2] >= timestampBeginTokenId; - - if (lastWasTimestamp) { - if (penultimateWasTimestamp) { - for (let i = timestampBeginTokenId; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } else { - const upper = Math.min(eosTokenId, logits.length); - for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } - } - - if (generated.length === beginIndex && Number.isFinite(maxInitialTimestampIndex)) { - const lastAllowed = timestampBeginTokenId + maxInitialTimestampIndex; - for (let i = lastAllowed + 1; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } - - const textUpper = Math.min(timestampBeginTokenId, logits.length); - if (textUpper <= 0 || textUpper >= logits.length) return; - - const logprobs = logSoftmax(logits); - - let maxTextTokenLogprob = Number.NEGATIVE_INFINITY; - for (let i = 0; i < textUpper; i += 1) { - if (logprobs[i] > maxTextTokenLogprob) maxTextTokenLogprob = logprobs[i]; - } - - let timestampProbMass = 0; - for (let i = textUpper; i < logprobs.length; i += 1) { - timestampProbMass += Math.exp(logprobs[i]); - } - const timestampLogprob = timestampProbMass > 0 ? Math.log(timestampProbMass) : Number.NEGATIVE_INFINITY; - - if (timestampLogprob > maxTextTokenLogprob) { - for (let i = 0; i < textUpper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } -} - async function getRuntime(): Promise { if (state.runtimePromise) return state.runtimePromise; diff --git a/compute/core/src/whisper/assets/LICENSE.txt b/packages/compute-worker/src/inference/whisper/assets/LICENSE.txt similarity index 100% rename from compute/core/src/whisper/assets/LICENSE.txt rename to packages/compute-worker/src/inference/whisper/assets/LICENSE.txt diff --git a/compute/core/src/whisper/assets/manifest.json b/packages/compute-worker/src/inference/whisper/assets/manifest.json similarity index 100% rename from compute/core/src/whisper/assets/manifest.json rename to packages/compute-worker/src/inference/whisper/assets/manifest.json diff --git a/compute/core/src/whisper/assets/mel_filters.npz b/packages/compute-worker/src/inference/whisper/assets/mel_filters.npz similarity index 100% rename from compute/core/src/whisper/assets/mel_filters.npz rename to packages/compute-worker/src/inference/whisper/assets/mel_filters.npz diff --git a/packages/compute-worker/src/inference/whisper/decoder.ts b/packages/compute-worker/src/inference/whisper/decoder.ts new file mode 100644 index 0000000..777047e --- /dev/null +++ b/packages/compute-worker/src/inference/whisper/decoder.ts @@ -0,0 +1,111 @@ +export function argmax(values: Float32Array): number | null { + let bestIdx = 0; + let bestScore = Number.NEGATIVE_INFINITY; + + for (let i = 0; i < values.length; i += 1) { + const score = values[i]; + if (score > bestScore) { + bestScore = score; + bestIdx = i; + } + } + + return Number.isFinite(bestScore) ? bestIdx : null; +} + +export function applyTokenSuppression(logits: Float32Array, tokens: Set) { + for (const tokenId of tokens) { + if (tokenId >= 0 && tokenId < logits.length) { + logits[tokenId] = Number.NEGATIVE_INFINITY; + } + } +} + +function logSoftmax(input: Float32Array): Float32Array { + let max = Number.NEGATIVE_INFINITY; + for (let i = 0; i < input.length; i += 1) { + if (input[i] > max) max = input[i]; + } + if (!Number.isFinite(max)) { + return new Float32Array(input.length).fill(Number.NEGATIVE_INFINITY); + } + + let sum = 0; + for (let i = 0; i < input.length; i += 1) { + sum += Math.exp(input[i] - max); + } + const logSum = Math.log(sum); + + const out = new Float32Array(input.length); + for (let i = 0; i < input.length; i += 1) { + out[i] = input[i] - max - logSum; + } + return out; +} + +export function applyWhisperTimestampLogitsRules(input: { + logits: Float32Array; + generated: number[]; + beginIndex: number; + eosTokenId: number; + noTimestampsTokenId: number; + timestampBeginTokenId: number; + maxInitialTimestampIndex: number; +}) { + const { + logits, + generated, + beginIndex, + eosTokenId, + noTimestampsTokenId, + timestampBeginTokenId, + maxInitialTimestampIndex, + } = input; + + if (noTimestampsTokenId >= 0 && noTimestampsTokenId < logits.length) { + logits[noTimestampsTokenId] = Number.NEGATIVE_INFINITY; + } + + if (generated.length === beginIndex) { + const upper = Math.min(timestampBeginTokenId, logits.length); + for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + + const seq = generated.slice(beginIndex); + const lastWasTimestamp = seq.length >= 1 && seq[seq.length - 1] >= timestampBeginTokenId; + const penultimateWasTimestamp = seq.length < 2 || seq[seq.length - 2] >= timestampBeginTokenId; + + if (lastWasTimestamp) { + if (penultimateWasTimestamp) { + for (let i = timestampBeginTokenId; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } else { + const upper = Math.min(eosTokenId, logits.length); + for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + } + + if (generated.length === beginIndex && Number.isFinite(maxInitialTimestampIndex)) { + const lastAllowed = timestampBeginTokenId + maxInitialTimestampIndex; + for (let i = lastAllowed + 1; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + + const textUpper = Math.min(timestampBeginTokenId, logits.length); + if (textUpper <= 0 || textUpper >= logits.length) return; + + const logprobs = logSoftmax(logits); + + let maxTextTokenLogprob = Number.NEGATIVE_INFINITY; + for (let i = 0; i < textUpper; i += 1) { + if (logprobs[i] > maxTextTokenLogprob) maxTextTokenLogprob = logprobs[i]; + } + + let timestampProbMass = 0; + for (let i = textUpper; i < logprobs.length; i += 1) { + timestampProbMass += Math.exp(logprobs[i]); + } + const timestampLogprob = timestampProbMass > 0 ? Math.log(timestampProbMass) : Number.NEGATIVE_INFINITY; + + if (timestampLogprob > maxTextTokenLogprob) { + for (let i = 0; i < textUpper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } +} diff --git a/compute/core/src/whisper/model.ts b/packages/compute-worker/src/inference/whisper/model.ts similarity index 99% rename from compute/core/src/whisper/model.ts rename to packages/compute-worker/src/inference/whisper/model.ts index 4c5a021..9cf0c4f 100644 --- a/compute/core/src/whisper/model.ts +++ b/packages/compute-worker/src/inference/whisper/model.ts @@ -3,7 +3,7 @@ import { fileURLToPath } from 'url'; import { createHash } from 'crypto'; import { readFileSync } from 'fs'; import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises'; -import { DOCSTORE_DIR } from '../platform/docstore'; +import { DOCSTORE_DIR } from '../../infrastructure/platform'; const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped'); diff --git a/compute/core/src/whisper/token-timestamps.ts b/packages/compute-worker/src/inference/whisper/timestamps.ts similarity index 93% rename from compute/core/src/whisper/token-timestamps.ts rename to packages/compute-worker/src/inference/whisper/timestamps.ts index 47edc1d..e6e2b96 100644 --- a/compute/core/src/whisper/token-timestamps.ts +++ b/packages/compute-worker/src/inference/whisper/timestamps.ts @@ -1,5 +1,6 @@ import type { Tokenizer } from '@huggingface/tokenizers'; import type * as ort from 'onnxruntime-node'; +import type { TTSSentenceAlignment, TTSSentenceWord } from '../../api/types'; const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E'; const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu'); @@ -12,6 +13,29 @@ export interface WhisperWordTiming { endSec: number; } +export interface WhisperWord { + start: number; + end: number; + word: string; +} + +export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { + const lowerSentence = sentence.toLowerCase(); + let cursor = 0; + const alignedWords: TTSSentenceWord[] = words.map((word) => { + const token = word.word.trim(); + if (!token) { + return { text: '', startSec: word.start, endSec: word.end, charStart: cursor, charEnd: cursor }; + } + const index = lowerSentence.indexOf(token.toLowerCase(), cursor); + const start = index >= 0 ? index : cursor; + const end = Math.min(sentence.length, start + token.length); + cursor = Math.max(cursor, end); + return { text: token, startSec: word.start, endSec: word.end, charStart: start, charEnd: end }; + }).filter((word) => word.text.length > 0); + return { sentence, sentenceIndex: 0, words: alignedWords }; +} + function medianFilter(data: Float32Array, windowSize: number): Float32Array { if (windowSize % 2 === 0 || windowSize <= 0) { throw new Error('Window size must be a positive odd number'); diff --git a/compute/core/src/config/timeout.ts b/packages/compute-worker/src/infrastructure/config.ts similarity index 68% rename from compute/core/src/config/timeout.ts rename to packages/compute-worker/src/infrastructure/config.ts index 40fd406..791ac4e 100644 --- a/compute/core/src/config/timeout.ts +++ b/packages/compute-worker/src/infrastructure/config.ts @@ -19,7 +19,13 @@ export type IdleTimeoutAndHardCapInput = { label: string; }; -function readPositiveIntEnv(name: string, fallback: number): number { +export function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +export function readPositiveIntEnv(name: string, fallback: number): number { const raw = process.env[name]?.trim(); if (!raw) return fallback; const parsed = Number(raw); @@ -27,6 +33,34 @@ function readPositiveIntEnv(name: string, fallback: number): number { return Math.floor(parsed); } +export function readBoolEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + return ['1', 'true', 'yes', 'on'].includes(raw.toLowerCase()); +} + +export function normalizeNatsReplicas(value: number): number { + return value === 3 || value === 5 ? value : 1; +} + +export function buildLoggerConfig(): boolean | Record { + const format = process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'; + const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; + if (format === 'json') return { level, base: null }; + return { + level, + base: null, + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + }, + }; +} + let timeoutConfigCache: ComputeTimeoutConfig | null = null; let opStaleMsCache: number | null = null; @@ -117,3 +151,23 @@ export async function withIdleTimeoutAndHardCap(input: IdleTimeoutAndHardCapI throw error; } } + +export function getComputeJobConcurrency(): number { + return readPositiveIntEnv('COMPUTE_JOB_CONCURRENCY', 1); +} + +export function getAvailableCpuCores(): number { + if (typeof os.availableParallelism === 'function') { + const value = os.availableParallelism(); + if (Number.isFinite(value) && value >= 1) return Math.floor(value); + } + const fallback = os.cpus().length; + return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1; +} + +export function getOnnxThreadsPerJob(): number { + const concurrency = getComputeJobConcurrency(); + const usableCores = Math.max(1, getAvailableCpuCores() - 1); + return Math.max(1, Math.floor(usableCores / concurrency)); +} +import os from 'node:os'; diff --git a/compute/worker/src/control-plane/json-codec.ts b/packages/compute-worker/src/infrastructure/json-codec.ts similarity index 100% rename from compute/worker/src/control-plane/json-codec.ts rename to packages/compute-worker/src/infrastructure/json-codec.ts diff --git a/compute/worker/src/control-plane/jetstream.ts b/packages/compute-worker/src/infrastructure/nats-adapters.ts similarity index 99% rename from compute/worker/src/control-plane/jetstream.ts rename to packages/compute-worker/src/infrastructure/nats-adapters.ts index 9225d96..d6e4ddb 100644 --- a/compute/worker/src/control-plane/jetstream.ts +++ b/packages/compute-worker/src/infrastructure/nats-adapters.ts @@ -8,12 +8,12 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '@openreader/compute-core/control-plane'; +} from '../operations'; import type { PdfLayoutJobRequest, WhisperAlignJobRequest, WorkerOperationKind, -} from '@openreader/compute-core/api-contracts'; +} from '../operations/contracts'; import { createJsonCodec } from './json-codec'; export interface KvEntryLike { diff --git a/packages/compute-worker/src/infrastructure/nats-session.ts b/packages/compute-worker/src/infrastructure/nats-session.ts new file mode 100644 index 0000000..356a0de --- /dev/null +++ b/packages/compute-worker/src/infrastructure/nats-session.ts @@ -0,0 +1,204 @@ +import type { Consumer, JetStreamClient, JetStreamManager } from '@nats-io/jetstream'; +import { jetstream, jetstreamManager } from '@nats-io/jetstream'; +import { Kvm } from '@nats-io/kv'; +import { connect, type ConnectionOptions, type NatsConnection } from '@nats-io/transport-node'; +import { + COMPUTE_STATE_BUCKET, + COMPUTE_STATE_TTL_MS, + JOBS_STREAM_NAME, + LAYOUT_CONSUMER_NAME, + NATS_API_TIMEOUT_MS, + WHISPER_CONSUMER_NAME, + ensureJetStreamResources, +} from './nats'; + +const IDLE_DISCONNECT_MS = 120_000; +const IDLE_CHECK_INTERVAL_MS = 5_000; +const IDLE_STATUS_LOG_INTERVAL_MS = 60_000; +const ORPHAN_SWEEP_INTERVAL_MS = 15_000; + +export interface NatsSession { + nc: NatsConnection; + js: JetStreamClient; + jsm: JetStreamManager; + kv: Awaited>; + whisperConsumer: Consumer; + layoutConsumer: Consumer; +} + +interface NatsSessionLogger { + info(data: unknown, message?: string): void; + info(message: string): void; + error(data: unknown, message?: string): void; +} + +export interface NatsActivitySnapshot { + activeSse: number; + inFlightHttp: number; + inFlightJobs: number; + lastActivityAt: number; + lastActivityReason: string; +} + +function toErrorMessage(error: unknown): string { + return error instanceof Error && error.message ? error.message : String(error); +} + +export function createNatsSessionManager(input: { + connectOptions: ConnectionOptions; + logger: NatsSessionLogger; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + pdfAttempts: number; + jobsStreamMaxBytes: number; + eventsStreamMaxBytes: number; + jobStatesMaxBytes: number; + natsReplicas: number; + isStopping: () => boolean; + getActivity: () => NatsActivitySnapshot; + markActivity: (reason: string) => void; + startWorkers: (session: NatsSession) => void; + stopWorkers: () => Promise; + runReconciliation: () => Promise; +}) { + let session: NatsSession | null = null; + let connecting: Promise | null = null; + let idleTimer: NodeJS.Timeout | null = null; + let orphanSweepTimer: NodeJS.Timeout | null = null; + let lastIdleStatusLogAt = 0; + let generation = -1; + + const clearTimers = () => { + if (idleTimer) clearInterval(idleTimer); + if (orphanSweepTimer) clearInterval(orphanSweepTimer); + idleTimer = null; + orphanSweepTimer = null; + }; + + const disconnect = async (reason: string): Promise => { + const current = session; + if (!current) return; + const activity = input.getActivity(); + input.logger.info({ + reason, + activeSse: activity.activeSse, + inFlightHttp: activity.inFlightHttp, + inFlightJobs: activity.inFlightJobs, + idleForMs: Date.now() - activity.lastActivityAt, + }, 'nats dropping connection'); + session = null; + clearTimers(); + try { + await current.nc.close(); + } catch { + // Closing an already-closed session is harmless. + } + await input.stopWorkers(); + input.logger.info({ reason }, 'nats disconnected'); + }; + + const startTimers = () => { + if (!idleTimer) { + idleTimer = setInterval(() => { + if (!session || input.isStopping()) return; + const activity = input.getActivity(); + const now = Date.now(); + const idleForMs = now - activity.lastActivityAt; + if (now - lastIdleStatusLogAt >= IDLE_STATUS_LOG_INTERVAL_MS) { + lastIdleStatusLogAt = now; + input.logger.info({ + ...activity, + idleForMs, + disconnectEligible: activity.inFlightHttp === 0 + && activity.inFlightJobs === 0 + && idleForMs >= IDLE_DISCONNECT_MS, + }, 'nats idle status'); + } + if (activity.inFlightHttp > 0 || activity.inFlightJobs > 0 || idleForMs < IDLE_DISCONNECT_MS) return; + void disconnect('idle'); + }, IDLE_CHECK_INTERVAL_MS); + idleTimer.unref?.(); + } + if (!orphanSweepTimer) { + orphanSweepTimer = setInterval(() => { + if (!session || input.isStopping()) return; + void input.runReconciliation().catch((error) => { + input.logger.error({ error: toErrorMessage(error) }, 'periodic orphaned operation recovery failed'); + }); + }, ORPHAN_SWEEP_INTERVAL_MS); + orphanSweepTimer.unref?.(); + } + }; + + const ensureConnected = async (): Promise => { + if (session) return session; + if (connecting) return connecting; + connecting = (async () => { + const nc = await connect(input.connectOptions); + const js = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); + const jsm = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); + await ensureJetStreamResources({ + jsm, + whisperTimeoutMs: input.whisperTimeoutMs, + pdfTimeoutMs: input.pdfTimeoutMs, + pdfAttempts: input.pdfAttempts, + jobsMaxBytes: input.jobsStreamMaxBytes, + eventsMaxBytes: input.eventsStreamMaxBytes, + natsReplicas: input.natsReplicas, + }); + const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { + replicas: input.natsReplicas, + history: 1, + ttl: COMPUTE_STATE_TTL_MS, + max_bytes: input.jobStatesMaxBytes, + }); + const next: NatsSession = { + nc, + js, + jsm, + kv, + whisperConsumer: await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME), + layoutConsumer: await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME), + }; + session = next; + generation += 1; + input.markActivity('nats_connected'); + input.startWorkers(next); + startTimers(); + void nc.closed().then(() => { + if (session?.nc === nc) session = null; + }); + input.logger.info('nats connected'); + return next; + })(); + try { + return await connecting; + } finally { + connecting = null; + } + }; + + return { + ensureConnected, + disconnect, + isConnected: () => session !== null, + isOwnerActive: (owner: object) => session === owner, + getGeneration: () => generation, + async close(): Promise { + clearTimers(); + await input.stopWorkers(); + const current = session; + session = null; + if (!current) return; + try { + await current.nc.drain(); + } catch { + try { + await current.nc.close(); + } catch { + // Closing an already-closed session is harmless. + } + } + }, + }; +} diff --git a/packages/compute-worker/src/infrastructure/nats.ts b/packages/compute-worker/src/infrastructure/nats.ts new file mode 100644 index 0000000..429a03d --- /dev/null +++ b/packages/compute-worker/src/infrastructure/nats.ts @@ -0,0 +1,104 @@ +import { + AckPolicy, + DeliverPolicy, + ReplayPolicy, + RetentionPolicy, + StorageType, + type JetStreamManager, +} from '@nats-io/jetstream'; +import { nanos } from '@nats-io/transport-node'; +import { OP_EVENTS_SUBJECT_WILDCARD } from './nats-adapters'; + +export const JOBS_STREAM_NAME = 'compute_jobs'; +export const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; +export const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; +export const WHISPER_CONSUMER_NAME = 'compute_whisper'; +export const LAYOUT_CONSUMER_NAME = 'compute_layout'; +export const EVENTS_STREAM_NAME = 'compute_events'; +export const COMPUTE_STATE_BUCKET = 'compute_state'; +export const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; +export const NATS_API_TIMEOUT_MS = 60_000; + +const WHISPER_MAX_DELIVER = 1; + +function isAlreadyExistsError(error: unknown): boolean { + const message = (error instanceof Error ? error.message : String(error)).toLowerCase(); + return message.includes('already in use') || message.includes('already exists'); +} + +export async function ensureJetStreamResources(input: { + jsm: JetStreamManager; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + pdfAttempts: number; + jobsMaxBytes: number; + eventsMaxBytes: number; + natsReplicas: number; +}): Promise { + const streamConfig = { + name: JOBS_STREAM_NAME, + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + retention: RetentionPolicy.Workqueue, + storage: StorageType.File, + max_bytes: input.jobsMaxBytes, + num_replicas: input.natsReplicas, + }; + try { + await input.jsm.streams.add(streamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await input.jsm.streams.update(JOBS_STREAM_NAME, { + subjects: streamConfig.subjects, + max_bytes: input.jobsMaxBytes, + num_replicas: input.natsReplicas, + }); + } + + const eventsStreamConfig = { + name: EVENTS_STREAM_NAME, + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + retention: RetentionPolicy.Limits, + storage: StorageType.File, + max_bytes: input.eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: input.natsReplicas, + }; + try { + await input.jsm.streams.add(eventsStreamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await input.jsm.streams.update(EVENTS_STREAM_NAME, { + subjects: eventsStreamConfig.subjects, + max_bytes: input.eventsMaxBytes, + max_age: eventsStreamConfig.max_age, + num_replicas: input.natsReplicas, + }); + } + + const ensureConsumer = async (name: string, subject: string, ackWaitMs: number, maxDeliver: number) => { + const config = { + durable_name: name, + ack_policy: AckPolicy.Explicit, + deliver_policy: DeliverPolicy.All, + replay_policy: ReplayPolicy.Instant, + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }; + try { + await input.jsm.consumers.add(JOBS_STREAM_NAME, config); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await input.jsm.consumers.update(JOBS_STREAM_NAME, name, { + filter_subject: subject, + ack_wait: config.ack_wait, + max_deliver: maxDeliver, + }); + } + }; + + await Promise.all([ + ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, input.whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), + ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, input.pdfTimeoutMs + 15_000, input.pdfAttempts), + ]); +} diff --git a/packages/compute-worker/src/infrastructure/platform.ts b/packages/compute-worker/src/infrastructure/platform.ts new file mode 100644 index 0000000..3752a17 --- /dev/null +++ b/packages/compute-worker/src/infrastructure/platform.ts @@ -0,0 +1,68 @@ +import fs from 'fs'; +import path from 'path'; +import ffmpegStatic from 'ffmpeg-static'; + +function findMonorepoRoot(startDir: string): string | null { + let current = path.resolve(startDir); + for (;;) { + const marker = path.join(current, 'pnpm-workspace.yaml'); + if (fs.existsSync(marker)) return current; + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} + +function resolveDocstoreDir(): string { + // 1. Try to find the monorepo root (works in local development) + const repoRoot = findMonorepoRoot(process.cwd()); + if (repoRoot) return path.join(repoRoot, 'docstore'); + + // 2. In a containerized environment, the parent bootstrap process runs in /app + // and the child process inherits process.env.PWD = '/app'. + if (process.env.PWD) { + const pwdDocstore = path.join(process.env.PWD, 'docstore'); + if (fs.existsSync(pwdDocstore)) return pwdDocstore; + } + + // 3. Fallback to the standard container app docstore path if it exists + if (fs.existsSync('/app/docstore')) { + return '/app/docstore'; + } + + // 4. Fallback to the process cwd docstore + return path.join(process.cwd(), 'docstore'); +} + +export const DOCSTORE_DIR = resolveDocstoreDir(); + +function normalizePath(value: unknown): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string { + if (envValue) { + if ((envValue.includes('/') || envValue.includes('\\')) && !fs.existsSync(envValue)) { + throw new Error(`${envVarName} points to a missing binary: ${envValue}`); + } + return envValue; + } + if (!bundledValue) { + throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`); + } + if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !fs.existsSync(bundledValue)) { + throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`); + } + return bundledValue; +} + +export function getFFmpegPath(): string { + return resolveBinary( + normalizePath(process.env.FFMPEG_BIN), + normalizePath(ffmpegStatic), + 'FFMPEG_BIN', + 'ffmpeg-static', + ); +} diff --git a/packages/compute-worker/src/infrastructure/storage.ts b/packages/compute-worker/src/infrastructure/storage.ts new file mode 100644 index 0000000..1190da3 --- /dev/null +++ b/packages/compute-worker/src/infrastructure/storage.ts @@ -0,0 +1,127 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { parsedPdfArtifactKey } from '../storage/artifact-addressing'; + +export interface ArtifactStorage { + readObject(key: string): Promise; + objectExists(key: string): Promise; + deleteObject(key: string): Promise; + putParsedPdf(documentId: string, namespace: string | null, parsed: unknown): Promise; +} + +export interface ArtifactStorageConfig { + bucket: string; + prefix: string; + client: S3Client; +} + +function bodyToBuffer(body: unknown): Promise | Buffer { + if (!body) return Buffer.alloc(0); + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return maybe.transformToByteArray().then((bytes) => Buffer.from(bytes)); + } + } + if (typeof body === 'object' && body !== null && 'on' in body) { + return (async () => { + const chunks: Buffer[] = []; + for await (const chunk of body as NodeJS.ReadableStream) { + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); + } + return Buffer.concat(chunks); + })(); + } + throw new Error('Unsupported S3 response body type'); +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function isNotFound(error: unknown): boolean { + const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; + return maybe.$metadata?.httpStatusCode === 404 + || maybe.name === 'NotFound' + || maybe.name === 'NoSuchKey' + || maybe.Code === 'NotFound' + || maybe.Code === 'NoSuchKey'; +} + +export function normalizeS3Prefix(prefix: string | undefined): string { + const value = (prefix || 'openreader').trim(); + return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; +} + +export function createS3ClientFromEnv(requireEnv: (name: string) => string): S3Client { + return new S3Client({ + region: requireEnv('S3_REGION'), + endpoint: process.env.S3_ENDPOINT?.trim() || undefined, + forcePathStyle: ['1', 'true', 'yes', 'on'].includes(process.env.S3_FORCE_PATH_STYLE?.trim().toLowerCase() ?? ''), + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId: requireEnv('S3_ACCESS_KEY_ID'), + secretAccessKey: requireEnv('S3_SECRET_ACCESS_KEY'), + }, + }); +} + +export function createArtifactStorage(config: ArtifactStorageConfig): ArtifactStorage { + const safeKey = (key: string): string => { + const trimmed = key.trim(); + if (!trimmed.startsWith(`${config.prefix}/`)) throw new Error('Object key prefix mismatch'); + return trimmed; + }; + + return { + async readObject(key) { + const response = await config.client.send(new GetObjectCommand({ + Bucket: config.bucket, + Key: safeKey(key), + })); + return toArrayBuffer(new Uint8Array(await bodyToBuffer(response.Body))); + }, + async objectExists(key) { + try { + await config.client.send(new HeadObjectCommand({ + Bucket: config.bucket, + Key: safeKey(key), + })); + return true; + } catch (error) { + if (isNotFound(error)) return false; + throw error; + } + }, + async deleteObject(key) { + await config.client.send(new DeleteObjectCommand({ + Bucket: config.bucket, + Key: safeKey(key), + })); + }, + async putParsedPdf(documentId, namespace, parsed) { + const key = parsedPdfArtifactKey({ documentId, namespace, prefix: config.prefix }); + await config.client.send(new PutObjectCommand({ + Bucket: config.bucket, + Key: key, + Body: Buffer.from(JSON.stringify(parsed)), + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + })); + return key; + }, + }; +} diff --git a/packages/compute-worker/src/jobs/handlers.ts b/packages/compute-worker/src/jobs/handlers.ts new file mode 100644 index 0000000..0f64c24 --- /dev/null +++ b/packages/compute-worker/src/jobs/handlers.ts @@ -0,0 +1,126 @@ +import { z } from 'zod'; +import { + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, +} from '../inference/runtime'; +import { withIdleTimeoutAndHardCap, withTimeout } from '../infrastructure/config'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + PdfLayoutProgress, + WhisperAlignJobRequest, + WhisperAlignJobResult, +} from '../operations/contracts'; +import type { ArtifactStorage } from '../infrastructure/storage'; +import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; +import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; + +const whisperRequestSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +const pdfRequestSchema = z.object({ + documentId: z.string().trim().min(1), + namespace: z.string().trim().min(1).max(128).nullable(), + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +export interface JobHandlers { + runWhisper(payload: WhisperAlignJobRequest, queueWaitMs: number): Promise; + runPdfLayout( + payload: PdfLayoutJobRequest, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ): Promise; +} + +export function createJobHandlers(input: { + storage: ArtifactStorage; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + pdfHardCapMs: number; +}): JobHandlers { + return { + async runWhisper(payload, queueWaitMs) { + const parsed = whisperRequestSchema.parse(payload); + const s3FetchStartedAt = Date.now(); + const audioBuffer = await withTimeout( + input.storage.readObject(parsed.audioObjectKey), + input.whisperTimeoutMs, + 'whisper s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + const computeStartedAt = Date.now(); + const result = await withTimeout( + runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: parsed.text, + cacheKey: parsed.cacheKey, + lang: parsed.lang, + }), + input.whisperTimeoutMs, + 'whisper alignment job', + ); + return { + ...result, + timing: { queueWaitMs, s3FetchMs, computeMs: Date.now() - computeStartedAt }, + }; + }, + + async runPdfLayout(payload, queueWaitMs, hooks) { + const parsed = pdfRequestSchema.parse(payload); + const s3FetchStartedAt = Date.now(); + const pdfBytes = await withTimeout( + input.storage.readObject(parsed.documentObjectKey), + Math.max(input.pdfTimeoutMs, 1_000), + 'pdf s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + let lastTotalPages = 0; + let lastPagesParsed = 0; + const computeStartedAt = Date.now(); + const result = await withIdleTimeoutAndHardCap({ + idleTimeoutMs: Math.max(input.pdfTimeoutMs, 1_000), + hardCapMs: input.pdfHardCapMs, + label: 'pdf layout job', + run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ + documentId: parsed.documentId, + pdfBytes, + onPageStarted: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + await hooks?.onProgress?.(buildInferProgressForPageStart({ pageNumber, totalPages })); + }, + onPageParsed: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + lastPagesParsed = pageNumber; + await hooks?.onProgress?.(buildInferProgressForPageParsed({ pageNumber, totalPages })); + }, + }), + }); + const computeMs = Date.now() - computeStartedAt; + if (hooks?.onProgress && lastTotalPages > 0) { + await hooks.onProgress({ + totalPages: lastTotalPages, + pagesParsed: lastPagesParsed, + currentPage: lastPagesParsed || undefined, + phase: 'merge', + }); + } + const parsedObjectKey = await persistParsedPdfWhileSourceExists({ + sourceObjectKey: parsed.documentObjectKey, + sourceExists: input.storage.objectExists, + putParsedObject: () => input.storage.putParsedPdf(parsed.documentId, parsed.namespace, result.parsed), + deleteParsedObject: input.storage.deleteObject, + }); + return { + parsedObjectKey, + timing: { queueWaitMs, s3FetchMs, computeMs }, + }; + }, + }; +} diff --git a/compute/worker/src/pdf-artifact-persistence.ts b/packages/compute-worker/src/jobs/pdf-artifact-persistence.ts similarity index 100% rename from compute/worker/src/pdf-artifact-persistence.ts rename to packages/compute-worker/src/jobs/pdf-artifact-persistence.ts diff --git a/compute/worker/src/pdf-progress.ts b/packages/compute-worker/src/jobs/pdf-progress.ts similarity index 87% rename from compute/worker/src/pdf-progress.ts rename to packages/compute-worker/src/jobs/pdf-progress.ts index 756a256..acc3867 100644 --- a/compute/worker/src/pdf-progress.ts +++ b/packages/compute-worker/src/jobs/pdf-progress.ts @@ -1,4 +1,4 @@ -import type { PdfLayoutProgress } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutProgress } from '../operations/contracts'; export function buildInferProgressForPageStart(input: { pageNumber: number; diff --git a/compute/worker/src/worker-loop-policy.ts b/packages/compute-worker/src/jobs/worker-loop-policy.ts similarity index 89% rename from compute/worker/src/worker-loop-policy.ts rename to packages/compute-worker/src/jobs/worker-loop-policy.ts index 7462b4c..c5f9c4e 100644 --- a/compute/worker/src/worker-loop-policy.ts +++ b/packages/compute-worker/src/jobs/worker-loop-policy.ts @@ -1,4 +1,4 @@ -import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts'; +import type { WorkerOperationKind } from '../operations/contracts'; export type RetryAction = 'nak_retry' | 'term_fail'; diff --git a/packages/compute-worker/src/jobs/worker-loop.ts b/packages/compute-worker/src/jobs/worker-loop.ts new file mode 100644 index 0000000..27c5afe --- /dev/null +++ b/packages/compute-worker/src/jobs/worker-loop.ts @@ -0,0 +1,327 @@ +import type { Consumer, JsMsg } from '@nats-io/jetstream'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + PdfLayoutProgress, + WhisperAlignJobRequest, + WhisperAlignJobResult, + WorkerJobTiming, + WorkerOperationKind, +} from '../operations/contracts'; +import type { JsonCodec } from '../infrastructure/json-codec'; +import type { JobHandlers } from './handlers'; +import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; + +const LOOP_ERROR_BACKOFF_MS = 500; +const RUNNING_HEARTBEAT_MS = 5000; +const PULL_EXPIRES_MS = 5_000; +const WHISPER_MAX_DELIVER = 1; +const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { + whisper_align: 15_000, + pdf_layout: 120_000, +}; + +export interface QueuedJob { + jobId: string; + opId: string; + opKey: string; + kind: WorkerOperationKind; + queuedAt: number; + payload: TPayload; +} + +export interface WorkerLoopOrchestrator { + markRunning(input: { opId: string; startedAt?: number; updatedAt?: number; timing?: WorkerJobTiming }): Promise; + markProgress(input: { + opId: string; + progress: PdfLayoutProgress; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { opId: string; result: unknown; updatedAt?: number; timing?: WorkerJobTiming }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +export interface WorkerLogger { + info(data: unknown, message?: string): void; + warn(data: unknown, message?: string): void; + error(data: unknown, message?: string): void; +} + +class ConcurrencyGate { + private inFlight = 0; + private readonly queue: Array<() => void> = []; + + constructor(private readonly limit: number) {} + + async acquire(): Promise { + if (this.inFlight < this.limit) { + this.inFlight += 1; + return; + } + await new Promise((resolve) => { + this.queue.push(() => { + this.inFlight += 1; + resolve(); + }); + }); + } + + release(): void { + this.inFlight = Math.max(0, this.inFlight - 1); + this.queue.shift()?.(); + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function toErrorMessage(error: unknown): string { + return error instanceof Error && error.message ? error.message : String(error); +} + +function safeDurationMs(start: number, end: number): number { + return Math.max(0, Math.floor(end - start)); +} + +function extractTiming(result: unknown): WorkerJobTiming | undefined { + if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; + return (result as { timing?: WorkerJobTiming }).timing; +} + +function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { + if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; + const maybe = result as { parsedObjectKey?: unknown }; + return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; +} + +export function createWorkerLoopController(input: { + orchestrator: WorkerLoopOrchestrator; + handlers: JobHandlers; + logger: WorkerLogger; + jobConcurrency: number; + pdfAttempts: number; + whisperCodec: JsonCodec>; + pdfCodec: JsonCodec>; + isOwnerActive: (owner: object) => boolean; + isStopping: () => boolean; + markActivity: (reason: string) => void; + onInFlightJobsChanged: (delta: number) => void; +}) { + const gate = new ConcurrencyGate(Math.max(1, Math.floor(input.jobConcurrency))); + let loops: Promise[] = []; + let stopRequested = false; + + type Context = { + decoded: QueuedJob; + workerLabel: string; + startedAt: number; + queueWaitTiming?: { queueWaitMs: number }; + latestProgress?: PdfLayoutProgress; + }; + + type JobRunner = ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + + type WorkDefinition = { + codec: JsonCodec>; + run: JobRunner; + }; + + const markRunning = async (context: Context, updatedAt: number): Promise => { + if (context.latestProgress) { + await input.orchestrator.markProgress({ + opId: context.decoded.opId, + progress: context.latestProgress, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + return; + } + await input.orchestrator.markRunning({ + opId: context.decoded.opId, + startedAt: context.startedAt, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + }; + + const processMessage = async (work: WorkDefinition & { + msg: JsMsg; + workerLabel: string; + }): Promise => { + let context: Context | null = null; + let heartbeat: NodeJS.Timeout | null = null; + try { + const decoded = work.codec.decode(work.msg.data); + const startedAt = Date.now(); + context = { + decoded, + workerLabel: work.workerLabel, + startedAt, + queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), + }; + await markRunning(context, startedAt); + input.logger.info({ + worker: work.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, + deliveryCount: work.msg.info.deliveryCount, + }, 'job.started'); + heartbeat = setInterval(() => { + void markRunning(context!, Date.now()).catch((error) => { + input.logger.error({ + worker: work.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(error), + }, 'failed to persist operation heartbeat state'); + }); + }, RUNNING_HEARTBEAT_MS); + const result = await work.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { + onProgress: async (progress) => { + try { + work.msg.working(); + } catch (error) { + input.logger.warn({ + worker: work.workerLabel, + kind: context?.decoded.kind, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(error), + }, 'failed to extend JetStream ack wait on progress'); + } + context!.latestProgress = progress; + await markRunning(context!, Date.now()); + }, + }); + const timing = extractTiming(result); + const now = Date.now(); + await input.orchestrator.markSucceeded({ + opId: decoded.opId, + result, + updatedAt: now, + ...(timing ? { timing } : {}), + }); + work.msg.ack(); + const durationMs = safeDurationMs(startedAt, now); + if (durationMs >= SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]) { + input.logger.info({ worker: work.workerLabel, kind: decoded.kind, opId: decoded.opId, jobId: decoded.jobId, durationMs, timing: timing ?? null }, 'job.stage'); + } + input.logger.info({ + worker: work.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + status: 'succeeded', + durationMs, + resultRef: extractResultRef(decoded.kind, result), + timing: timing ?? null, + }, 'job.terminal'); + } catch (error) { + const errorMessage = toErrorMessage(error); + const deliveryCount = work.msg.info.deliveryCount; + const kind = context?.decoded.kind ?? 'pdf_layout'; + const action = decideRetryAction({ kind, deliveryCount, pdfAttempts: input.pdfAttempts, whisperMaxDeliver: WHISPER_MAX_DELIVER }); + const timing = context ? buildQueueWaitTiming(context.decoded.queuedAt, Date.now()) : undefined; + if (context) { + const update = action === 'nak_retry' + ? markRunning(context, Date.now()) + : input.orchestrator.markFailed({ + opId: context.decoded.opId, + error: { message: errorMessage }, + updatedAt: Date.now(), + ...(timing ? { timing } : {}), + }); + await update.catch((stateError) => input.logger.error({ + worker: context?.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation state')); + } + if (action === 'nak_retry') work.msg.nak(); + else work.msg.term(errorMessage); + input.logger.error({ + worker: context?.workerLabel, + kind: context?.decoded.kind, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + status: action === 'nak_retry' ? 'running' : 'failed', + error: errorMessage, + deliveryCount, + retryAction: action === 'nak_retry' ? 'nack_retry' : 'term', + }, 'job.terminal'); + } finally { + if (heartbeat) clearInterval(heartbeat); + } + }; + + const runLoop = async (work: WorkDefinition & { + owner: object; + consumer: Consumer; + workerLabel: string; + }): Promise => { + const detached = () => input.isStopping() || stopRequested || !input.isOwnerActive(work.owner); + while (!detached()) { + let msg: JsMsg | null = null; + try { + try { + msg = await work.consumer.next({ expires: PULL_EXPIRES_MS }); + } catch (error) { + if (detached()) return; + input.logger.error({ error: toErrorMessage(error), worker: work.workerLabel }, 'worker pull failed'); + await sleep(LOOP_ERROR_BACKOFF_MS); + continue; + } + if (!msg) continue; + input.markActivity(`job_received:${work.workerLabel}`); + input.onInFlightJobsChanged(1); + await gate.acquire(); + if (detached()) return; + await processMessage({ ...work, msg }); + } finally { + if (msg) { + gate.release(); + input.onInFlightJobsChanged(-1); + input.markActivity(`job_completed:${work.workerLabel}`); + } + } + } + }; + + return { + start(owner: object, consumers: { whisper: Consumer; pdfLayout: Consumer }): void { + stopRequested = false; + loops = []; + const whisperWork: WorkDefinition = { + codec: input.whisperCodec, + run: input.handlers.runWhisper, + }; + const pdfWork: WorkDefinition = { + codec: input.pdfCodec, + run: input.handlers.runPdfLayout, + }; + for (let i = 0; i < input.jobConcurrency; i += 1) { + loops.push(runLoop({ owner, consumer: consumers.whisper, ...whisperWork, workerLabel: `whisper-${i + 1}` })); + loops.push(runLoop({ owner, consumer: consumers.pdfLayout, ...pdfWork, workerLabel: `layout-${i + 1}` })); + } + }, + async stop(): Promise { + stopRequested = true; + await Promise.allSettled(loops); + loops = []; + }, + }; +} diff --git a/compute/core/src/api-contracts/index.ts b/packages/compute-worker/src/operations/contracts.ts similarity index 87% rename from compute/core/src/api-contracts/index.ts rename to packages/compute-worker/src/operations/contracts.ts index 487a618..5180972 100644 --- a/compute/core/src/api-contracts/index.ts +++ b/packages/compute-worker/src/operations/contracts.ts @@ -1,24 +1,26 @@ -import type { TTSSentenceAlignment } from '../types/tts'; -import type { ParsedPdfDocument } from '../types/parsed-pdf'; +import type { TTSSentenceAlignment, ParsedPdfDocument } from '../api/types'; export type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from '../types/tts'; +} from '../api/types'; export type { ParsedPdfBlockKind, ParsedPdfBlockFragment, ParsedPdfBlock, ParsedPdfPage, ParsedPdfDocument, -} from '../types/parsed-pdf'; +} from '../api/types'; export const ALIGN_QUEUE_NAME = 'whisper-align'; export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout'; -export { PDF_PARSER_VERSION } from '../pdf/parser-version'; -export { encodeParserVersion } from '../pdf/parser-version-key'; +export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69'; + +export function encodeParserVersion(parserVersion: string, defaultVersion = PDF_PARSER_VERSION): string { + return encodeURIComponent(parserVersion.trim() || defaultVersion); +} export interface WhisperAlignJobBase { text: string; diff --git a/compute/core/src/control-plane/index.ts b/packages/compute-worker/src/operations/index.ts similarity index 71% rename from compute/core/src/control-plane/index.ts rename to packages/compute-worker/src/operations/index.ts index 5b8255d..35699ba 100644 --- a/compute/core/src/control-plane/index.ts +++ b/packages/compute-worker/src/operations/index.ts @@ -1,4 +1,4 @@ export * from './types'; export * from './state-machine'; -export * from './orchestrator'; +export * from './service'; export * from './sse'; diff --git a/packages/compute-worker/src/operations/keys.ts b/packages/compute-worker/src/operations/keys.ts new file mode 100644 index 0000000..02fc2c8 --- /dev/null +++ b/packages/compute-worker/src/operations/keys.ts @@ -0,0 +1,56 @@ +import { createHash } from 'node:crypto'; +import { PDF_PARSER_VERSION } from './contracts'; + +function sha256Hex(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +export function buildWhisperOperationKey(input: { + audioObjectKey: string; + text: string; + lang?: string; + cacheKey?: string; +}): string { + const cacheKey = input.cacheKey?.trim(); + if (cacheKey) { + return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`; + } + return [ + 'whisper_align', + 'v1', + input.audioObjectKey, + input.lang ?? '', + sha256Hex(input.text), + ].join('|'); +} + +export function buildPdfOperationKey(input: { + documentId: string; + namespace: string | null; + documentObjectKey: string; + replaceToken?: string; +}, parserVersion = PDF_PARSER_VERSION): string { + return [ + 'pdf_layout', + 'v1', + parserVersion, + input.documentId, + input.namespace ?? '', + input.documentObjectKey, + input.replaceToken?.trim() || '', + ].join('|'); +} + +export function pdfSubjectFromOperationKey(opKey: string): { + kind: 'pdf_layout'; + documentId: string; + namespace: string | null; +} | null { + const [kind, version, , documentId, namespace] = opKey.split('|'); + if (kind !== 'pdf_layout' || version !== 'v1' || !documentId) return null; + return { + kind: 'pdf_layout', + documentId, + namespace: namespace || null, + }; +} diff --git a/packages/compute-worker/src/operations/reconciliation.ts b/packages/compute-worker/src/operations/reconciliation.ts new file mode 100644 index 0000000..921587f --- /dev/null +++ b/packages/compute-worker/src/operations/reconciliation.ts @@ -0,0 +1,95 @@ +import type { WorkerJobTiming } from './contracts'; +import { + recoverOrphanedOperations, + type StreamedOperationState, +} from './recovery'; + +export interface ReconciliationStateStore { + getOpState(opId: string): Promise; + getOpStateRecord?(opId: string): Promise<{ state: StreamedOperationState; revision: number } | null>; + getOpIndex?(opKey: string): Promise<{ opId: string } | null>; + listOpStates?(): Promise; +} + +export interface ReconciliationOrchestrator { + markFailedIfUnchanged?(input: { + current: StreamedOperationState; + expectedRevision: number; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +interface ReconciliationLogger { + warn(data: unknown, message?: string): void; +} + +function supportsRecovery( + stateStore: ReconciliationStateStore, + orchestrator: ReconciliationOrchestrator, +): boolean { + return typeof stateStore.listOpStates === 'function' + && typeof stateStore.getOpStateRecord === 'function' + && typeof orchestrator.markFailedIfUnchanged === 'function'; +} + +export function createOperationReconciler(input: { + stateStore: ReconciliationStateStore; + orchestrator: ReconciliationOrchestrator; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + opStaleMs: number; + getGeneration: () => number; + logger: ReconciliationLogger; +}) { + let recoveryPromise: Promise | null = null; + let recoveredGeneration = -1; + + const run = async (options?: { force?: boolean }): Promise => { + if (!supportsRecovery(input.stateStore, input.orchestrator)) return; + const generation = input.getGeneration(); + if (!options?.force && recoveredGeneration === generation) return; + if (recoveryPromise) return await recoveryPromise; + + recoveryPromise = (async () => { + const recoveredStates = await recoverOrphanedOperations({ + operationStateStore: { + getOpStateRecord: (opId) => input.stateStore.getOpStateRecord!(opId), + listOpStates: () => input.stateStore.listOpStates!(), + }, + orchestrator: { + markFailedIfUnchanged: async (request) => { + const result = await input.orchestrator.markFailedIfUnchanged!(request); + return result as StreamedOperationState | null; + }, + }, + whisperTimeoutMs: input.whisperTimeoutMs, + pdfTimeoutMs: input.pdfTimeoutMs, + opStaleMs: input.opStaleMs, + }); + if (recoveredStates.length > 0) { + input.logger.warn({ + recoveredCount: recoveredStates.length, + ops: recoveredStates.map((state) => ({ + opId: state.opId, + kind: state.kind, + status: state.status, + })), + }, 'recovered stale in-flight operations during reconciliation'); + } + recoveredGeneration = generation; + })().finally(() => { + recoveryPromise = null; + }); + await recoveryPromise; + }; + + return { + run, + async getOpState(opId: string): Promise { + await run(); + return await input.stateStore.getOpState(opId); + }, + }; +} diff --git a/compute/worker/src/orphan-recovery.ts b/packages/compute-worker/src/operations/recovery.ts similarity index 98% rename from compute/worker/src/orphan-recovery.ts rename to packages/compute-worker/src/operations/recovery.ts index 882ea94..b453a9a 100644 --- a/compute/worker/src/orphan-recovery.ts +++ b/packages/compute-worker/src/operations/recovery.ts @@ -4,7 +4,7 @@ import type { WorkerJobTiming, WorkerJobState, WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; +} from '../operations/contracts'; export type StreamedOperationState = WorkerOperationState; diff --git a/compute/core/src/control-plane/orchestrator.ts b/packages/compute-worker/src/operations/service.ts similarity index 99% rename from compute/core/src/control-plane/orchestrator.ts rename to packages/compute-worker/src/operations/service.ts index 7f621a4..67430e9 100644 --- a/compute/core/src/control-plane/orchestrator.ts +++ b/packages/compute-worker/src/operations/service.ts @@ -5,7 +5,7 @@ import type { WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, -} from '../api-contracts'; +} from '../operations/contracts'; import { buildQueuedState, createErrorShape, diff --git a/compute/core/src/control-plane/sse.ts b/packages/compute-worker/src/operations/sse.ts similarity index 100% rename from compute/core/src/control-plane/sse.ts rename to packages/compute-worker/src/operations/sse.ts diff --git a/compute/core/src/control-plane/state-machine.ts b/packages/compute-worker/src/operations/state-machine.ts similarity index 98% rename from compute/core/src/control-plane/state-machine.ts rename to packages/compute-worker/src/operations/state-machine.ts index d280215..cd4076f 100644 --- a/compute/core/src/control-plane/state-machine.ts +++ b/packages/compute-worker/src/operations/state-machine.ts @@ -4,7 +4,7 @@ import type { WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, -} from '../api-contracts'; +} from '../operations/contracts'; export function isTerminalStatus(status: WorkerJobState): boolean { return status === 'succeeded' || status === 'failed'; diff --git a/compute/core/src/control-plane/types.ts b/packages/compute-worker/src/operations/types.ts similarity index 98% rename from compute/core/src/control-plane/types.ts rename to packages/compute-worker/src/operations/types.ts index 9fc66ff..d3f24c2 100644 --- a/compute/core/src/control-plane/types.ts +++ b/packages/compute-worker/src/operations/types.ts @@ -2,7 +2,7 @@ import type { WorkerOperationEvent, WorkerOperationKind, WorkerOperationState, -} from '../api-contracts'; +} from '../operations/contracts'; export type OperationState = WorkerOperationState; export type OperationEvent = WorkerOperationEvent; diff --git a/compute/worker/src/server.ts b/packages/compute-worker/src/server.ts similarity index 71% rename from compute/worker/src/server.ts rename to packages/compute-worker/src/server.ts index 1bf176c..3a110cc 100644 --- a/compute/worker/src/server.ts +++ b/packages/compute-worker/src/server.ts @@ -1,4 +1,4 @@ -import { startComputeWorkerFromEnv } from './runtime'; +import { startComputeWorkerFromEnv } from './api/app'; void startComputeWorkerFromEnv().catch((error) => { console.error('[compute-worker] fatal startup error', error); diff --git a/packages/compute-worker/src/storage/artifact-addressing.ts b/packages/compute-worker/src/storage/artifact-addressing.ts new file mode 100644 index 0000000..10bb820 --- /dev/null +++ b/packages/compute-worker/src/storage/artifact-addressing.ts @@ -0,0 +1,21 @@ +import { PDF_PARSER_VERSION } from '../operations/contracts'; +import { encodeParserVersion } from '../operations/contracts'; + +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; + +export function parsedPdfArtifactKey(input: { + documentId: string; + namespace: string | null; + prefix: string; + parserVersion?: string; +}): string { + if (!DOCUMENT_ID_REGEX.test(input.documentId)) { + throw new Error(`Invalid document id: ${input.documentId}`); + } + const namespace = input.namespace && SAFE_NAMESPACE_REGEX.test(input.namespace) + ? input.namespace + : null; + const namespaceSegment = namespace ? `ns/${namespace}/` : ''; + return `${input.prefix}/documents_v1/parsed_v2/${namespaceSegment}${input.documentId}/${encodeParserVersion(input.parserVersion ?? PDF_PARSER_VERSION)}.json`; +} diff --git a/compute/worker/tests/api/routes.test.ts b/packages/compute-worker/tests/api/routes.test.ts similarity index 53% rename from compute/worker/tests/api/routes.test.ts rename to packages/compute-worker/tests/api/routes.test.ts index dad89d9..4dea817 100644 --- a/compute/worker/tests/api/routes.test.ts +++ b/packages/compute-worker/tests/api/routes.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from 'vitest'; -import { createComputeWorkerApp } from '../../src/runtime'; +import { createComputeWorkerApp } from '../../src/api/app'; import { FakeControlPlane } from '../fixtures/fake-control-plane'; const AUTH = { authorization: 'Bearer test-token' }; @@ -25,53 +25,51 @@ describe('compute worker API routes', () => { const live = await runtime.app.inject({ method: 'GET', url: '/health/live' }); expect(live.statusCode).toBe(200); - const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' }); + const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/v1/operations/op-1' }); expect(protectedRoute.statusCode).toBe(401); }); test('validates operation creation body and returns 400 for invalid payload', async () => { const response = await runtime.app.inject({ method: 'POST', - url: '/ops', + url: '/v1/pdf-layout/operations', headers: AUTH, payload: { - kind: 'pdf_layout', - opKey: '', - payload: { - documentId: 'd1', - namespace: null, - documentObjectKey: 's3://bucket/doc.pdf', - }, + documentId: '', + namespace: null, + documentObjectKey: 'openreader/doc.pdf', }, }); expect(response.statusCode).toBe(400); - expect(response.json()).toMatchObject({ error: 'Invalid request body' }); + expect(response.json()).toMatchObject({ error: 'Bad Request' }); }); test('creates operation and fetches state by op id', async () => { + const documentId = 'c'.repeat(64); const create = await runtime.app.inject({ method: 'POST', - url: '/ops', + url: '/v1/pdf-layout/operations', headers: AUTH, payload: { - kind: 'pdf_layout', - opKey: 'doc-1:layout', - payload: { - documentId: 'doc-1', - namespace: null, - documentObjectKey: 'openreader/doc-1.pdf', - }, + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, }, }); expect(create.statusCode).toBe(202); const created = create.json(); - expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' }); + expect(created).toMatchObject({ + subject: { kind: 'pdf_layout', documentId, namespace: null }, + status: 'queued', + }); + expect(created).not.toHaveProperty('opKey'); + expect(created).not.toHaveProperty('jobId'); const fetch = await runtime.app.inject({ method: 'GET', - url: `/ops/${created.opId}`, + url: `/v1/operations/${created.opId}`, headers: AUTH, }); @@ -79,17 +77,120 @@ describe('compute worker API routes', () => { expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' }); }); + test('reuses idempotent PDF requests and replaces them only with an explicit token', async () => { + const documentId = 'e'.repeat(64); + const payload = { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }; + const create = (body: typeof payload & { replaceToken?: string }) => runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/operations', + headers: AUTH, + payload: body, + }); + + const initial = (await create(payload)).json(); + const reused = (await create(payload)).json(); + const replacement = (await create({ ...payload, replaceToken: 'replace-1' })).json(); + + expect(reused.opId).toBe(initial.opId); + expect(replacement.opId).not.toBe(initial.opId); + expect(replacement.subject).toEqual(initial.subject); + }); + + test('creates whisper alignment operations without exposing internal keys', async () => { + const response = await runtime.app.inject({ + method: 'POST', + url: '/v1/whisper-align/operations', + headers: AUTH, + payload: { + text: 'Canonical worker text', + audioObjectKey: 'openreader/audio.mp3', + lang: 'en', + }, + }); + + expect(response.statusCode).toBe(202); + expect(response.json()).toMatchObject({ + subject: { kind: 'whisper_align' }, + status: 'queued', + }); + expect(response.json()).not.toHaveProperty('opKey'); + }); + + test('resolves the current PDF artifact and operation without exposing parser identity', async () => { + const documentId = 'a'.repeat(64); + const create = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/operations', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + expect(create.statusCode).toBe(202); + + const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`; + fake.seedArtifact(artifactKey); + const resolve = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/resolve', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + + expect(resolve.statusCode).toBe(200); + expect(resolve.json()).toMatchObject({ + artifact: { objectKey: artifactKey }, + operation: { + subject: { kind: 'pdf_layout', documentId, namespace: null }, + }, + }); + expect(resolve.json().operation).not.toHaveProperty('opKey'); + }); + + test('resolves a current PDF artifact after operation state has expired', async () => { + const documentId = 'd'.repeat(64); + const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`; + fake.seedArtifact(artifactKey); + + const resolve = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/resolve', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + + expect(resolve.statusCode).toBe(200); + expect(resolve.json()).toEqual({ + artifact: { objectKey: artifactKey }, + operation: null, + }); + }); + test('returns not found for unknown operation and event stream lookups', async () => { const opResponse = await runtime.app.inject({ method: 'GET', - url: '/ops/missing', + url: '/v1/operations/missing', headers: AUTH, }); expect(opResponse.statusCode).toBe(404); const eventsResponse = await runtime.app.inject({ method: 'GET', - url: '/ops/missing/events', + url: '/v1/operations/missing/events', headers: AUTH, }); expect(eventsResponse.statusCode).toBe(404); @@ -109,7 +210,7 @@ describe('compute worker API routes', () => { const stream = await runtime.app.inject({ method: 'GET', - url: '/ops/op-terminal/events?sinceEventId=7', + url: '/v1/operations/op-terminal/events?sinceEventId=7', headers: AUTH, }); @@ -159,11 +260,11 @@ describe('compute worker API routes', () => { updatedAt: now - 310_000, }); - // GET /ops/:opId resolves via getOpState(), which first awaits the shared + // GET /v1/operations/:opId resolves via getOpState(), which first awaits the shared // orphanRecoveryPromise path through ensureOrphanedOpRecovery(). const fetch = await runtime.app.inject({ method: 'GET', - url: '/ops/op-stale-whisper-running', + url: '/v1/operations/op-stale-whisper-running', headers: AUTH, }); diff --git a/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts b/packages/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts similarity index 95% rename from tests/unit/pdf-merge-text-with-regions.vitest.spec.ts rename to packages/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts index 449824e..cb18eee 100644 --- a/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts +++ b/packages/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { mergeTextWithRegions } from '@openreader/compute-core'; +import { mergeTextWithRegions } from '../../../src/inference/pdf/document-layout'; describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { diff --git a/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts b/packages/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts similarity index 96% rename from tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts rename to packages/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts index b61bd26..f93cf25 100644 --- a/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts +++ b/packages/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { normalizeTextItemsForLayout } from '@openreader/compute-core'; +import { normalizeTextItemsForLayout } from '../../../src/inference/pdf/normalize-text'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; function makeTextItem( diff --git a/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts b/packages/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts similarity index 96% rename from tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts rename to packages/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts index 0aa37e6..3fd35a6 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts +++ b/packages/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { stitchCrossPageBlocks } from '@openreader/compute-core'; -import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf'; +import { stitchCrossPageBlocks } from '../../../src/inference/pdf/stitch'; +import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/api/types'; import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( diff --git a/packages/compute-worker/tests/compute/algorithms/support/document-fixtures.ts b/packages/compute-worker/tests/compute/algorithms/support/document-fixtures.ts new file mode 100644 index 0000000..4d8d23d --- /dev/null +++ b/packages/compute-worker/tests/compute/algorithms/support/document-fixtures.ts @@ -0,0 +1,54 @@ +import type { BaseDocument } from '../../../../../../src/types/documents'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/api/types'; + +export function makeBaseDocument(overrides: Partial = {}): BaseDocument { + return { + id: 'doc-1', + name: 'document.pdf', + size: 1_024, + lastModified: 1_700_000_000_000, + type: 'pdf', + ...overrides, + }; +} + +export function makeParsedPdfBlock(input: { + id: string; + kind: ParsedPdfBlockKind; + text: string; + page: number; + readingOrder: number; +}): ParsedPdfBlock { + return { + id: input.id, + kind: input.kind, + text: input.text, + fragments: [ + { + page: input.page, + bbox: [0, 0, 100, 10], + text: input.text, + readingOrder: input.readingOrder, + }, + ], + }; +} + +export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage { + return { + pageNumber, + width: 800, + height: 1_200, + blocks, + }; +} + +export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument { + return { + schemaVersion: 1, + documentId: 'doc-fixture', + parserVersion: 'test', + parsedAt: 1_700_000_000_000, + pages, + }; +} diff --git a/tests/unit/whisper-alignment-mapping.vitest.spec.ts b/packages/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts similarity index 96% rename from tests/unit/whisper-alignment-mapping.vitest.spec.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts index e5c9575..201c9c6 100644 --- a/tests/unit/whisper-alignment-mapping.vitest.spec.ts +++ b/packages/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, -} from '@openreader/compute-core'; +} from '../../../src/inference/whisper/timestamps'; describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { diff --git a/packages/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts b/packages/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts new file mode 100644 index 0000000..07f405e --- /dev/null +++ b/packages/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from 'vitest'; +import { + applyTokenSuppression, + applyWhisperTimestampLogitsRules, + argmax, +} from '../../../src/inference/whisper/decoder'; + +describe('whisper decoder logits', () => { + test('selects finite maximums and suppresses valid token ids', () => { + const logits = new Float32Array([1, 5, 3]); + + applyTokenSuppression(logits, new Set([-1, 1, 10])); + + expect(argmax(logits)).toBe(2); + expect(logits[1]).toBe(Number.NEGATIVE_INFINITY); + expect(argmax(new Float32Array([Number.NEGATIVE_INFINITY]))).toBeNull(); + }); + + test('forces an initial timestamp within the configured range', () => { + const logits = new Float32Array([4, 3, 2, 1, 8, 7, 6, 5]); + + applyWhisperTimestampLogitsRules({ + logits, + generated: [10, 11], + beginIndex: 2, + eosTokenId: 3, + noTimestampsTokenId: 2, + timestampBeginTokenId: 4, + maxInitialTimestampIndex: 1, + }); + + expect(Array.from(logits.slice(0, 4))).toEqual([ + Number.NEGATIVE_INFINITY, + Number.NEGATIVE_INFINITY, + Number.NEGATIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]); + expect(logits[4]).toBe(8); + expect(logits[5]).toBe(7); + expect(logits[6]).toBe(Number.NEGATIVE_INFINITY); + expect(logits[7]).toBe(Number.NEGATIVE_INFINITY); + }); + + test('requires text after a timestamp pair', () => { + const logits = new Float32Array([4, 3, 2, 1, 8, 7]); + + applyWhisperTimestampLogitsRules({ + logits, + generated: [10, 4, 5], + beginIndex: 1, + eosTokenId: 3, + noTimestampsTokenId: -1, + timestampBeginTokenId: 4, + maxInitialTimestampIndex: Number.POSITIVE_INFINITY, + }); + + expect(logits[4]).toBe(Number.NEGATIVE_INFINITY); + expect(logits[5]).toBe(Number.NEGATIVE_INFINITY); + expect(logits[0]).toBe(4); + }); +}); diff --git a/tests/unit/whisper-spectral.vitest.spec.ts b/packages/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts similarity index 92% rename from tests/unit/whisper-spectral.vitest.spec.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts index 2b87a04..1794290 100644 --- a/tests/unit/whisper-spectral.vitest.spec.ts +++ b/packages/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core'; +import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference/whisper/align'; function dftPower(samples: Float32Array, k: number): number { const n = samples.length; diff --git a/tests/unit/whisper-token-timestamps.vitest.spec.ts b/packages/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts similarity index 97% rename from tests/unit/whisper-token-timestamps.vitest.spec.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts index c2aded0..e1cdc18 100644 --- a/tests/unit/whisper-token-timestamps.vitest.spec.ts +++ b/packages/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts @@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from '@openreader/compute-core'; +} from '../../../src/inference/whisper/timestamps'; describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { diff --git a/compute/core/tests/control-plane/orchestrator.test.ts b/packages/compute-worker/tests/compute/control-plane/orchestrator.test.ts similarity index 96% rename from compute/core/tests/control-plane/orchestrator.test.ts rename to packages/compute-worker/tests/compute/control-plane/orchestrator.test.ts index 80bd9fb..092cfdb 100644 --- a/compute/core/tests/control-plane/orchestrator.test.ts +++ b/packages/compute-worker/tests/compute/control-plane/orchestrator.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationRequest } from '../../src/api-contracts'; -import { OperationOrchestrator } from '../../src/control-plane'; +import type { WorkerOperationRequest } from '../../../src/operations/contracts'; +import { OperationOrchestrator } from '../../../src/operations'; import { InMemoryOperationEventStream, InMemoryOperationQueue, diff --git a/compute/core/tests/control-plane/run-layout-model.test.ts b/packages/compute-worker/tests/compute/control-plane/run-layout-model.test.ts similarity index 93% rename from compute/core/tests/control-plane/run-layout-model.test.ts rename to packages/compute-worker/tests/compute/control-plane/run-layout-model.test.ts index e951bf3..cc6a34d 100644 --- a/compute/core/tests/control-plane/run-layout-model.test.ts +++ b/packages/compute-worker/tests/compute/control-plane/run-layout-model.test.ts @@ -70,13 +70,13 @@ vi.mock('@napi-rs/canvas', () => { }; }); -vi.mock('../../src/pdf/model', () => ({ +vi.mock('../../../src/inference/pdf/model', () => ({ ensureModel: vi.fn(async () => '/tmp/model.onnx'), MODEL_CONFIG_PATH: '/tmp/model-config.json', MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json', })); -vi.mock('../../src/config/cpu-budget', () => ({ +vi.mock('../../../src/infrastructure/config', () => ({ getOnnxThreadsPerJob: vi.fn(() => 1), })); @@ -107,7 +107,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../src/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, @@ -154,7 +154,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../src/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, diff --git a/compute/core/tests/control-plane/sse.test.ts b/packages/compute-worker/tests/compute/control-plane/sse.test.ts similarity index 97% rename from compute/core/tests/control-plane/sse.test.ts rename to packages/compute-worker/tests/compute/control-plane/sse.test.ts index 9b8e5b9..2697d6d 100644 --- a/compute/core/tests/control-plane/sse.test.ts +++ b/packages/compute-worker/tests/compute/control-plane/sse.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/operations/sse'; describe('sse codec', () => { test('encodes event id and payload and decodes both reliably', () => { diff --git a/compute/core/tests/control-plane/state-machine.test.ts b/packages/compute-worker/tests/compute/control-plane/state-machine.test.ts similarity index 93% rename from compute/core/tests/control-plane/state-machine.test.ts rename to packages/compute-worker/tests/compute/control-plane/state-machine.test.ts index e9617cb..667084f 100644 --- a/compute/core/tests/control-plane/state-machine.test.ts +++ b/packages/compute-worker/tests/compute/control-plane/state-machine.test.ts @@ -1,11 +1,11 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationState } from '../../src/api-contracts'; +import type { WorkerOperationState } from '../../../src/operations/contracts'; import { explainReplacementReason, isInflightStatus, isTerminalStatus, shouldReuseExistingOperation, -} from '../../src/control-plane/state-machine'; +} from '../../../src/operations/state-machine'; function runningState(overrides: Partial = {}): WorkerOperationState { return { diff --git a/compute/core/tests/helpers/in-memory-control-plane.ts b/packages/compute-worker/tests/compute/helpers/in-memory-control-plane.ts similarity index 97% rename from compute/core/tests/helpers/in-memory-control-plane.ts rename to packages/compute-worker/tests/compute/helpers/in-memory-control-plane.ts index cc9b9ce..0348bff 100644 --- a/compute/core/tests/helpers/in-memory-control-plane.ts +++ b/packages/compute-worker/tests/compute/helpers/in-memory-control-plane.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { WorkerOperationKind } from '../../src/api-contracts'; +import type { WorkerOperationKind } from '../../../src/operations/contracts'; import type { OperationEvent, OperationEventStream, @@ -8,7 +8,7 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '../../src/control-plane/types'; +} from '../../../src/operations/types'; function topicFor(opId: string): string { return `op.${opId}`; diff --git a/compute/worker/tests/fixtures/fake-control-plane.ts b/packages/compute-worker/tests/fixtures/fake-control-plane.ts similarity index 92% rename from compute/worker/tests/fixtures/fake-control-plane.ts rename to packages/compute-worker/tests/fixtures/fake-control-plane.ts index 9822523..0cd1c86 100644 --- a/compute/worker/tests/fixtures/fake-control-plane.ts +++ b/packages/compute-worker/tests/fixtures/fake-control-plane.ts @@ -4,8 +4,8 @@ import type { WorkerOperationEvent, WorkerOperationRequest, WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; -import type { ComputeWorkerRouteDeps } from '../../src/runtime'; +} from '../../src/operations/contracts'; +import type { ComputeWorkerRouteDeps } from '../../src/api/app'; type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; type ComputeState = WorkerOperationState; @@ -16,6 +16,7 @@ export class FakeControlPlane { private readonly revisionByOpId = new Map(); private readonly opIdByOpKey = new Map(); private readonly eventsByOpId = new Map(); + private readonly artifactKeys = new Set(); private nextOpId = 1; readonly deps: ComputeWorkerRouteDeps = { @@ -58,6 +59,10 @@ export class FakeControlPlane { }; }, listOpStates: async () => Array.from(this.stateByOpId.values()), + getOpIndex: async (opKey) => { + const opId = this.opIdByOpKey.get(opKey); + return opId ? { opId } : null; + }, }, operationEventStream: { subscribe: async ({ opId, sinceEventId, onEvent }) => { @@ -69,6 +74,7 @@ export class FakeControlPlane { return () => undefined; }, }, + artifactExists: async (key) => this.artifactKeys.has(key), }; seedState(state: ComputeState): void { @@ -83,6 +89,10 @@ export class FakeControlPlane { this.eventsByOpId.set(opId, list); } + seedArtifact(key: string): void { + this.artifactKeys.add(key); + } + getState(opId: string): ComputeState | null { return this.stateByOpId.get(opId) ?? null; } diff --git a/compute/worker/tests/setup-env.ts b/packages/compute-worker/tests/setup-env.ts similarity index 100% rename from compute/worker/tests/setup-env.ts rename to packages/compute-worker/tests/setup-env.ts diff --git a/compute/worker/tests/tsconfig.json b/packages/compute-worker/tests/tsconfig.json similarity index 100% rename from compute/worker/tests/tsconfig.json rename to packages/compute-worker/tests/tsconfig.json diff --git a/compute/worker/tests/unit/jetstream-adapters.test.ts b/packages/compute-worker/tests/unit/jetstream-adapters.test.ts similarity index 92% rename from compute/worker/tests/unit/jetstream-adapters.test.ts rename to packages/compute-worker/tests/unit/jetstream-adapters.test.ts index 9f30ccc..d694594 100644 --- a/compute/worker/tests/unit/jetstream-adapters.test.ts +++ b/packages/compute-worker/tests/unit/jetstream-adapters.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts'; +import { OperationOrchestrator } from '../../src/operations'; +import type { WorkerOperationRequest } from '../../src/operations/contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, @@ -11,7 +11,7 @@ import { opStateKvKey, type KvEntryLike, type KvStoreLike, -} from '../../src/control-plane/jetstream'; +} from '../../src/infrastructure/nats-adapters'; class FakeKvStore implements KvStoreLike { private readonly data = new Map(); @@ -45,6 +45,17 @@ class FakeKvStore implements KvStoreLike { this.revision += 1; this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } + + async keys(filter?: string | string[]): Promise> { + const keys = Array.from(this.data.keys()); + return { + async *[Symbol.asyncIterator]() { + for (const key of keys) { + yield key; + } + } + }; + } } class FakeJetStream { diff --git a/compute/worker/tests/unit/key-and-progress.test.ts b/packages/compute-worker/tests/unit/key-and-progress.test.ts similarity index 59% rename from compute/worker/tests/unit/key-and-progress.test.ts rename to packages/compute-worker/tests/unit/key-and-progress.test.ts index 6887cad..dc1656a 100644 --- a/compute/worker/tests/unit/key-and-progress.test.ts +++ b/packages/compute-worker/tests/unit/key-and-progress.test.ts @@ -1,9 +1,11 @@ import { describe, expect, test } from 'vitest'; -import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/control-plane/jetstream'; +import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/infrastructure/nats-adapters'; import { buildInferProgressForPageParsed, buildInferProgressForPageStart, -} from '../../src/pdf-progress'; +} from '../../src/jobs/pdf-progress'; +import { buildPdfOperationKey } from '../../src/operations/keys'; +import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing'; describe('compute worker helpers', () => { test('hash and kv key helpers are stable and deterministic', () => { @@ -34,4 +36,17 @@ describe('compute worker helpers', () => { phase: 'infer', }); }); + + test('parser-version changes rotate worker-owned operation and artifact identities', () => { + const request = { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: `openreader/${'a'.repeat(64)}.pdf`, + }; + + expect(buildPdfOperationKey(request, 'parser-v1')) + .not.toBe(buildPdfOperationKey(request, 'parser-v2')); + expect(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v1' })) + .not.toBe(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v2' })); + }); }); diff --git a/compute/worker/tests/unit/orphan-recovery.test.ts b/packages/compute-worker/tests/unit/orphan-recovery.test.ts similarity index 84% rename from compute/worker/tests/unit/orphan-recovery.test.ts rename to packages/compute-worker/tests/unit/orphan-recovery.test.ts index 712fdf6..54db314 100644 --- a/compute/worker/tests/unit/orphan-recovery.test.ts +++ b/packages/compute-worker/tests/unit/orphan-recovery.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; -import { recoverOrphanedOperations } from '../../src/orphan-recovery'; +import { recoverOrphanedOperations } from '../../src/operations/recovery'; import { FakeControlPlane } from '../fixtures/fake-control-plane'; describe('orphan recovery', () => { @@ -24,8 +24,8 @@ describe('orphan recovery', () => { }); const firstSweep = await recoverOrphanedOperations({ - operationStateStore: fake.deps.operationStateStore, - orchestrator: fake.deps.orchestrator, + operationStateStore: fake.deps.operationStateStore!, + orchestrator: fake.deps.orchestrator!, whisperTimeoutMs: 30_000, pdfTimeoutMs: 300_000, opStaleMs: 1_800_000, @@ -38,8 +38,8 @@ describe('orphan recovery', () => { vi.advanceTimersByTime(31_000); const secondSweep = await recoverOrphanedOperations({ - operationStateStore: fake.deps.operationStateStore, - orchestrator: fake.deps.orchestrator, + operationStateStore: fake.deps.operationStateStore!, + orchestrator: fake.deps.orchestrator!, whisperTimeoutMs: 30_000, pdfTimeoutMs: 300_000, opStaleMs: 1_800_000, diff --git a/compute/worker/tests/unit/pdf-artifact-persistence.test.ts b/packages/compute-worker/tests/unit/pdf-artifact-persistence.test.ts similarity index 92% rename from compute/worker/tests/unit/pdf-artifact-persistence.test.ts rename to packages/compute-worker/tests/unit/pdf-artifact-persistence.test.ts index 9378435..68f3de5 100644 --- a/compute/worker/tests/unit/pdf-artifact-persistence.test.ts +++ b/packages/compute-worker/tests/unit/pdf-artifact-persistence.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, vi } from 'vitest'; -import { persistParsedPdfWhileSourceExists } from '../../src/pdf-artifact-persistence'; +import { persistParsedPdfWhileSourceExists } from '../../src/jobs/pdf-artifact-persistence'; describe('PDF artifact persistence', () => { test('does not write parsed output after the source was deleted', async () => { diff --git a/compute/worker/tests/unit/worker-loop-policy.test.ts b/packages/compute-worker/tests/unit/worker-loop-policy.test.ts similarity index 96% rename from compute/worker/tests/unit/worker-loop-policy.test.ts rename to packages/compute-worker/tests/unit/worker-loop-policy.test.ts index 818e459..c200e70 100644 --- a/compute/worker/tests/unit/worker-loop-policy.test.ts +++ b/packages/compute-worker/tests/unit/worker-loop-policy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildQueueWaitTiming, decideRetryAction } from '../../src/worker-loop-policy'; +import { buildQueueWaitTiming, decideRetryAction } from '../../src/jobs/worker-loop-policy'; describe('worker loop policy', () => { test('returns queue wait timing with non-negative clamped duration', () => { diff --git a/packages/compute-worker/tests/unit/worker-loop.test.ts b/packages/compute-worker/tests/unit/worker-loop.test.ts new file mode 100644 index 0000000..3b9ba07 --- /dev/null +++ b/packages/compute-worker/tests/unit/worker-loop.test.ts @@ -0,0 +1,170 @@ +import type { Consumer, JsMsg } from '@nats-io/jetstream'; +import { describe, expect, test, vi } from 'vitest'; +import type { PdfLayoutProgress } from '../../src/operations/contracts'; +import { createJsonCodec } from '../../src/infrastructure/json-codec'; +import { + createWorkerLoopController, + type QueuedJob, + type WorkerLoopOrchestrator, +} from '../../src/jobs/worker-loop'; + +function createMessage(job: QueuedJob, deliveryCount = 1) { + const codec = createJsonCodec>(); + const ack = vi.fn(); + const nak = vi.fn(); + const term = vi.fn(); + const working = vi.fn(); + return { + codec, + msg: { + data: codec.encode(job), + info: { deliveryCount }, + ack, + nak, + term, + working, + } as unknown as JsMsg, + ack, + nak, + term, + working, + }; +} + +function createConsumer(message?: JsMsg): Consumer { + let delivered = false; + return { + next: async () => { + if (!delivered && message) { + delivered = true; + return message; + } + await new Promise((resolve) => setTimeout(resolve, 1)); + return null; + }, + } as unknown as Consumer; +} + +function createOrchestrator() { + const calls: Array<{ method: string; input: unknown }> = []; + const record = (method: string) => async (input: unknown) => { + calls.push({ method, input }); + return input; + }; + const orchestrator: WorkerLoopOrchestrator = { + markRunning: record('running'), + markProgress: record('progress'), + markSucceeded: record('succeeded'), + markFailed: record('failed'), + }; + return { orchestrator, calls }; +} + +describe('worker loop controller', () => { + test('processes PDF progress, persists success, ACKs, and releases in-flight activity', async () => { + const owner = {}; + let active = true; + let inFlight = 0; + const { orchestrator, calls } = createOrchestrator(); + let complete!: () => void; + const completed = new Promise((resolve) => { complete = resolve; }); + const progress: PdfLayoutProgress = { totalPages: 2, pagesParsed: 1, currentPage: 1, phase: 'infer' }; + const pdf = createMessage({ + jobId: 'job-pdf', + opId: 'op-pdf', + opKey: 'pdf-key', + kind: 'pdf_layout', + queuedAt: Date.now() - 10, + payload: { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: 'openreader/doc.pdf', + }, + }); + const whisperCodec = createJsonCodec>(); + const controller = createWorkerLoopController({ + orchestrator, + handlers: { + runWhisper: async () => ({ alignments: [] }), + runPdfLayout: async (_payload, _queueWaitMs, hooks) => { + await hooks?.onProgress?.(progress); + active = false; + complete(); + return { parsedObjectKey: 'openreader/parsed.json' }; + }, + }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + jobConcurrency: 1, + pdfAttempts: 2, + whisperCodec, + pdfCodec: pdf.codec, + isOwnerActive: () => active, + isStopping: () => false, + markActivity: vi.fn(), + onInFlightJobsChanged: (delta) => { inFlight += delta; }, + }); + + controller.start(owner, { whisper: createConsumer(), pdfLayout: createConsumer(pdf.msg) }); + await completed; + await controller.stop(); + + expect(pdf.working).toHaveBeenCalledOnce(); + expect(pdf.ack).toHaveBeenCalledOnce(); + expect(pdf.nak).not.toHaveBeenCalled(); + expect(inFlight).toBe(0); + expect(calls.map((call) => call.method)).toContain('progress'); + expect(calls.map((call) => call.method)).toContain('succeeded'); + }); + + test('NAKs a retryable PDF failure and does not mark it terminal', async () => { + const owner = {}; + let active = true; + const { orchestrator, calls } = createOrchestrator(); + let attempted!: () => void; + const attemptCompleted = new Promise((resolve) => { attempted = resolve; }); + const pdf = createMessage({ + jobId: 'job-pdf', + opId: 'op-pdf', + opKey: 'pdf-key', + kind: 'pdf_layout', + queuedAt: Date.now(), + payload: { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: 'openreader/doc.pdf', + }, + }); + const controller = createWorkerLoopController({ + orchestrator, + handlers: { + runWhisper: async () => ({ alignments: [] }), + runPdfLayout: async () => { + active = false; + attempted(); + throw new Error('retry me'); + }, + }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + jobConcurrency: 1, + pdfAttempts: 2, + whisperCodec: createJsonCodec(), + pdfCodec: pdf.codec, + isOwnerActive: () => active, + isStopping: () => false, + markActivity: vi.fn(), + onInFlightJobsChanged: vi.fn(), + }); + + controller.start(owner, { whisper: createConsumer(), pdfLayout: createConsumer(pdf.msg) }); + await attemptCompleted; + await new Promise((resolve) => setTimeout(resolve, 0)); + await controller.stop(); + + expect(pdf.nak).toHaveBeenCalledOnce(); + expect(pdf.term).not.toHaveBeenCalled(); + expect(calls.map((call) => call.method)).not.toContain('failed'); + }); +}); diff --git a/compute/worker/tsconfig.json b/packages/compute-worker/tsconfig.json similarity index 100% rename from compute/worker/tsconfig.json rename to packages/compute-worker/tsconfig.json diff --git a/packages/database/bin/migrate.mjs b/packages/database/bin/migrate.mjs new file mode 100755 index 0000000..39fd805 --- /dev/null +++ b/packages/database/bin/migrate.mjs @@ -0,0 +1,9 @@ +#!/usr/bin/env node +import { runMigrations } from '../src/migrate.mjs'; + +try { + await runMigrations(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/drizzle.config.pg.ts b/packages/database/drizzle.config.pg.ts similarity index 72% rename from drizzle.config.pg.ts rename to packages/database/drizzle.config.pg.ts index 663a9f7..b2ff9f6 100644 --- a/drizzle.config.pg.ts +++ b/packages/database/drizzle.config.pg.ts @@ -10,8 +10,8 @@ if (!url) { } export default { - schema: ['./src/db/schema_postgres.ts', './src/db/schema_auth_postgres.ts'], - out: './drizzle/postgres', + schema: ['./packages/database/src/schema_postgres.ts', './packages/database/src/schema_auth_postgres.ts'], + out: './packages/database/migrations/postgres', dialect: 'postgresql', dbCredentials: { url, diff --git a/drizzle.config.sqlite.ts b/packages/database/drizzle.config.sqlite.ts similarity index 57% rename from drizzle.config.sqlite.ts rename to packages/database/drizzle.config.sqlite.ts index 74cc605..ec64ec4 100644 --- a/drizzle.config.sqlite.ts +++ b/packages/database/drizzle.config.sqlite.ts @@ -4,8 +4,8 @@ import * as dotenv from 'dotenv'; dotenv.config(); export default { - schema: ['./src/db/schema_sqlite.ts', './src/db/schema_auth_sqlite.ts'], - out: './drizzle/sqlite', + schema: ['./packages/database/src/schema_sqlite.ts', './packages/database/src/schema_auth_sqlite.ts'], + out: './packages/database/migrations/sqlite', dialect: 'sqlite', dbCredentials: { url: 'docstore/sqlite3.db', diff --git a/drizzle/postgres/0000_black_lucky_pierre.sql b/packages/database/migrations/postgres/0000_black_lucky_pierre.sql similarity index 100% rename from drizzle/postgres/0000_black_lucky_pierre.sql rename to packages/database/migrations/postgres/0000_black_lucky_pierre.sql diff --git a/drizzle/postgres/0001_tts_segments.sql b/packages/database/migrations/postgres/0001_tts_segments.sql similarity index 100% rename from drizzle/postgres/0001_tts_segments.sql rename to packages/database/migrations/postgres/0001_tts_segments.sql diff --git a/drizzle/postgres/0002_add_segment_key_to_tts_segments.sql b/packages/database/migrations/postgres/0002_add_segment_key_to_tts_segments.sql similarity index 100% rename from drizzle/postgres/0002_add_segment_key_to_tts_segments.sql rename to packages/database/migrations/postgres/0002_add_segment_key_to_tts_segments.sql diff --git a/drizzle/postgres/0003_tts_segments_v2_split.sql b/packages/database/migrations/postgres/0003_tts_segments_v2_split.sql similarity index 100% rename from drizzle/postgres/0003_tts_segments_v2_split.sql rename to packages/database/migrations/postgres/0003_tts_segments_v2_split.sql diff --git a/drizzle/postgres/0004_admin_panel.sql b/packages/database/migrations/postgres/0004_admin_panel.sql similarity index 100% rename from drizzle/postgres/0004_admin_panel.sql rename to packages/database/migrations/postgres/0004_admin_panel.sql diff --git a/drizzle/postgres/0005_pdf_layout_and_compute.sql b/packages/database/migrations/postgres/0005_pdf_layout_and_compute.sql similarity index 100% rename from drizzle/postgres/0005_pdf_layout_and_compute.sql rename to packages/database/migrations/postgres/0005_pdf_layout_and_compute.sql diff --git a/drizzle/postgres/0006_preview-defaults-cleanup.sql b/packages/database/migrations/postgres/0006_preview-defaults-cleanup.sql similarity index 100% rename from drizzle/postgres/0006_preview-defaults-cleanup.sql rename to packages/database/migrations/postgres/0006_preview-defaults-cleanup.sql diff --git a/drizzle/postgres/0007_pdf_parse_progress.sql b/packages/database/migrations/postgres/0007_pdf_parse_progress.sql similarity index 100% rename from drizzle/postgres/0007_pdf_parse_progress.sql rename to packages/database/migrations/postgres/0007_pdf_parse_progress.sql diff --git a/drizzle/postgres/0008_user_job_events.sql b/packages/database/migrations/postgres/0008_user_job_events.sql similarity index 100% rename from drizzle/postgres/0008_user_job_events.sql rename to packages/database/migrations/postgres/0008_user_job_events.sql diff --git a/drizzle/postgres/0009_drop_pdf_parse.sql b/packages/database/migrations/postgres/0009_drop_pdf_parse.sql similarity index 100% rename from drizzle/postgres/0009_drop_pdf_parse.sql rename to packages/database/migrations/postgres/0009_drop_pdf_parse.sql diff --git a/drizzle/postgres/0010_user-data-cleanup-cascades.sql b/packages/database/migrations/postgres/0010_user-data-cleanup-cascades.sql similarity index 100% rename from drizzle/postgres/0010_user-data-cleanup-cascades.sql rename to packages/database/migrations/postgres/0010_user-data-cleanup-cascades.sql diff --git a/drizzle/postgres/0011_scheduled-tasks.sql b/packages/database/migrations/postgres/0011_scheduled-tasks.sql similarity index 100% rename from drizzle/postgres/0011_scheduled-tasks.sql rename to packages/database/migrations/postgres/0011_scheduled-tasks.sql diff --git a/drizzle/postgres/meta/0000_snapshot.json b/packages/database/migrations/postgres/meta/0000_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0000_snapshot.json rename to packages/database/migrations/postgres/meta/0000_snapshot.json diff --git a/drizzle/postgres/meta/0001_snapshot.json b/packages/database/migrations/postgres/meta/0001_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0001_snapshot.json rename to packages/database/migrations/postgres/meta/0001_snapshot.json diff --git a/drizzle/postgres/meta/0002_snapshot.json b/packages/database/migrations/postgres/meta/0002_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0002_snapshot.json rename to packages/database/migrations/postgres/meta/0002_snapshot.json diff --git a/drizzle/postgres/meta/0003_snapshot.json b/packages/database/migrations/postgres/meta/0003_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0003_snapshot.json rename to packages/database/migrations/postgres/meta/0003_snapshot.json diff --git a/drizzle/postgres/meta/0004_snapshot.json b/packages/database/migrations/postgres/meta/0004_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0004_snapshot.json rename to packages/database/migrations/postgres/meta/0004_snapshot.json diff --git a/drizzle/postgres/meta/0005_snapshot.json b/packages/database/migrations/postgres/meta/0005_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0005_snapshot.json rename to packages/database/migrations/postgres/meta/0005_snapshot.json diff --git a/drizzle/postgres/meta/0006_snapshot.json b/packages/database/migrations/postgres/meta/0006_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0006_snapshot.json rename to packages/database/migrations/postgres/meta/0006_snapshot.json diff --git a/drizzle/postgres/meta/0007_snapshot.json b/packages/database/migrations/postgres/meta/0007_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0007_snapshot.json rename to packages/database/migrations/postgres/meta/0007_snapshot.json diff --git a/drizzle/postgres/meta/0008_snapshot.json b/packages/database/migrations/postgres/meta/0008_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0008_snapshot.json rename to packages/database/migrations/postgres/meta/0008_snapshot.json diff --git a/drizzle/postgres/meta/0009_snapshot.json b/packages/database/migrations/postgres/meta/0009_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0009_snapshot.json rename to packages/database/migrations/postgres/meta/0009_snapshot.json diff --git a/drizzle/postgres/meta/0010_snapshot.json b/packages/database/migrations/postgres/meta/0010_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0010_snapshot.json rename to packages/database/migrations/postgres/meta/0010_snapshot.json diff --git a/drizzle/postgres/meta/0011_snapshot.json b/packages/database/migrations/postgres/meta/0011_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0011_snapshot.json rename to packages/database/migrations/postgres/meta/0011_snapshot.json diff --git a/drizzle/postgres/meta/_journal.json b/packages/database/migrations/postgres/meta/_journal.json similarity index 100% rename from drizzle/postgres/meta/_journal.json rename to packages/database/migrations/postgres/meta/_journal.json diff --git a/drizzle/sqlite/0000_familiar_caretaker.sql b/packages/database/migrations/sqlite/0000_familiar_caretaker.sql similarity index 100% rename from drizzle/sqlite/0000_familiar_caretaker.sql rename to packages/database/migrations/sqlite/0000_familiar_caretaker.sql diff --git a/drizzle/sqlite/0001_tts_segments.sql b/packages/database/migrations/sqlite/0001_tts_segments.sql similarity index 100% rename from drizzle/sqlite/0001_tts_segments.sql rename to packages/database/migrations/sqlite/0001_tts_segments.sql diff --git a/drizzle/sqlite/0002_add_segment_key_to_tts_segments.sql b/packages/database/migrations/sqlite/0002_add_segment_key_to_tts_segments.sql similarity index 100% rename from drizzle/sqlite/0002_add_segment_key_to_tts_segments.sql rename to packages/database/migrations/sqlite/0002_add_segment_key_to_tts_segments.sql diff --git a/drizzle/sqlite/0003_tts_segments_v2_split.sql b/packages/database/migrations/sqlite/0003_tts_segments_v2_split.sql similarity index 100% rename from drizzle/sqlite/0003_tts_segments_v2_split.sql rename to packages/database/migrations/sqlite/0003_tts_segments_v2_split.sql diff --git a/drizzle/sqlite/0004_admin_panel.sql b/packages/database/migrations/sqlite/0004_admin_panel.sql similarity index 100% rename from drizzle/sqlite/0004_admin_panel.sql rename to packages/database/migrations/sqlite/0004_admin_panel.sql diff --git a/drizzle/sqlite/0005_pdf_layout_and_compute.sql b/packages/database/migrations/sqlite/0005_pdf_layout_and_compute.sql similarity index 100% rename from drizzle/sqlite/0005_pdf_layout_and_compute.sql rename to packages/database/migrations/sqlite/0005_pdf_layout_and_compute.sql diff --git a/drizzle/sqlite/0006_preview-defaults-cleanup.sql b/packages/database/migrations/sqlite/0006_preview-defaults-cleanup.sql similarity index 100% rename from drizzle/sqlite/0006_preview-defaults-cleanup.sql rename to packages/database/migrations/sqlite/0006_preview-defaults-cleanup.sql diff --git a/drizzle/sqlite/0007_pdf_parse_progress.sql b/packages/database/migrations/sqlite/0007_pdf_parse_progress.sql similarity index 100% rename from drizzle/sqlite/0007_pdf_parse_progress.sql rename to packages/database/migrations/sqlite/0007_pdf_parse_progress.sql diff --git a/drizzle/sqlite/0008_user_job_events.sql b/packages/database/migrations/sqlite/0008_user_job_events.sql similarity index 100% rename from drizzle/sqlite/0008_user_job_events.sql rename to packages/database/migrations/sqlite/0008_user_job_events.sql diff --git a/drizzle/sqlite/0009_drop_pdf_parse.sql b/packages/database/migrations/sqlite/0009_drop_pdf_parse.sql similarity index 100% rename from drizzle/sqlite/0009_drop_pdf_parse.sql rename to packages/database/migrations/sqlite/0009_drop_pdf_parse.sql diff --git a/drizzle/sqlite/0010_user-data-cleanup-cascades.sql b/packages/database/migrations/sqlite/0010_user-data-cleanup-cascades.sql similarity index 100% rename from drizzle/sqlite/0010_user-data-cleanup-cascades.sql rename to packages/database/migrations/sqlite/0010_user-data-cleanup-cascades.sql diff --git a/drizzle/sqlite/0011_scheduled-tasks.sql b/packages/database/migrations/sqlite/0011_scheduled-tasks.sql similarity index 100% rename from drizzle/sqlite/0011_scheduled-tasks.sql rename to packages/database/migrations/sqlite/0011_scheduled-tasks.sql diff --git a/drizzle/sqlite/meta/0000_snapshot.json b/packages/database/migrations/sqlite/meta/0000_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0000_snapshot.json rename to packages/database/migrations/sqlite/meta/0000_snapshot.json diff --git a/drizzle/sqlite/meta/0001_snapshot.json b/packages/database/migrations/sqlite/meta/0001_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0001_snapshot.json rename to packages/database/migrations/sqlite/meta/0001_snapshot.json diff --git a/drizzle/sqlite/meta/0002_snapshot.json b/packages/database/migrations/sqlite/meta/0002_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0002_snapshot.json rename to packages/database/migrations/sqlite/meta/0002_snapshot.json diff --git a/drizzle/sqlite/meta/0003_snapshot.json b/packages/database/migrations/sqlite/meta/0003_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0003_snapshot.json rename to packages/database/migrations/sqlite/meta/0003_snapshot.json diff --git a/drizzle/sqlite/meta/0004_snapshot.json b/packages/database/migrations/sqlite/meta/0004_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0004_snapshot.json rename to packages/database/migrations/sqlite/meta/0004_snapshot.json diff --git a/drizzle/sqlite/meta/0005_snapshot.json b/packages/database/migrations/sqlite/meta/0005_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0005_snapshot.json rename to packages/database/migrations/sqlite/meta/0005_snapshot.json diff --git a/drizzle/sqlite/meta/0006_snapshot.json b/packages/database/migrations/sqlite/meta/0006_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0006_snapshot.json rename to packages/database/migrations/sqlite/meta/0006_snapshot.json diff --git a/drizzle/sqlite/meta/0007_snapshot.json b/packages/database/migrations/sqlite/meta/0007_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0007_snapshot.json rename to packages/database/migrations/sqlite/meta/0007_snapshot.json diff --git a/drizzle/sqlite/meta/0008_snapshot.json b/packages/database/migrations/sqlite/meta/0008_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0008_snapshot.json rename to packages/database/migrations/sqlite/meta/0008_snapshot.json diff --git a/drizzle/sqlite/meta/0009_snapshot.json b/packages/database/migrations/sqlite/meta/0009_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0009_snapshot.json rename to packages/database/migrations/sqlite/meta/0009_snapshot.json diff --git a/drizzle/sqlite/meta/0010_snapshot.json b/packages/database/migrations/sqlite/meta/0010_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0010_snapshot.json rename to packages/database/migrations/sqlite/meta/0010_snapshot.json diff --git a/drizzle/sqlite/meta/0011_snapshot.json b/packages/database/migrations/sqlite/meta/0011_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0011_snapshot.json rename to packages/database/migrations/sqlite/meta/0011_snapshot.json diff --git a/drizzle/sqlite/meta/_journal.json b/packages/database/migrations/sqlite/meta/_journal.json similarity index 100% rename from drizzle/sqlite/meta/_journal.json rename to packages/database/migrations/sqlite/meta/_journal.json diff --git a/packages/database/package.json b/packages/database/package.json new file mode 100644 index 0000000..ee40dda --- /dev/null +++ b/packages/database/package.json @@ -0,0 +1,33 @@ +{ + "name": "@openreader/database", + "version": "0.0.0", + "private": true, + "type": "module", + "files": [ + "bin", + "migrations", + "src" + ], + "exports": { + ".": "./src/index.ts", + "./migrate": "./src/migrate.mjs", + "./schema": "./src/schema.ts", + "./schema-postgres": "./src/schema_postgres.ts", + "./schema-sqlite": "./src/schema_sqlite.ts", + "./schema-auth-postgres": "./src/schema_auth_postgres.ts", + "./schema-auth-sqlite": "./src/schema_auth_sqlite.ts", + "./run-in-transaction": "./src/run-in-transaction.ts" + }, + "bin": { + "openreader-migrate": "./bin/migrate.mjs" + }, + "scripts": { + "migrate": "node bin/migrate.mjs" + }, + "dependencies": { + "better-sqlite3": "^12.10.0", + "dotenv": "^17.4.2", + "drizzle-orm": "^0.45.2", + "pg": "^8.21.0" + } +} diff --git a/drizzle/scripts/generate.mjs b/packages/database/scripts/generate.mjs similarity index 91% rename from drizzle/scripts/generate.mjs rename to packages/database/scripts/generate.mjs index 320c334..8e56aa3 100644 --- a/drizzle/scripts/generate.mjs +++ b/packages/database/scripts/generate.mjs @@ -43,7 +43,7 @@ console.log('\n--- Generating Better Auth schema (SQLite) ---'); delete envSqlite.POSTGRES_URL; // force SQLite adapter const result = spawnSync('npx', [ '@better-auth/cli', 'generate', - '--output', 'src/db/schema_auth_sqlite.ts', + '--output', 'packages/database/src/schema_auth_sqlite.ts', '--yes', ], { stdio: 'inherit', env: envSqlite }); if ((result.status ?? 1) !== 0) process.exit(result.status ?? 1); @@ -58,7 +58,7 @@ console.log('\n--- Generating Better Auth schema (Postgres) ---'); } const result = spawnSync('npx', [ '@better-auth/cli', 'generate', - '--output', 'src/db/schema_auth_postgres.ts', + '--output', 'packages/database/src/schema_auth_postgres.ts', '--yes', ], { stdio: 'inherit', env: envPg }); if ((result.status ?? 1) !== 0) process.exit(result.status ?? 1); @@ -67,7 +67,10 @@ console.log('\n--- Generating Better Auth schema (Postgres) ---'); // --------------------------------------------------------------------------- // Step 2: Generate Drizzle migrations for both dialects. // --------------------------------------------------------------------------- -for (const configFile of ['drizzle.config.sqlite.ts', 'drizzle.config.pg.ts']) { +for (const configFile of [ + 'packages/database/drizzle.config.sqlite.ts', + 'packages/database/drizzle.config.pg.ts', +]) { console.log(`\n--- Drizzle generate (${configFile}) ---`); const result = spawnSync('drizzle-kit', ['generate', '--config', configFile, ...extraArgs], { stdio: 'inherit', diff --git a/src/db/index.ts b/packages/database/src/index.ts similarity index 91% rename from src/db/index.ts rename to packages/database/src/index.ts index a6e43ec..f20fd79 100644 --- a/src/db/index.ts +++ b/packages/database/src/index.ts @@ -59,6 +59,21 @@ export function ensureSystemUserExists(userId: string) { } } +function findWorkspaceRoot(startDir: string = process.cwd()): string { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + function getDrizzleDB() { if (dbInstance) return dbInstance; @@ -79,7 +94,8 @@ function getDrizzleDB() { const { drizzle: drizzleSqlite } = require('drizzle-orm/better-sqlite3'); // eslint-disable-next-line @typescript-eslint/no-require-imports const Database = require('better-sqlite3'); - const dbPath = path.join(process.cwd(), 'docstore', 'sqlite3.db'); + const workspaceRoot = findWorkspaceRoot(process.cwd()); + const dbPath = path.join(workspaceRoot, 'docstore', 'sqlite3.db'); const dir = path.dirname(dbPath); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); diff --git a/packages/database/src/migrate.mjs b/packages/database/src/migrate.mjs new file mode 100644 index 0000000..b022aad --- /dev/null +++ b/packages/database/src/migrate.mjs @@ -0,0 +1,68 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as dotenv from 'dotenv'; + +const packageDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +function loadEnvFiles(cwd) { + const envPath = path.join(cwd, '.env'); + const envLocalPath = path.join(cwd, '.env.local'); + + if (fs.existsSync(envPath)) dotenv.config({ path: envPath }); + if (fs.existsSync(envLocalPath)) dotenv.config({ path: envLocalPath, override: true }); +} + +function findWorkspaceRoot(startDir = process.cwd()) { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + +export async function runMigrations({ cwd = process.cwd(), env = process.env } = {}) { + const workspaceRoot = findWorkspaceRoot(cwd); + loadEnvFiles(workspaceRoot); + + if (env.POSTGRES_URL?.trim()) { + const [{ drizzle }, { migrate }, { Pool }] = await Promise.all([ + import('drizzle-orm/node-postgres'), + import('drizzle-orm/node-postgres/migrator'), + import('pg'), + ]); + const pool = new Pool({ connectionString: env.POSTGRES_URL }); + try { + await migrate(drizzle(pool), { + migrationsFolder: path.join(packageDir, 'migrations', 'postgres'), + }); + } finally { + await pool.end(); + } + return; + } + + const dbPath = path.join(workspaceRoot, 'docstore', 'sqlite3.db'); + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const [{ drizzle }, { migrate }, { default: BetterSqlite3 }] = await Promise.all([ + import('drizzle-orm/better-sqlite3'), + import('drizzle-orm/better-sqlite3/migrator'), + import('better-sqlite3'), + ]); + const sqlite = new BetterSqlite3(dbPath); + try { + migrate(drizzle(sqlite), { + migrationsFolder: path.join(packageDir, 'migrations', 'sqlite'), + }); + } finally { + sqlite.close(); + } +} + diff --git a/src/db/run-in-transaction.ts b/packages/database/src/run-in-transaction.ts similarity index 96% rename from src/db/run-in-transaction.ts rename to packages/database/src/run-in-transaction.ts index 1ec6ecd..8813e88 100644 --- a/src/db/run-in-transaction.ts +++ b/packages/database/src/run-in-transaction.ts @@ -1,4 +1,4 @@ -import { db } from '@/db'; +import { db } from './index'; /** * Run `fn` with a database connection, wrapped in a transaction on Postgres. diff --git a/src/db/schema.ts b/packages/database/src/schema.ts similarity index 100% rename from src/db/schema.ts rename to packages/database/src/schema.ts diff --git a/src/db/schema_auth_postgres.ts b/packages/database/src/schema_auth_postgres.ts similarity index 100% rename from src/db/schema_auth_postgres.ts rename to packages/database/src/schema_auth_postgres.ts diff --git a/src/db/schema_auth_sqlite.ts b/packages/database/src/schema_auth_sqlite.ts similarity index 100% rename from src/db/schema_auth_sqlite.ts rename to packages/database/src/schema_auth_sqlite.ts diff --git a/src/db/schema_postgres.ts b/packages/database/src/schema_postgres.ts similarity index 100% rename from src/db/schema_postgres.ts rename to packages/database/src/schema_postgres.ts diff --git a/src/db/schema_sqlite.ts b/packages/database/src/schema_sqlite.ts similarity index 100% rename from src/db/schema_sqlite.ts rename to packages/database/src/schema_sqlite.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6e0608..0ca9ff3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,6 @@ importers: '@headlessui/react': specifier: ^2.2.10 version: 2.2.10(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@huggingface/tokenizers': - specifier: ^0.1.3 - version: 0.1.3 '@microsoft/antissrf': specifier: ^1.0.0 version: 1.0.0 @@ -32,6 +29,9 @@ importers: '@napi-rs/canvas': specifier: ^0.1.100 version: 0.1.100 + '@openreader/database': + specifier: workspace:* + version: link:packages/database '@speech-sdk/core': specifier: ^0.15.0 version: 0.15.0 @@ -49,10 +49,7 @@ importers: version: 7.0.1 better-auth: specifier: ^1.6.14 - version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))) - better-sqlite3: - specifier: ^12.10.0 - version: 12.10.0 + version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))) cmpstr: specifier: ^3.3.0 version: 3.3.0 @@ -71,9 +68,6 @@ importers: dotenv: specifier: ^17.4.2 version: 17.4.2 - drizzle-kit: - specifier: ^0.31.10 - version: 0.31.10 drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0) @@ -101,18 +95,12 @@ importers: next: specifier: ^15.5.19 version: 15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - onnxruntime-node: - specifier: ^1.26.0 - version: 1.26.0 openai: specifier: ^6.41.0 version: 6.41.0(zod@4.4.3) pdfjs-dist: specifier: 4.8.69 version: 4.8.69 - pg: - specifier: ^8.21.0 - version: 8.21.0 pino: specifier: ^10.3.1 version: 10.3.1 @@ -170,7 +158,7 @@ importers: version: 1.60.0 '@tailwindcss/typography': specifier: ^0.5.19 - version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)) + version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0)) '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 @@ -198,33 +186,81 @@ importers: '@types/uuid': specifier: ^10.0.0 version: 10.0.0 + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 eslint: specifier: ^9.39.4 version: 9.39.4(jiti@1.21.7) eslint-config-next: specifier: ^15.5.19 version: 15.5.19(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3) + openapi-typescript: + specifier: ^7.10.1 + version: 7.13.0(typescript@5.9.3) postcss: specifier: ^8.5.15 version: 8.5.15 tailwindcss: specifier: ^3.4.19 - version: 3.4.19(tsx@4.22.4) + version: 3.4.19(tsx@4.22.4)(yaml@2.9.0) typescript: specifier: ^5.9.3 version: 5.9.3 vitest: specifier: ^4.1.8 - version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) - compute/core: + packages/bootstrap: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1061.0 + version: 3.1061.0 + '@openreader/database': + specifier: workspace:* + version: link:../database + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + dotenv: + specifier: ^17.4.2 + version: 17.4.2 + ffmpeg-static: + specifier: ^5.3.0 + version: 5.3.0 + pg: + specifier: ^8.21.0 + version: 8.21.0 + + packages/compute-worker: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1061.0 + version: 3.1061.0 + '@fastify/swagger': + specifier: ^9.0.0 + version: 9.7.0 '@huggingface/tokenizers': specifier: ^0.1.3 version: 0.1.3 '@napi-rs/canvas': specifier: ^0.1.100 version: 0.1.100 + '@nats-io/jetstream': + specifier: ^3.4.0 + version: 3.4.0 + '@nats-io/kv': + specifier: ^3.4.0 + version: 3.4.0 + '@nats-io/transport-node': + specifier: ^3.4.0 + version: 3.4.0 + fastify: + specifier: ^5.8.5 + version: 5.8.5 ffmpeg-static: specifier: ^5.3.0 version: 5.3.0 @@ -237,27 +273,6 @@ importers: pdfjs-dist: specifier: 4.8.69 version: 4.8.69 - - compute/worker: - dependencies: - '@aws-sdk/client-s3': - specifier: ^3.1061.0 - version: 3.1061.0 - '@nats-io/jetstream': - specifier: ^3.4.0 - version: 3.4.0 - '@nats-io/kv': - specifier: ^3.4.0 - version: 3.4.0 - '@nats-io/transport-node': - specifier: ^3.4.0 - version: 3.4.0 - '@openreader/compute-core': - specifier: workspace:* - version: link:../core - fastify: - specifier: ^5.8.5 - version: 5.8.5 pino: specifier: ^10.3.1 version: 10.3.1 @@ -271,26 +286,17 @@ importers: specifier: ^4.4.3 version: 4.4.3 - docker/entrypoint-migration-tools: + packages/database: dependencies: - '@aws-sdk/client-s3': - specifier: ^3.1061.0 - version: 3.1061.0 better-sqlite3: specifier: ^12.10.0 version: 12.10.0 dotenv: specifier: ^17.4.2 version: 17.4.2 - drizzle-kit: - specifier: ^0.31.10 - version: 0.31.10 drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0) - ffmpeg-static: - specifier: ^5.3.0 - version: 5.3.0 pg: specifier: ^8.21.0 version: 8.21.0 @@ -424,6 +430,14 @@ packages: resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} @@ -1031,6 +1045,9 @@ packages: '@fastify/proxy-addr@5.1.0': resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} + '@fastify/swagger@9.7.0': + resolution: {integrity: sha512-Vp1SC1GC2Hrkd3faFILv86BzUNyFz5N4/xdExqtCgkGASOzn/x+eMe4qXIGq7cdT6wif/P/oa6r1Ruqx19paZA==} + '@floating-ui/core@1.7.5': resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==} @@ -1508,6 +1525,16 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1 + '@redocly/ajv@8.11.2': + resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} + + '@redocly/config@0.22.0': + resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==} + + '@redocly/openapi-core@1.34.15': + resolution: {integrity: sha512-HAwCnNyKcs5XGQqms+9t7OdAPM/5TDstmhF+0i7tdCFato2QKuYIlyWETwkXd8c5zbltr1oB+6y9NTeQLr2d6Q==} + engines: {node: '>=18.17.0', npm: '>=9.5.0'} + '@rolldown/binding-android-arm64@1.0.2': resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2036,6 +2063,10 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -2050,6 +2081,10 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -2384,6 +2419,9 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + character-entities-html4@2.1.0: resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} @@ -2421,6 +2459,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@1.4.0: + resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} @@ -3011,6 +3052,9 @@ packages: resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true + fastify-plugin@5.1.0: + resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==} + fastify@5.8.5: resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==} @@ -3212,6 +3256,10 @@ packages: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -3234,6 +3282,10 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} + index-to-position@1.2.0: + resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} + engines: {node: '>=18'} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -3413,9 +3465,17 @@ packages: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + js-yaml@4.2.0: resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} hasBin: true @@ -3426,6 +3486,10 @@ packages: json-schema-ref-resolver@3.0.0: resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} + json-schema-resolver@3.0.0: + resolution: {integrity: sha512-HqMnbz0tz2DaEJ3ntsqtx3ezzZyDE7G56A/pPY/NGmrPu76UzsWquOpHFRAf5beTNXoH2LU5cQePVvRli1nchA==} + engines: {node: '>=20'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -3927,6 +3991,15 @@ packages: zod: optional: true + openapi-types@12.1.3: + resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + + openapi-typescript@7.13.0: + resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==} + hasBin: true + peerDependencies: + typescript: ^5.x + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3963,6 +4036,10 @@ packages: parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-json@8.3.0: + resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==} + engines: {node: '>=18'} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4073,6 +4150,10 @@ packages: engines: {node: '>=18'} hasBin: true + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -4615,6 +4696,10 @@ packages: suffix-thumb@5.0.2: resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -4728,6 +4813,10 @@ packages: resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} engines: {node: '>=10'} + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + type@2.7.3: resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==} @@ -4786,6 +4875,9 @@ packages: unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} + uri-js-replace@1.0.1: + resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -4943,6 +5035,18 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + yaml-ast-parser@0.0.43: + resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -5224,6 +5328,14 @@ snapshots: '@aws/lambda-invoke-store@0.2.4': {} + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/runtime@7.29.7': {} '@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)': @@ -5546,7 +5658,7 @@ snapshots: '@eslint/config-array@0.21.2': dependencies: '@eslint/object-schema': 2.1.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) minimatch: 3.1.5 transitivePeerDependencies: - supports-color @@ -5562,7 +5674,7 @@ snapshots: '@eslint/eslintrc@3.3.5': dependencies: ajv: 6.15.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) espree: 10.4.0 globals: 14.0.0 ignore: 5.3.2 @@ -5605,6 +5717,16 @@ snapshots: '@fastify/forwarded': 3.0.1 ipaddr.js: 2.4.0 + '@fastify/swagger@9.7.0': + dependencies: + fastify-plugin: 5.1.0 + json-schema-resolver: 3.0.0 + openapi-types: 12.1.3 + rfdc: 1.4.1 + yaml: 2.9.0 + transitivePeerDependencies: + - supports-color + '@floating-ui/core@1.7.5': dependencies: '@floating-ui/utils': 0.2.11 @@ -5968,6 +6090,29 @@ snapshots: dependencies: react: 19.2.7 + '@redocly/ajv@8.11.2': + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js-replace: 1.0.1 + + '@redocly/config@0.22.0': {} + + '@redocly/openapi-core@1.34.15(supports-color@10.2.2)': + dependencies: + '@redocly/ajv': 8.11.2 + '@redocly/config': 0.22.0 + colorette: 1.4.0 + https-proxy-agent: 7.0.6(supports-color@10.2.2) + js-levenshtein: 1.1.6 + js-yaml: 4.1.1 + minimatch: 5.1.9 + pluralize: 8.0.0 + yaml-ast-parser: 0.0.43 + transitivePeerDependencies: + - supports-color + '@rolldown/binding-android-arm64@1.0.2': optional: true @@ -6088,10 +6233,10 @@ snapshots: dependencies: tslib: 2.8.1 - '@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4))': + '@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))': dependencies: postcss-selector-parser: 6.0.10 - tailwindcss: 3.4.19(tsx@4.22.4) + tailwindcss: 3.4.19(tsx@4.22.4)(yaml@2.9.0) '@tanstack/query-core@5.101.0': {} @@ -6220,7 +6365,7 @@ snapshots: '@typescript-eslint/types': 8.60.1 '@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) typescript: 5.9.3 transitivePeerDependencies: @@ -6230,7 +6375,7 @@ snapshots: dependencies: '@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3) '@typescript-eslint/types': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -6249,7 +6394,7 @@ snapshots: '@typescript-eslint/types': 8.60.1 '@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3) '@typescript-eslint/utils': 8.60.1(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3) - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 @@ -6264,7 +6409,7 @@ snapshots: '@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3) '@typescript-eslint/types': 8.60.1 '@typescript-eslint/visitor-keys': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) minimatch: 10.2.5 semver: 7.8.1 tinyglobby: 0.2.17 @@ -6375,13 +6520,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))': + '@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.8 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4) + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0) '@vitest/pretty-format@4.1.8': dependencies: @@ -6425,10 +6570,12 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + agent-base@7.1.4: {} + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -6447,6 +6594,8 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ansi-colors@4.1.3: {} + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -6632,7 +6781,7 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))): + better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))): dependencies: '@better-auth/core': 1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/drizzle-adapter': 1.6.14(@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.1)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0)) @@ -6659,7 +6808,7 @@ snapshots: pg: 8.21.0 react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) transitivePeerDependencies: - '@cloudflare/workers-types' - '@opentelemetry/api' @@ -6765,6 +6914,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + change-case@5.4.4: {} + character-entities-html4@2.1.0: {} character-entities-legacy@3.0.0: {} @@ -6799,6 +6950,8 @@ snapshots: color-name@1.1.4: {} + colorette@1.4.0: {} + colorette@2.0.20: {} comma-separated-tokens@2.0.3: {} @@ -6896,9 +7049,11 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.4.3: + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 decode-named-character-reference@1.3.0: dependencies: @@ -7265,7 +7420,7 @@ snapshots: eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@1.21.7)): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) get-tsconfig: 4.14.0 is-bun-module: 2.0.0 @@ -7390,7 +7545,7 @@ snapshots: ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) escape-string-regexp: 4.0.0 eslint-scope: 8.4.0 eslint-visitor-keys: 4.2.1 @@ -7535,6 +7690,8 @@ snapshots: strnum: 2.3.0 xml-naming: 0.1.0 + fastify-plugin@5.1.0: {} + fastify@5.8.5: dependencies: '@fastify/ajv-compiler': 4.0.5 @@ -7777,7 +7934,14 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6(supports-color@10.2.2): + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -7796,6 +7960,8 @@ snapshots: imurmurhash@0.1.4: {} + index-to-position@1.2.0: {} + inherits@2.0.4: {} ini@1.3.8: {} @@ -7972,8 +8138,14 @@ snapshots: joycon@3.1.1: {} + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + js-yaml@4.2.0: dependencies: argparse: 2.0.1 @@ -7984,6 +8156,14 @@ snapshots: dependencies: dequal: 2.0.3 + json-schema-resolver@3.0.0: + dependencies: + debug: 4.4.3(supports-color@10.2.2) + fast-uri: 3.1.2 + rfdc: 1.4.1 + transitivePeerDependencies: + - supports-color + json-schema-traverse@0.4.1: {} json-schema-traverse@1.0.0: {} @@ -8482,7 +8662,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -8658,6 +8838,18 @@ snapshots: optionalDependencies: zod: 4.4.3 + openapi-types@12.1.3: {} + + openapi-typescript@7.13.0(typescript@5.9.3): + dependencies: + '@redocly/openapi-core': 1.34.15(supports-color@10.2.2) + ansi-colors: 4.1.3 + change-case: 5.4.4 + parse-json: 8.3.0 + supports-color: 10.2.2 + typescript: 5.9.3 + yargs-parser: 21.1.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -8705,6 +8897,12 @@ snapshots: is-decimal: 2.0.1 is-hexadecimal: 2.0.1 + parse-json@8.3.0: + dependencies: + '@babel/code-frame': 7.29.7 + index-to-position: 1.2.0 + type-fest: 4.41.0 + path-exists@4.0.0: {} path-expression-matcher@1.5.0: {} @@ -8819,6 +9017,8 @@ snapshots: optionalDependencies: fsevents: 2.3.2 + pluralize@8.0.0: {} + possible-typed-array-names@1.1.0: {} postcss-import@15.1.0(postcss@8.5.15): @@ -8833,13 +9033,14 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.5.15 - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 postcss: 8.5.15 tsx: 4.22.4 + yaml: 2.9.0 postcss-nested@6.2.0(postcss@8.5.15): dependencies: @@ -9504,6 +9705,8 @@ snapshots: suffix-thumb@5.0.2: {} + supports-color@10.2.2: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -9512,7 +9715,7 @@ snapshots: tabbable@6.4.0: {} - tailwindcss@3.4.19(tsx@4.22.4): + tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0): dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -9531,7 +9734,7 @@ snapshots: postcss: 8.5.15 postcss-import: 15.1.0(postcss@8.5.15) postcss-js: 4.1.0(postcss@8.5.15) - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0) postcss-nested: 6.2.0(postcss@8.5.15) postcss-selector-parser: 6.1.2 resolve: 1.22.12 @@ -9651,6 +9854,8 @@ snapshots: type-fest@0.20.2: {} + type-fest@4.41.0: {} + type@2.7.3: {} typed-array-buffer@1.0.3: @@ -9761,6 +9966,8 @@ snapshots: '@unrs/resolver-binding-win32-ia32-msvc': 1.12.2 '@unrs/resolver-binding-win32-x64-msvc': 1.12.2 + uri-js-replace@1.0.1: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 @@ -9783,7 +9990,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4): + vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -9796,11 +10003,12 @@ snapshots: fsevents: 2.3.3 jiti: 1.21.7 tsx: 4.22.4 + yaml: 2.9.0 - vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)): + vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.8 - '@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + '@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.8 '@vitest/runner': 4.1.8 '@vitest/snapshot': 4.1.8 @@ -9817,7 +10025,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4) + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.41 @@ -9898,6 +10106,12 @@ snapshots: xtend@4.0.2: {} + yaml-ast-parser@0.0.43: {} + + yaml@2.9.0: {} + + yargs-parser@21.1.1: {} + yocto-queue@0.1.0: {} zip-stream@6.0.1: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fcc6ecc..d7d7a9f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,6 @@ packages: - . - - compute/* - - docker/* + - packages/* allowBuilds: '@napi-rs/canvas': true diff --git a/scripts/check-compute-boundary.mjs b/scripts/check-compute-boundary.mjs new file mode 100644 index 0000000..62ad85b --- /dev/null +++ b/scripts/check-compute-boundary.mjs @@ -0,0 +1,39 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const root = process.cwd(); +const forbidden = [ + '@openreader/compute-core', + '@openreader/compute-worker', + 'compute/core', + 'compute/worker', + 'packages/compute-worker/src', +]; +const failures = []; + +function walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + continue; + } + if (!entry.isFile() || !/\.(ts|tsx|js|mjs|cjs)$/.test(entry.name)) continue; + const text = fs.readFileSync(fullPath, 'utf8'); + for (const pattern of forbidden) { + if (text.includes(pattern)) failures.push({ file: fullPath, pattern }); + } + } +} + +walk(path.join(root, 'src')); + +if (failures.length > 0) { + console.error('[compute-boundary] Forbidden app-to-worker source coupling detected:'); + for (const failure of failures) { + console.error(`- ${failure.pattern} in ${path.relative(root, failure.file)}`); + } + process.exit(1); +} + +console.info('[compute-boundary] OK: app communicates with compute worker through protocol/client only'); diff --git a/scripts/check-next-server-bundle.mjs b/scripts/check-next-server-bundle.mjs index 9621cb4..795c60b 100644 --- a/scripts/check-next-server-bundle.mjs +++ b/scripts/check-next-server-bundle.mjs @@ -12,7 +12,8 @@ if (!fs.existsSync(serverDir)) { const forbidden = [ 'onnxruntime-node', '@huggingface/tokenizers', - '@openreader/compute-core/local-runtime', + '@openreader/compute-worker', + '/packages/compute-worker/src/inference/', ]; const includeExt = new Set(['.js', '.mjs', '.cjs']); diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 7aa881a..f080fbe 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -5,8 +5,8 @@ import { tmpdir } from 'os'; import { join } from 'path'; import { randomUUID } from 'crypto'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; diff --git a/src/app/api/audiobook/route.ts b/src/app/api/audiobook/route.ts index 336c772..a2955dc 100644 --- a/src/app/api/audiobook/route.ts +++ b/src/app/api/audiobook/route.ts @@ -4,8 +4,8 @@ import { mkdtemp, readFile, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; diff --git a/src/app/api/audiobook/status/route.ts b/src/app/api/audiobook/status/route.ts index 6416dc3..14519ca 100644 --- a/src/app/api/audiobook/status/route.ts +++ b/src/app/api/audiobook/status/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getAudiobookObjectBuffer, isMissingBlobError, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore'; import { decodeChapterFileName } from '@/lib/server/audiobooks/chapters'; diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index a0d3dfd..2a49029 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -1,9 +1,9 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { fetchPdfParseOperation, @@ -45,14 +45,6 @@ async function loadOwnedDocumentRow(input: { return rows[0] ?? null; } -function workerEventsUrl(baseUrl: string, opId: string, sinceEventId: string | null): string { - const url = new URL(`${baseUrl}/ops/${encodeURIComponent(opId)}/events`); - if (sinceEventId) { - url.searchParams.set('sinceEventId', sinceEventId); - } - return url.toString(); -} - export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { const { logger } = createRequestLogger({ route: '/api/documents/[id]/parsed/events', @@ -92,17 +84,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Operation not found' }, { status: 404 }); } - const cfg = getWorkerClientConfigFromEnv(); const lastEventId = req.headers.get('last-event-id'); const sinceEventId = req.nextUrl.searchParams.get('sinceEventId') || lastEventId; - const upstream = await fetch(workerEventsUrl(cfg.baseUrl, opId, sinceEventId), { - method: 'GET', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'text/event-stream', - ...(lastEventId ? { 'Last-Event-ID': lastEventId } : {}), - }, - cache: 'no-store', + const upstream = await getComputeWorkerClient().openOperationEvents(opId, { + sinceEventId, + lastEventId, signal: req.signal, }); diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 0234ade..898dac7 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -1,8 +1,8 @@ import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { diff --git a/src/app/api/documents/[id]/settings/route.ts b/src/app/api/documents/[id]/settings/route.ts index 0b2b282..f618022 100644 --- a/src/app/api/documents/[id]/settings/route.ts +++ b/src/app/api/documents/[id]/settings/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { documentSettings, documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documentSettings, documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { mergeDocumentSettings } from '@/lib/shared/document-settings'; import { DEFAULT_DOCUMENT_SETTINGS, type DocumentSettings } from '@/types/document-settings'; diff --git a/src/app/api/documents/blob/get/fallback/route.ts b/src/app/api/documents/blob/get/fallback/route.ts index 0db67e0..7ca9006 100644 --- a/src/app/api/documents/blob/get/fallback/route.ts +++ b/src/app/api/documents/blob/get/fallback/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { contentTypeForName } from '@/lib/server/storage/library-mount'; import { getDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; diff --git a/src/app/api/documents/blob/get/presign/route.ts b/src/app/api/documents/blob/get/presign/route.ts index 4992354..685e4dd 100644 --- a/src/app/api/documents/blob/get/presign/route.ts +++ b/src/app/api/documents/blob/get/presign/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId, presignGet } from '@/lib/server/documents/blobstore'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; diff --git a/src/app/api/documents/blob/preview/fallback/route.ts b/src/app/api/documents/blob/preview/fallback/route.ts index 004aa74..4341a08 100644 --- a/src/app/api/documents/blob/preview/fallback/route.ts +++ b/src/app/api/documents/blob/preview/fallback/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; diff --git a/src/app/api/documents/blob/preview/utils.ts b/src/app/api/documents/blob/preview/utils.ts index 937942f..ad19bce 100644 --- a/src/app/api/documents/blob/preview/utils.ts +++ b/src/app/api/documents/blob/preview/utils.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { isPreviewableDocumentType } from '@/lib/server/documents/previews'; diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index c7d21d8..328591f 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorToLog, serverLogger } from '@/lib/server/logger'; diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index 28d7a0a..f93e551 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents, ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents, ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; import { isS3Configured, getS3Config } from '@/lib/server/storage/s3'; import { generateTTSBuffer } from '@/lib/server/tts/generate'; import { diff --git a/src/app/api/tts/segments/manifest/route.ts b/src/app/api/tts/segments/manifest/route.ts index f33155d..89621f3 100644 --- a/src/app/api/tts/segments/manifest/route.ts +++ b/src/app/api/tts/segments/manifest/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, asc, eq, gt, inArray, or } from 'drizzle-orm'; -import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; import { decodeManifestCursor, diff --git a/src/app/api/user/claim/route.ts b/src/app/api/user/claim/route.ts index 5745cd7..8d4b918 100644 --- a/src/app/api/user/claim/route.ts +++ b/src/app/api/user/claim/route.ts @@ -1,8 +1,8 @@ import { NextRequest, NextResponse } from 'next/server'; import { claimAnonymousData } from '@/lib/server/user/claim-data'; import { auth } from '@/lib/server/auth/auth'; -import { db } from '@/db'; -import { audiobooks, documentSettings, documents, userDocumentProgress, userPreferences } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, documentSettings, documents, userDocumentProgress, userPreferences } from '@openreader/database/schema'; import { count, eq, ne } from 'drizzle-orm'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { errorToLog, serverLogger } from '@/lib/server/logger'; diff --git a/src/app/api/user/export/route.ts b/src/app/api/user/export/route.ts index e65b035..eae1047 100644 --- a/src/app/api/user/export/route.ts +++ b/src/app/api/user/export/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { PassThrough, Readable } from 'stream'; import { auth } from '@/lib/server/auth/auth'; -import { db } from '@/db'; +import { db } from '@openreader/database'; import { documents, audiobooks, @@ -13,9 +13,9 @@ import { userJobEvents, userPreferences, userTtsChars, -} from '@/db/schema'; -import * as authSchemaSqlite from '@/db/schema_auth_sqlite'; -import * as authSchemaPostgres from '@/db/schema_auth_postgres'; +} from '@openreader/database/schema'; +import * as authSchemaSqlite from '@openreader/database/schema-auth-sqlite'; +import * as authSchemaPostgres from '@openreader/database/schema-auth-postgres'; import { and, desc, eq, inArray } from 'drizzle-orm'; import archiver from 'archiver'; import { appendUserExportArchive } from '@/lib/server/user/data-export'; diff --git a/src/app/api/user/state/changelog/version-check/route.ts b/src/app/api/user/state/changelog/version-check/route.ts index b36d74f..498467b 100644 --- a/src/app/api/user/state/changelog/version-check/route.ts +++ b/src/app/api/user/state/changelog/version-check/route.ts @@ -1,7 +1,7 @@ import { eq } from 'drizzle-orm'; import { NextRequest, NextResponse } from 'next/server'; -import { db } from '@/db'; -import { userPreferences } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userPreferences } from '@openreader/database/schema'; import { normalizeVersion, shouldOpenChangelogForVersionChange } from '@/lib/shared/changelog'; import { nowTimestampMs } from '@/lib/shared/timestamps'; import { resolveUserStateScope } from '@/lib/server/user/resolve-state-scope'; diff --git a/src/app/api/user/state/preferences/route.ts b/src/app/api/user/state/preferences/route.ts index 9eeae34..4c81de2 100644 --- a/src/app/api/user/state/preferences/route.ts +++ b/src/app/api/user/state/preferences/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { userPreferences } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userPreferences } from '@openreader/database/schema'; import { type SyncedPreferencesPatch } from '@/types/user-state'; import { resolveUserStateScope } from '@/lib/server/user/resolve-state-scope'; import { coerceTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; diff --git a/src/app/api/user/state/progress/route.ts b/src/app/api/user/state/progress/route.ts index 1fef9f5..b365588 100644 --- a/src/app/api/user/state/progress/route.ts +++ b/src/app/api/user/state/progress/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { userDocumentProgress } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userDocumentProgress } from '@openreader/database/schema'; import type { ReaderType } from '@/types/user-state'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { resolveUserStateScope } from '@/lib/server/user/resolve-state-scope'; diff --git a/src/lib/server/admin/email-sync.ts b/src/lib/server/admin/email-sync.ts index 105ec17..1c9696b 100644 --- a/src/lib/server/admin/email-sync.ts +++ b/src/lib/server/admin/email-sync.ts @@ -1,5 +1,5 @@ import { eq } from 'drizzle-orm'; -import { db } from '@/db'; +import { db } from '@openreader/database'; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; @@ -7,9 +7,9 @@ import { logDegraded } from '@/lib/server/errors/logging'; // we import them lazily to avoid coupling this module to a single dialect. function getUserTable() { // eslint-disable-next-line @typescript-eslint/no-require-imports - const sqlite = require('@/db/schema_auth_sqlite'); + const sqlite = require('@openreader/database/schema-auth-sqlite'); // eslint-disable-next-line @typescript-eslint/no-require-imports - const pg = require('@/db/schema_auth_postgres'); + const pg = require('@openreader/database/schema-auth-postgres'); return process.env.POSTGRES_URL ? pg.user : sqlite.user; } diff --git a/src/lib/server/admin/providers.ts b/src/lib/server/admin/providers.ts index a562c9c..9befbb9 100644 --- a/src/lib/server/admin/providers.ts +++ b/src/lib/server/admin/providers.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, asc, desc, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { adminProviders } from '@/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders } from '@openreader/database/schema'; import { apiKeyLast4, decryptSecret, encryptSecret } from '@/lib/server/crypto/secrets'; import { type TtsProviderId } from '@/lib/shared/tts-provider-catalog'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; diff --git a/src/lib/server/admin/seed.ts b/src/lib/server/admin/seed.ts index 0409299..f120f62 100644 --- a/src/lib/server/admin/seed.ts +++ b/src/lib/server/admin/seed.ts @@ -1,5 +1,5 @@ -import { db } from '@/db'; -import { adminProviders, adminSettings } from '@/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders, adminSettings } from '@openreader/database/schema'; import { encryptSecret, apiKeyLast4 } from '@/lib/server/crypto/secrets'; import { randomUUID } from 'node:crypto'; import { and, eq } from 'drizzle-orm'; diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index 99e1b87..4da8900 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -1,6 +1,6 @@ import { asc, desc, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { adminProviders, adminSettings } from '@/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders, adminSettings } from '@openreader/database/schema'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; diff --git a/src/lib/server/audiobooks/prune.ts b/src/lib/server/audiobooks/prune.ts index 35d6669..6f1e3bb 100644 --- a/src/lib/server/audiobooks/prune.ts +++ b/src/lib/server/audiobooks/prune.ts @@ -1,7 +1,7 @@ import { and, eq, notInArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; export async function pruneAudiobookIfMissingDir(bookId: string, userId: string, intermediateDirExists: boolean): Promise { if (intermediateDirExists) return; diff --git a/src/lib/server/auth/auth.ts b/src/lib/server/auth/auth.ts index d0359d1..c9345c5 100644 --- a/src/lib/server/auth/auth.ts +++ b/src/lib/server/auth/auth.ts @@ -4,13 +4,13 @@ import { anonymous } from "better-auth/plugins"; import { drizzleAdapter } from "better-auth/adapters/drizzle"; import { NextResponse } from 'next/server'; import type { NextRequest } from 'next/server'; -import { db } from "@/db"; +import { db } from "@openreader/database"; import { getRequiredAuthEnv, isAnonymousAuthSessionsEnabled } from "@/lib/server/auth/config"; import { isAdminEmail, syncAdminFlag } from "@/lib/server/admin/email-sync"; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { assertUserSignupAllowed } from '@/lib/server/auth/signup-policy'; -import * as authSchemaSqlite from "@/db/schema_auth_sqlite"; -import * as authSchemaPostgres from "@/db/schema_auth_postgres"; +import * as authSchemaSqlite from "@openreader/database/schema-auth-sqlite"; +import * as authSchemaPostgres from "@openreader/database/schema-auth-postgres"; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded, logServerError } from '@/lib/server/errors/logging'; import { tryGetOrigin } from "@/lib/shared/urls"; diff --git a/src/lib/server/compute-worker/client.ts b/src/lib/server/compute-worker/client.ts new file mode 100644 index 0000000..a8f35bf --- /dev/null +++ b/src/lib/server/compute-worker/client.ts @@ -0,0 +1,249 @@ +import type { + PdfLayoutRequest, + PdfLayoutResolution, + PdfLayoutResult, + WhisperAlignRequest, + WhisperAlignResult, + ComputeOperation, + ComputeOperationEvent, +} from './protocol'; +import { isComputeOperation } from './protocol'; + +class WorkerHttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly retryAfterMs: number | null = null, + ) { + super(message); + this.name = 'WorkerHttpError'; + } +} + +function readRequiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required for compute worker client`); + return value; +} + +function normalizeWorkerBaseUrl(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty'); + const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed) + ? trimmed + : (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed) + ? `http://${trimmed}` + : `https://${trimmed}`); + const parsed = new URL(withScheme); + parsed.pathname = parsed.pathname.replace(/\/+$/, ''); + return parsed.toString().replace(/\/+$/, ''); +} + +export function getComputeWorkerConfigFromEnv(): { baseUrl: string; token: string } { + return { + baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')), + token: readRequiredEnv('COMPUTE_WORKER_TOKEN'), + }; +} + +export function isComputeWorkerAvailable(): boolean { + try { + getComputeWorkerConfigFromEnv(); + return true; + } catch { + return false; + } +} + +function readPositiveIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; +} + +function alignmentWaitTimeoutMs(): number { + return Math.max(60_000, readPositiveIntEnv('COMPUTE_WHISPER_TIMEOUT_MS', 30_000) + 15_000); +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const seconds = Number(value); + if (Number.isFinite(seconds)) return Math.max(0, Math.floor(seconds * 1000)); + const date = Date.parse(value); + return Number.isNaN(date) ? null : Math.max(0, date - Date.now()); +} + +function parseSseFrame(frame: string): { id: number | null; data: string | null } { + let id: number | null = null; + const data: string[] = []; + for (const line of frame.split(/\r?\n/)) { + if (line.startsWith('id:')) { + const parsed = Number(line.slice(3).trim()); + if (Number.isFinite(parsed) && parsed >= 0) id = Math.floor(parsed); + } else if (line.startsWith('data:')) { + data.push(line.slice(5).trimStart()); + } + } + return { id, data: data.length > 0 ? data.join('\n') : null }; +} + +function isTerminal(operation: ComputeOperation): boolean { + return operation.status === 'succeeded' || operation.status === 'failed'; +} + +export class ComputeWorkerClient { + private readonly baseUrl: string; + private readonly token: string; + + constructor(config = getComputeWorkerConfigFromEnv()) { + this.baseUrl = config.baseUrl; + this.token = config.token; + } + + async alignWords(input: WhisperAlignRequest): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= 2; attempt += 1) { + try { + const operation = await this.requestJson>( + 'POST', + '/v1/whisper-align/operations', + input, + ); + const final = isTerminal(operation) + ? operation + : await this.waitForOperation(operation.opId, alignmentWaitTimeoutMs()); + if (final.status !== 'succeeded' || !final.result) { + throw new Error(final.error?.message || 'Whisper worker operation did not complete'); + } + return final.result; + } catch (error) { + lastError = error; + if (attempt >= 2 || !this.shouldRetry(error)) break; + const delay = error instanceof WorkerHttpError && error.retryAfterMs !== null + ? error.retryAfterMs + : attempt * 250; + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw lastError instanceof Error ? lastError : new Error('Unknown compute worker failure'); + } + + createPdfLayoutOperation(input: PdfLayoutRequest): Promise> { + return this.requestJson('POST', '/v1/pdf-layout/operations', input); + } + + resolvePdfLayout(input: PdfLayoutRequest): Promise { + return this.requestJson('POST', '/v1/pdf-layout/resolve', { + documentId: input.documentId, + namespace: input.namespace, + documentObjectKey: input.documentObjectKey, + }); + } + + async getOperation(opId: string): Promise | null> { + try { + return await this.requestJson('GET', `/v1/operations/${encodeURIComponent(opId)}`); + } catch (error) { + if (error instanceof WorkerHttpError && error.status === 404) return null; + throw error; + } + } + + async openOperationEvents(opId: string, input?: { + sinceEventId?: string | null; + lastEventId?: string | null; + signal?: AbortSignal; + }): Promise { + const url = new URL(`${this.baseUrl}/v1/operations/${encodeURIComponent(opId)}/events`); + if (input?.sinceEventId) url.searchParams.set('sinceEventId', input.sinceEventId); + return fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${this.token}`, + Accept: 'text/event-stream', + ...(input?.lastEventId ? { 'Last-Event-ID': input.lastEventId } : {}), + }, + cache: 'no-store', + signal: input?.signal, + }); + } + + private async requestJson(method: 'GET' | 'POST', path: string, body?: unknown): Promise { + const response = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.token}`, + Accept: 'application/json', + ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), + }, + ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), + cache: 'no-store', + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new WorkerHttpError( + `Worker request failed (${method} ${path}): ${response.status}${detail ? ` ${detail}` : ''}`, + response.status, + parseRetryAfterMs(response.headers.get('retry-after')), + ); + } + const parsed = await response.json() as T; + return parsed; + } + + private shouldRetry(error: unknown): boolean { + if (error instanceof WorkerHttpError) { + return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; + } + return error instanceof Error && /network|timeout|fetch failed/i.test(error.message); + } + + private async waitForOperation(opId: string, timeoutMs: number): Promise> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let lastEventId: number | null = null; + try { + while (!controller.signal.aborted) { + const response = await this.openOperationEvents(opId, { + sinceEventId: lastEventId === null ? null : String(lastEventId), + lastEventId: lastEventId === null ? null : String(lastEventId), + signal: controller.signal, + }); + if (!response.ok || !response.body) { + throw new WorkerHttpError(`Worker event stream failed: ${response.status}`, response.status); + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + while (true) { + const end = buffer.indexOf('\n\n'); + if (end < 0) break; + const frame = parseSseFrame(buffer.slice(0, end)); + buffer = buffer.slice(end + 2); + if (frame.id !== null) lastEventId = frame.id; + if (!frame.data) continue; + const event = JSON.parse(frame.data) as ComputeOperationEvent; + if (!event?.snapshot || !isComputeOperation(event.snapshot) || event.snapshot.opId !== opId) continue; + if (isTerminal(event.snapshot)) return event.snapshot; + } + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`Operation stream timed out for ${opId}`); + } finally { + clearTimeout(timeout); + } + } +} + +let client: ComputeWorkerClient | null = null; + +export function getComputeWorkerClient(): ComputeWorkerClient { + client ??= new ComputeWorkerClient(); + return client; +} diff --git a/src/lib/server/compute-worker/generated.ts b/src/lib/server/compute-worker/generated.ts new file mode 100644 index 0000000..cdac950 --- /dev/null +++ b/src/lib/server/compute-worker/generated.ts @@ -0,0 +1,700 @@ +/** + * This file was auto-generated by openapi-typescript. + * Do not make direct changes to the file. + */ + +export interface paths { + "/health/live": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + ok: boolean; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/health/ready": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + ok: boolean; + natsConnected: boolean; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/whisper-align/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + text: string; + lang?: string; + cacheKey?: string; + audioObjectKey: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/pdf-layout/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + documentId: string; + namespace: string | null; + documentObjectKey: string; + replaceToken?: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/pdf-layout/resolve": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + documentId: string; + namespace: string | null; + documentObjectKey: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + artifact: { + objectKey: string; + } | null; + operation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + } | null; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/operations/{opId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path: { + opId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + /** @description Default Response */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/operations/{opId}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: { + sinceEventId?: string | number; + }; + header?: never; + path: { + opId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Server-sent ComputeOperationEvent stream */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": string; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + /** @description Default Response */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + ParsedPdfDocument: { + /** @enum {number} */ + schemaVersion: 1; + documentId: string; + parserVersion: string; + parsedAt: number; + pages: { + pageNumber: number; + width: number; + height: number; + blocks: { + id: string; + /** @enum {string} */ + kind: "abstract" | "algorithm" | "aside_text" | "chart" | "content" | "formula" | "doc_title" | "figure_title" | "footer" | "footnote" | "formula_number" | "header" | "image" | "number" | "paragraph_title" | "reference" | "reference_content" | "seal" | "table" | "text" | "vision_footnote"; + fragments: { + page: number; + bbox: [ + number, + number, + number, + number + ]; + text: string; + readingOrder: number; + modelConfidence?: number; + }[]; + text: string; + parentSectionId?: string; + }[]; + }[]; + }; + TTSSentenceAlignment: { + sentence: string; + sentenceIndex: number; + words: { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; + }[]; + }; + ArtifactReference: { + objectKey: string; + }; + ErrorResponse: { + error: string; + } & { + [key: string]: unknown; + }; + OperationError: { + message: string; + code?: string; + }; + PdfLayoutProgress: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + ComputeOperation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + ComputeOperationEvent: { + eventId: number; + snapshot: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + PdfLayoutResolution: { + artifact: { + objectKey: string; + } | null; + operation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + } | null; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export type operations = Record; diff --git a/src/lib/server/compute-worker/protocol.ts b/src/lib/server/compute-worker/protocol.ts new file mode 100644 index 0000000..2fa0c83 --- /dev/null +++ b/src/lib/server/compute-worker/protocol.ts @@ -0,0 +1,57 @@ +import type { components, paths } from './generated'; + +export type ParsedPdfDocument = components['schemas']['ParsedPdfDocument']; +export type ParsedPdfPage = ParsedPdfDocument['pages'][number]; +export type ParsedPdfBlock = ParsedPdfPage['blocks'][number]; +export type ParsedPdfBlockKind = ParsedPdfBlock['kind']; +export type ParsedPdfBlockFragment = ParsedPdfBlock['fragments'][number]; + +export type TTSSentenceAlignment = components['schemas']['TTSSentenceAlignment']; +export type TTSSentenceWord = TTSSentenceAlignment['words'][number]; + +export type PdfLayoutProgress = NonNullable; +export type ComputeOperationStatus = components['schemas']['ComputeOperation']['status']; +export type ComputeOperationSubject = components['schemas']['ComputeOperation']['subject']; + +export type ComputeOperation = + Omit + & { result?: Result }; + +export type ComputeOperationEvent = { + eventId: number; + snapshot: ComputeOperation; +}; + +export type WhisperAlignRequest = + paths['/v1/whisper-align/operations']['post']['requestBody']['content']['application/json']; +export type PdfLayoutRequest = + paths['/v1/pdf-layout/operations']['post']['requestBody']['content']['application/json']; +export type PdfLayoutResolveRequest = + paths['/v1/pdf-layout/resolve']['post']['requestBody']['content']['application/json']; + +export type WhisperAlignResult = { + alignments: TTSSentenceAlignment[]; + timing?: components['schemas']['ComputeOperation']['timing']; +}; + +export type PdfLayoutResult = { + parsedObjectKey: string; + timing?: components['schemas']['ComputeOperation']['timing']; +}; + +export type PdfLayoutResolution = { + artifact: { objectKey: string } | null; + operation: ComputeOperation | null; +}; + +export function isComputeOperation(value: unknown): value is ComputeOperation { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + return typeof record.opId === 'string' + && !!record.subject + && typeof record.subject === 'object' + && typeof (record.subject as Record).kind === 'string' + && typeof record.status === 'string' + && typeof record.queuedAt === 'number' + && typeof record.updatedAt === 'number'; +} diff --git a/src/lib/server/compute/abort-like-error.ts b/src/lib/server/compute/abort-like-error.ts deleted file mode 100644 index 2ce848b..0000000 --- a/src/lib/server/compute/abort-like-error.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function isAbortLikeError(error: unknown): boolean { - if (!error) return false; - if (error instanceof DOMException) return error.name === 'AbortError'; - if (error instanceof Error) return error.name === 'AbortError' || error.message === 'This operation was aborted'; - if (typeof error === 'object' && error !== null && 'name' in error) { - return (error as { name?: unknown }).name === 'AbortError'; - } - return false; -} diff --git a/src/lib/server/compute/index.ts b/src/lib/server/compute/index.ts deleted file mode 100644 index ac62a56..0000000 --- a/src/lib/server/compute/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { ComputeBackend } from '@/lib/server/compute/types'; -import { isWorkerClientConfigAvailable, WorkerComputeBackend } from '@/lib/server/compute/worker'; - -let backendPromise: Promise | null = null; - -export async function getCompute(): Promise { - if (!backendPromise) { - backendPromise = Promise.resolve(new WorkerComputeBackend()).catch((error) => { - backendPromise = null; - throw error; - }); - } - return backendPromise; -} - -export function isComputeAvailable(): boolean { - return isWorkerClientConfigAvailable(); -} diff --git a/src/lib/server/compute/types.ts b/src/lib/server/compute/types.ts deleted file mode 100644 index ffd0951..0000000 --- a/src/lib/server/compute/types.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { - TTSAudioBuffer, - TTSSentenceAlignment, - ParsedPdfDocument, -} from '@openreader/compute-core/types'; -import type { PdfLayoutProgress, WhisperAlignJobBase } from '@openreader/compute-core/api-contracts'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; - -export interface WhisperAlignInput extends WhisperAlignJobBase { - audioBuffer?: TTSAudioBuffer; - audioObjectKey?: string; -} - -export interface WhisperAlignResult { - alignments: TTSSentenceAlignment[]; -} - -export interface PdfLayoutInput { - documentId: string; - namespace?: string | null; - documentObjectKey?: string; - pdfBytes?: ArrayBuffer; - forceToken?: string; - onProgress?: (progress: PdfLayoutProgress) => void | Promise; - onWorkerSnapshot?: (snapshot: WorkerOperationState) => void | Promise; -} - -export type PdfLayoutResult = - | { parsed: ParsedPdfDocument; parsedObjectKey?: never } - | { parsed?: never; parsedObjectKey: string }; - -export interface ComputeBackend { - mode: 'worker'; - alignWords(input: WhisperAlignInput): Promise; - parsePdfLayout(input: PdfLayoutInput): Promise; -} diff --git a/src/lib/server/compute/worker-op-create.ts b/src/lib/server/compute/worker-op-create.ts deleted file mode 100644 index 433b68e..0000000 --- a/src/lib/server/compute/worker-op-create.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { getWorkerClientConfigFromEnv, buildPdfOpKey } from '@/lib/server/compute/worker'; -import type { PdfLayoutInput } from '@/lib/server/compute/types'; -import type { - PdfLayoutJobResult, - WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; - -type CreatePdfWorkerOpInput = - Pick - & { documentObjectKey: string }; - -const CREATE_OP_TIMEOUT_MS = 10_000; - -export async function createOrReusePdfWorkerOperation( - input: CreatePdfWorkerOpInput, -): Promise> { - const cfg = getWorkerClientConfigFromEnv(); - const opKey = buildPdfOpKey({ - documentId: input.documentId, - namespace: input.namespace, - documentObjectKey: input.documentObjectKey, - forceToken: input.forceToken, - }); - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), CREATE_OP_TIMEOUT_MS); - try { - const res = await fetch(`${cfg.baseUrl}/ops`, { - method: 'POST', - headers: { - Authorization: `Bearer ${cfg.token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - kind: 'pdf_layout', - opKey, - payload: { - documentId: input.documentId, - namespace: input.namespace, - documentObjectKey: input.documentObjectKey, - }, - }), - cache: 'no-store', - signal: controller.signal, - }); - - if (!res.ok) { - const detail = await res.text().catch(() => ''); - throw new Error(`Worker op create failed: ${res.status}${detail ? ` ${detail}` : ''}`); - } - - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || typeof parsed.opId !== 'string' || !parsed.opId.trim()) { - throw new Error('Worker op create returned invalid response'); - } - return parsed; - } finally { - clearTimeout(timeout); - } -} diff --git a/src/lib/server/compute/worker-op-state.ts b/src/lib/server/compute/worker-op-state.ts deleted file mode 100644 index a6debf0..0000000 --- a/src/lib/server/compute/worker-op-state.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; -import type { WorkerOperationState } from '@openreader/compute-core/api-contracts'; -import { serverLogger } from '@/lib/server/logger'; -import { logDegraded } from '@/lib/server/errors/logging'; - -const WORKER_OP_REQUEST_TIMEOUT_MS = 2_500; - -export async function fetchWorkerOperationState( - opId: string | null | undefined, -): Promise | null> { - const normalized = opId?.trim(); - if (!normalized) return null; - - let cfg: { baseUrl: string; token: string }; - try { - cfg = getWorkerClientConfigFromEnv(); - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.config.invalid', - msg: 'Worker client env missing/invalid', - step: 'read_worker_config', - context: { opId: normalized }, - error, - }); - return null; - } - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), WORKER_OP_REQUEST_TIMEOUT_MS); - - try { - const res = await fetch(`${cfg.baseUrl}/ops/${encodeURIComponent(normalized)}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'application/json', - }, - cache: 'no-store', - signal: controller.signal, - }); - - if (!res.ok) { - const upstreamResponseBody = await res.text().catch(() => ''); - logDegraded(serverLogger, { - event: 'compute.worker_op_state.fetch.failed', - msg: 'Worker op request failed', - step: 'fetch_worker_op', - context: { - opId: normalized, - status: res.status, - upstreamResponseBody, - }, - error: { - name: 'WorkerOpStateFetchFailed', - message: `Worker op request failed with status ${res.status}`, - }, - }); - return null; - } - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || parsed.opId !== normalized) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.response.invalid', - msg: 'Worker op response invalid', - step: 'validate_worker_op_response', - context: { opId: normalized }, - }); - return null; - } - return parsed; - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.fetch.error', - msg: 'Worker op request threw', - step: 'fetch_worker_op', - context: { opId: normalized }, - error, - }); - return null; - } finally { - clearTimeout(timeout); - } -} - -export async function fetchWorkerOperationStateByKey( - opKey: string | null | undefined, -): Promise | null> { - const normalized = opKey?.trim(); - if (!normalized) return null; - - let cfg: { baseUrl: string; token: string }; - try { - cfg = getWorkerClientConfigFromEnv(); - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.config.invalid', - msg: 'Worker client env missing/invalid', - step: 'read_worker_config', - context: { opKey: normalized }, - error, - }); - return null; - } - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), WORKER_OP_REQUEST_TIMEOUT_MS); - - try { - const res = await fetch(`${cfg.baseUrl}/ops/lookup`, { - method: 'POST', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ opKey: normalized }), - cache: 'no-store', - signal: controller.signal, - }); - - if (res.status === 404) { - return null; - } - if (!res.ok) { - const upstreamResponseBody = await res.text().catch(() => ''); - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.fetch.failed', - msg: 'Worker op lookup failed', - step: 'fetch_worker_op_by_key', - context: { - opKey: normalized, - status: res.status, - upstreamResponseBody, - }, - error: { - name: 'WorkerOpLookupFailed', - message: `Worker op lookup failed with status ${res.status}`, - }, - }); - return null; - } - - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || typeof parsed.opId !== 'string' || !parsed.opId.trim()) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.response.invalid', - msg: 'Worker op lookup response invalid', - step: 'validate_worker_op_lookup_response', - context: { opKey: normalized }, - }); - return null; - } - if (typeof parsed.opKey !== 'string' || parsed.opKey.trim() !== normalized) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.response.mismatch', - msg: 'Worker op lookup response did not match requested op key', - step: 'validate_worker_op_lookup_response', - context: { - opKey: normalized, - responseOpId: parsed.opId, - responseOpKey: typeof parsed.opKey === 'string' ? parsed.opKey : null, - }, - }); - return null; - } - return parsed; - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.fetch.error', - msg: 'Worker op lookup threw', - step: 'fetch_worker_op_by_key', - context: { opKey: normalized }, - error, - }); - return null; - } finally { - clearTimeout(timeout); - } -} diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts deleted file mode 100644 index d97e34e..0000000 --- a/src/lib/server/compute/worker.ts +++ /dev/null @@ -1,706 +0,0 @@ -import { createHash, randomUUID } from 'node:crypto'; -import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; -import { parseSseEventId, parseSsePayload } from '@openreader/compute-core'; -import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core/api-contracts'; -import { errorToLog, serverLogger } from '@/lib/server/logger'; -import { logDegraded, logServerError } from '@/lib/server/errors/logging'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; - -class WorkerHttpError extends Error { - status: number; - retryAfterMs: number | null; - - constructor(message: string, status: number, retryAfterMs: number | null = null) { - super(message); - this.name = 'WorkerHttpError'; - this.status = status; - this.retryAfterMs = retryAfterMs; - } -} - -const DEFAULT_RETRIES = 2; -const MAX_LOG_DETAIL_CHARS = 600; -const LOG_EVENTS = new Set([ - 'align.request.failed', - 'align.request.attempt_error', - 'pdf_layout.request.failed', - 'pdf_layout.request.attempt_error', - 'http.request.failed', - 'sse.wait.http_failed', - 'sse.wait.ended_without_terminal', - 'sse.wait.failed', -]); - -type WorkerLogLevel = 'info' | 'warn' | 'error'; - -function truncateForLog(value: string, maxChars = MAX_LOG_DETAIL_CHARS): string { - if (value.length <= maxChars) return value; - return `${value.slice(0, maxChars)}...`; -} - -function logWorker(level: WorkerLogLevel, event: string, fields: Record): void { - if (!LOG_EVENTS.has(event)) return; - const eventName = `compute.worker_client.${event}`; - const msg = `Worker client ${event}`; - if (level === 'error') { - logServerError(serverLogger, { - event: eventName, - msg, - error: fields.error ?? new Error(msg), - context: { - operation: 'compute_worker_client', - ...fields, - }, - normalize: { code: 'COMPUTE_WORKER_CLIENT_EVENT_FAILED', errorClass: 'upstream' }, - }); - return; - } - if (level === 'warn') { - logDegraded(serverLogger, { - event: eventName, - msg, - step: 'worker_client', - context: { - operation: 'compute_worker_client', - ...fields, - }, - error: fields.error, - }); - return; - } - serverLogger.info({ - event: eventName, - operation: 'compute_worker_client', - ...fields, - }, `Worker client ${event}`); -} - -function opSummary(value: unknown): Record { - if (!value || typeof value !== 'object') return {}; - const record = value as Record; - const summary: Record = {}; - if (typeof record.opId === 'string') summary.opId = record.opId; - if (typeof record.status === 'string') summary.status = record.status; - if (typeof record.jobId === 'string') summary.jobId = record.jobId; - if (typeof record.updatedAt === 'string') summary.updatedAt = record.updatedAt; - return summary; -} - -function readRequiredEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required for compute worker client`); - return value; -} - -function normalizeWorkerBaseUrl(raw: string): string { - const trimmed = raw.trim(); - if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty'); - - const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed) - ? trimmed - : (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed) - ? `http://${trimmed}` - : `https://${trimmed}`); - - let parsed: URL; - try { - parsed = new URL(withScheme); - } catch { - throw new Error( - `Invalid COMPUTE_WORKER_URL="${raw}". Expected full URL like https://example.com (or http://localhost:4000).`, - ); - } - - parsed.pathname = parsed.pathname.replace(/\/+$/, ''); - return parsed.toString().replace(/\/+$/, ''); -} - -export function getWorkerClientConfigFromEnv(): { baseUrl: string; token: string } { - return { - baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')), - token: readRequiredEnv('COMPUTE_WORKER_TOKEN'), - }; -} - -export function isWorkerClientConfigAvailable(): boolean { - try { - getWorkerClientConfigFromEnv(); - return true; - } catch { - return false; - } -} - -function parseRetryAfterMs(value: string | null): number | null { - if (!value) return null; - const asNum = Number(value); - if (Number.isFinite(asNum)) { - return Math.max(0, Math.floor(asNum * 1000)); - } - const when = Date.parse(value); - if (Number.isNaN(when)) return null; - return Math.max(0, when - Date.now()); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function shouldRetry(error: unknown): boolean { - if (error instanceof WorkerHttpError) { - return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; - } - if (error instanceof Error) { - const msg = error.message.toLowerCase(); - return msg.includes('network') || msg.includes('timeout') || msg.includes('fetch failed'); - } - return false; -} - -function isTerminalStatus(status: string): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function sha256Hex(input: string): string { - return createHash('sha256').update(input).digest('hex'); -} - -function buildWhisperOpKey(input: WhisperAlignInput): string { - const cacheKey = input.cacheKey?.trim(); - if (cacheKey) { - return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`; - } - return [ - 'whisper_align', - 'v1', - input.audioObjectKey, - input.lang ?? '', - sha256Hex(input.text), - ].join('|'); -} - -export function buildPdfOpKey(input: PdfLayoutInput): string { - return [ - 'pdf_layout', - 'v1', - PDF_PARSER_VERSION, - input.documentId, - input.namespace ?? '', - input.documentObjectKey ?? '', - input.forceToken?.trim() || '', - ].join('|'); -} - -type RetryMeta = { - attempt: number, - maxAttempts: number, - willRetry: boolean, - delayMs: number | null, - error: unknown, -}; - -async function withRetries( - attempts: number, - operation: (attempt: number) => Promise, - onAttemptError?: (meta: RetryMeta) => void, -): Promise { - let lastError: unknown = null; - for (let attemptIndex = 0; attemptIndex < attempts; attemptIndex += 1) { - const attempt = attemptIndex + 1; - try { - return await operation(attempt); - } catch (error) { - lastError = error; - const willRetry = attemptIndex < attempts - 1 && shouldRetry(error); - let delayMs: number | null = null; - if (willRetry) { - if (error instanceof WorkerHttpError && typeof error.retryAfterMs === 'number') { - delayMs = error.retryAfterMs; - } else { - delayMs = attempt * 250; - } - } - onAttemptError?.({ - attempt, - maxAttempts: attempts, - willRetry, - delayMs, - error, - }); - if (!willRetry) break; - await sleep(delayMs ?? 0); - } - } - throw lastError instanceof Error ? lastError : new Error('Unknown worker compute failure'); -} - -export class WorkerComputeBackend implements ComputeBackend { - readonly mode = 'worker' as const; - private readonly baseUrl: string; - private readonly token: string; - private readonly waitTimeoutMsByKind: Record<'whisper_align' | 'pdf_layout', number>; - private readonly retries: number; - - constructor() { - const cfg = getWorkerClientConfigFromEnv(); - this.baseUrl = cfg.baseUrl; - this.token = cfg.token; - this.waitTimeoutMsByKind = { - whisper_align: getWorkerClientWaitTimeoutMs('whisper_align'), - pdf_layout: getWorkerClientWaitTimeoutMs('pdf_layout'), - }; - this.retries = DEFAULT_RETRIES; - } - - async alignWords(input: WhisperAlignInput): Promise { - if (!input.audioObjectKey) { - throw new Error('Worker compute alignment requires audioObjectKey'); - } - const payload: WhisperAlignJobRequest = { - text: input.text, - lang: input.lang, - cacheKey: input.cacheKey, - audioObjectKey: input.audioObjectKey, - }; - const traceId = randomUUID(); - const opKey = buildWhisperOpKey(input); - const opKeyHash = sha256Hex(opKey).slice(0, 16); - const startedAt = Date.now(); - logWorker('info', 'align.request.start', { - traceId, - kind: 'whisper_align', - opKeyHash, - audioObjectKey: input.audioObjectKey, - cacheKey: input.cacheKey ?? null, - lang: input.lang ?? null, - textLength: input.text.length, - waitTimeoutMs: this.waitTimeoutMsByKind.whisper_align, - maxRetries: this.retries, - }); - - try { - const result = await withRetries(this.retries, async (attempt) => { - const op = await this.requestJson>('POST', '/ops', { - kind: 'whisper_align', - opKey, - payload, - }, { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - }); - - const final = isTerminalStatus(op.status) - ? op - : await this.waitForOperation(op.opId, { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - waitTimeoutMs: this.waitTimeoutMsByKind.whisper_align, - }); - - if (final.status !== 'succeeded' || !final.result) { - throw new Error(final.error?.message || 'Whisper worker operation did not complete'); - } - return { alignments: final.result.alignments }; - }, ({ attempt, maxAttempts, willRetry, delayMs, error }) => { - logWorker(willRetry ? 'warn' : 'error', 'align.request.attempt_error', { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - maxAttempts, - willRetry, - delayMs, - error: errorToLog(error), - }); - }); - - logWorker('info', 'align.request.succeeded', { - traceId, - kind: 'whisper_align', - opKeyHash, - durationMs: Date.now() - startedAt, - }); - return result; - } catch (error) { - logWorker('error', 'align.request.failed', { - traceId, - kind: 'whisper_align', - opKeyHash, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } - } - - async parsePdfLayout(input: PdfLayoutInput) { - if (!input.documentObjectKey) { - throw new Error('Worker compute PDF layout requires documentObjectKey'); - } - const payload: PdfLayoutJobRequest = { - documentId: input.documentId, - namespace: input.namespace ?? null, - documentObjectKey: input.documentObjectKey, - }; - const traceId = randomUUID(); - const opKey = buildPdfOpKey(input); - const opKeyHash = sha256Hex(opKey).slice(0, 16); - const startedAt = Date.now(); - logWorker('info', 'pdf_layout.request.start', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - namespace: input.namespace ?? null, - documentObjectKey: input.documentObjectKey, - waitTimeoutMs: this.waitTimeoutMsByKind.pdf_layout, - maxRetries: this.retries, - }); - - try { - const result = await withRetries(this.retries, async (attempt) => { - const op = await this.requestJson>('POST', '/ops', { - kind: 'pdf_layout', - opKey, - payload, - }, { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - }); - await input.onWorkerSnapshot?.(op); - - const final = isTerminalStatus(op.status) - ? op - : await this.waitForOperation(op.opId, { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - waitTimeoutMs: this.waitTimeoutMsByKind.pdf_layout, - onSnapshot: (snapshot) => { - void input.onWorkerSnapshot?.(snapshot); - if (snapshot.progress) { - void input.onProgress?.(snapshot.progress); - } - }, - }); - - if (final.status !== 'succeeded' || !final.result) { - throw new Error(final.error?.message || 'PDF layout worker operation did not complete'); - } - if (final.result.parsedObjectKey) { - return { parsedObjectKey: final.result.parsedObjectKey }; - } - if (final.result.parsed) { - return { parsed: final.result.parsed }; - } - throw new Error('PDF layout worker operation completed without parsed output'); - }, ({ attempt, maxAttempts, willRetry, delayMs, error }) => { - logWorker(willRetry ? 'warn' : 'error', 'pdf_layout.request.attempt_error', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - maxAttempts, - willRetry, - delayMs, - error: errorToLog(error), - }); - }); - - logWorker('info', 'pdf_layout.request.succeeded', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - durationMs: Date.now() - startedAt, - }); - return result; - } catch (error) { - logWorker('error', 'pdf_layout.request.failed', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } - } - - private async requestJson( - method: 'GET' | 'POST', - path: string, - body?: unknown, - context: Record = {}, - ): Promise { - const startedAt = Date.now(); - const traceId = typeof context.traceId === 'string' ? context.traceId : randomUUID(); - logWorker('info', 'http.request.start', { - ...context, - traceId, - method, - path, - }); - const res = await fetch(`${this.baseUrl}${path}`, { - method, - headers: { - Authorization: `Bearer ${this.token}`, - 'x-openreader-trace-id': traceId, - ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), - }, - ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), - }); - - if (!res.ok) { - const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); - const upstreamResponseBody = await res.text().catch(() => ''); - const requestError = new WorkerHttpError( - `Worker request failed (${method} ${path}): ${res.status}${upstreamResponseBody ? ` ${upstreamResponseBody}` : ''}`, - res.status, - retryAfterMs, - ); - logWorker(res.status >= 500 ? 'warn' : 'error', 'http.request.failed', { - ...context, - traceId, - method, - path, - status: res.status, - retryAfterMs, - durationMs: Date.now() - startedAt, - upstreamResponseBody: truncateForLog(upstreamResponseBody), - error: errorToLog(requestError), - }); - throw requestError; - } - - const parsed = await res.json() as T; - const operationSummary = opSummary(parsed); - logWorker('info', 'http.request.succeeded', { - ...context, - traceId, - method, - path, - httpStatus: res.status, - durationMs: Date.now() - startedAt, - ...operationSummary, - }); - return parsed; - } - - private async waitForOperation( - opId: string, - context: Record & { - waitTimeoutMs?: number; - onSnapshot?: (snapshot: WorkerOperationState) => void - } = {}, - ): Promise> { - const waitTimeoutMs = typeof context.waitTimeoutMs === 'number' - ? context.waitTimeoutMs - : this.waitTimeoutMsByKind.whisper_align; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), waitTimeoutMs); - const startedAt = Date.now(); - const traceId = typeof context.traceId === 'string' ? context.traceId : randomUUID(); - logWorker('info', 'sse.wait.start', { - ...context, - traceId, - opId, - waitTimeoutMs, - }); - - try { - let latest: WorkerOperationState | null = null; - let lastEventId: number | null = null; - let eventCount = 0; - let lastStatus: string | null = null; - - while (!controller.signal.aborted) { - const query = lastEventId && lastEventId > 0 - ? `?sinceEventId=${encodeURIComponent(String(lastEventId))}` - : ''; - const res = await fetch(`${this.baseUrl}/ops/${encodeURIComponent(opId)}/events${query}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${this.token}`, - Accept: 'text/event-stream', - 'x-openreader-trace-id': traceId, - ...(lastEventId && lastEventId > 0 ? { 'Last-Event-ID': String(lastEventId) } : {}), - }, - signal: controller.signal, - }); - - if (!res.ok) { - const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); - const upstreamResponseBody = await res.text().catch(() => ''); - const requestError = new WorkerHttpError( - `Worker request failed (GET /ops/${encodeURIComponent(opId)}/events): ${res.status}${upstreamResponseBody ? ` ${upstreamResponseBody}` : ''}`, - res.status, - retryAfterMs, - ); - logWorker(res.status >= 500 ? 'warn' : 'error', 'sse.wait.http_failed', { - ...context, - traceId, - opId, - status: res.status, - retryAfterMs, - durationMs: Date.now() - startedAt, - upstreamResponseBody: truncateForLog(upstreamResponseBody), - error: errorToLog(requestError), - }); - throw requestError; - } - - if (!res.body) { - logWorker('error', 'sse.wait.no_body', { - ...context, - traceId, - opId, - durationMs: Date.now() - startedAt, - }); - throw new Error('Worker operation stream response has no body'); - } - - logWorker('info', 'sse.wait.connected', { - ...context, - traceId, - opId, - status: res.status, - }); - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - - while (true) { - const frameEnd = buffer.indexOf('\n\n'); - if (frameEnd < 0) break; - const frame = buffer.slice(0, frameEnd); - buffer = buffer.slice(frameEnd + 2); - - const eventId = parseSseEventId(frame); - if (eventId && eventId > 0) { - lastEventId = eventId; - } - const payload = parseSsePayload(frame); - if (!payload) continue; - - let event: WorkerOperationEvent | WorkerOperationState; - try { - event = JSON.parse(payload) as WorkerOperationEvent | WorkerOperationState; - } catch { - logWorker('warn', 'sse.wait.json_parse_skipped', { - ...context, - traceId, - opId, - sample: truncateForLog(payload), - }); - continue; - } - - const snapshot: WorkerOperationState = ( - event && typeof event === 'object' && 'snapshot' in event - ? (event as WorkerOperationEvent).snapshot - : (event as WorkerOperationState) - ); - if (!snapshot || typeof snapshot !== 'object') continue; - if (snapshot.opId !== opId) continue; - - eventCount += 1; - latest = snapshot; - context.onSnapshot?.(snapshot); - if (snapshot.status !== lastStatus) { - lastStatus = snapshot.status; - logWorker('info', 'sse.wait.status', { - ...context, - traceId, - opId, - eventCount, - status: snapshot.status, - jobId: snapshot.jobId ?? null, - updatedAt: snapshot.updatedAt ?? null, - }); - } - if (isTerminalStatus(snapshot.status)) { - logWorker('info', 'sse.wait.terminal', { - ...context, - traceId, - opId, - eventCount, - status: snapshot.status, - durationMs: Date.now() - startedAt, - }); - return snapshot; - } - } - } - - if (latest && isTerminalStatus(latest.status)) { - logWorker('info', 'sse.wait.terminal_after_close', { - ...context, - traceId, - opId, - eventCount, - status: latest.status, - durationMs: Date.now() - startedAt, - }); - return latest; - } - - if (controller.signal.aborted) break; - await sleep(250); - } - - if (latest && isTerminalStatus(latest.status)) { - return latest; - } - - logWorker('error', 'sse.wait.ended_without_terminal', { - ...context, - traceId, - opId, - eventCount, - latestStatus: latest?.status ?? null, - durationMs: Date.now() - startedAt, - error: { - name: 'WorkerSseWaitEndedWithoutTerminal', - message: `Operation stream ended before terminal state for op ${opId}`, - }, - }); - throw new Error(`Operation stream ended before terminal state for op ${opId}`); - } catch (error) { - logWorker('error', 'sse.wait.failed', { - ...context, - traceId, - opId, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } finally { - clearTimeout(timeout); - } - } -} diff --git a/src/lib/server/documents/blob-lease.ts b/src/lib/server/documents/blob-lease.ts index c5b82bf..162bd99 100644 --- a/src/lib/server/documents/blob-lease.ts +++ b/src/lib/server/documents/blob-lease.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, eq, lt } from 'drizzle-orm'; -import { db } from '@/db'; -import { documentBlobLeases } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documentBlobLeases } from '@openreader/database/schema'; import { errorToLog, serverLogger } from '@/lib/server/logger'; const DEFAULT_LEASE_MS = 15 * 60 * 1000; diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index 230bbed..42799da 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -8,7 +8,6 @@ import { PutObjectCommand, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { PDF_PARSER_VERSION, encodeParserVersion } from '@openreader/compute-core/api-contracts'; import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; @@ -104,24 +103,6 @@ export function documentKey(id: string, namespace: string | null): string { return `${cfg.prefix}/documents_v1/${nsSegment}${id}`; } -export function documentParsedKey(id: string, namespace: string | null): string { - return documentParsedKeyForVersion(id, namespace, PDF_PARSER_VERSION); -} - -export function documentParsedKeyForVersion( - id: string, - namespace: string | null, - parserVersion: string, -): string { - if (!isValidDocumentId(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const cfg = getS3Config(); - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(parserVersion)}.json`; -} - export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string { const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); @@ -342,19 +323,6 @@ export async function getTempDocumentFinalizeReceipt( } } -export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise { - const cfg = getS3Config(); - const client = getS3ProxyClient(); - const key = documentParsedKey(id, namespace); - const res = await client.send( - new GetObjectCommand({ - Bucket: cfg.bucket, - Key: key, - }), - ); - return bodyToBuffer(res.Body); -} - export async function getParsedDocumentBlobByKey(key: string): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -369,31 +337,6 @@ export async function getParsedDocumentBlobByKey(key: string): Promise { return bodyToBuffer(res.Body); } -export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: string | null): Promise { - return putParsedDocumentBlobForVersion(id, body, namespace, PDF_PARSER_VERSION); -} - -export async function putParsedDocumentBlobForVersion( - id: string, - body: Buffer, - namespace: string | null, - parserVersion: string, -): Promise { - const cfg = getS3Config(); - const client = getS3ProxyClient(); - const key = documentParsedKeyForVersion(id, namespace, parserVersion); - await client.send( - new PutObjectCommand({ - Bucket: cfg.bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - }), - ); - return key; -} - export async function putTempDocumentFinalizeReceipt( token: string, userId: string, @@ -502,7 +445,6 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); - const parsedKey = documentParsedKey(id, namespace); const legacyParsedKey = legacyDocumentParsedKey(id, namespace); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; @@ -510,7 +452,6 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): await deleteDocumentPrefix(parsedPrefix); await deleteDocumentPrefix(`${key}/`); - await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })); // Delete the source after the initial derived-artifact cleanup, then sweep // parsed output once more to catch a worker that finished during deletion. diff --git a/src/lib/server/documents/delete-owned.ts b/src/lib/server/documents/delete-owned.ts index 4b2d14e..1686a51 100644 --- a/src/lib/server/documents/delete-owned.ts +++ b/src/lib/server/documents/delete-owned.ts @@ -1,6 +1,6 @@ import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache'; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; diff --git a/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts b/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts new file mode 100644 index 0000000..4f041c5 --- /dev/null +++ b/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts @@ -0,0 +1,60 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +let cachedStandardFontDataUrl: string | null = null; +const pdfjsPackageName = 'pdfjs-dist'; + +function candidatePackageRoots(): string[] { + const roots = new Set(); + let dir = process.cwd(); + + for (let i = 0; i < 8; i += 1) { + roots.add(path.join(dir, 'node_modules', pdfjsPackageName)); + const next = path.dirname(dir); + if (next === dir) break; + dir = next; + } + + return [...roots]; +} + +function resolvePdfjsPackageFile(relativePath: string): string { + const normalizedRelativePath = relativePath.replace(/^\/+/, ''); + const candidates = candidatePackageRoots().map((root) => path.join(root, normalizedRelativePath)); + const found = candidates.find((candidate) => fs.existsSync(candidate)); + if (found) return found; + + throw new Error( + `pdfjs-dist file not found: ${normalizedRelativePath}; checked ${candidates.join(', ')}`, + ); +} + +export function resolvePdfjsStandardFontDataUrl(): string { + if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl; + + const pdfjsPackageDir = path.dirname(resolvePdfjsPackageFile('package.json')); + const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts'); + + if (!fs.existsSync(standardFontDir)) { + throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`); + } + + cachedStandardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`; + return cachedStandardFontDataUrl; +} + +export function resolvePdfjsWorkerSrc(): string { + return pathToFileURL(resolvePdfjsPackageFile('legacy/build/pdf.worker.mjs')).href; +} + +export function configurePdfjsNodeRuntime(pdfjs: { + GlobalWorkerOptions?: { + workerSrc?: string; + workerPort?: unknown; + }; +}): void { + if (!pdfjs.GlobalWorkerOptions) return; + pdfjs.GlobalWorkerOptions.workerSrc = resolvePdfjsWorkerSrc(); + pdfjs.GlobalWorkerOptions.workerPort = null; +} diff --git a/src/lib/server/documents/pdf-preview-renderer.ts b/src/lib/server/documents/pdf-preview-renderer.ts new file mode 100644 index 0000000..4ef78ac --- /dev/null +++ b/src/lib/server/documents/pdf-preview-renderer.ts @@ -0,0 +1,157 @@ +import type { Canvas } from '@napi-rs/canvas'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdf-preview-pdfjs-runtime'; + +type CanvasRuntime = { + DOMMatrixCtor: unknown; + Path2DCtor: unknown; + createCanvasFn: (width: number, height: number) => Canvas; +}; + +let canvasRuntimePromise: Promise | null = null; + +async function loadCanvasRuntime(): Promise { + if (!canvasRuntimePromise) { + canvasRuntimePromise = (async () => { + const mod = await import('@napi-rs/canvas'); + const namespace = mod as Record; + const fallback = (namespace.default ?? {}) as Record; + + const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as + | ((width: number, height: number) => Canvas) + | undefined; + const DOMMatrixCtor = namespace.DOMMatrix ?? fallback.DOMMatrix; + const Path2DCtor = namespace.Path2D ?? fallback.Path2D; + + if (typeof createCanvasFn !== 'function') { + throw new Error( + `Canvas runtime missing createCanvas export (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + if (!DOMMatrixCtor || !Path2DCtor) { + throw new Error( + `Canvas runtime missing DOMMatrix/Path2D exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + + return { + DOMMatrixCtor, + Path2DCtor, + createCanvasFn, + }; + })(); + } + return canvasRuntimePromise; +} + +function ensureNodeCanvasGlobals(runtime: CanvasRuntime): void { + const g = globalThis as Record; + if (typeof g.DOMMatrix === 'undefined') g.DOMMatrix = runtime.DOMMatrixCtor; + if (typeof g.Path2D === 'undefined') g.Path2D = runtime.Path2DCtor; +} + +interface RenderInput { + pdfBytes: ArrayBuffer; + pageNumber: number; + scale?: number; + targetWidth?: number; + format?: 'png' | 'jpeg'; + jpegQuality?: number; +} + +function createPdfjsCanvasFactory(runtime: CanvasRuntime) { + return class OpenReaderCanvasFactory { + create(width: number, height: number) { + const canvas = runtime.createCanvasFn(Math.max(1, Math.floor(width)), Math.max(1, Math.floor(height))); + return { + canvas, + context: canvas.getContext('2d') as unknown as CanvasRenderingContext2D, + }; + } + + reset(target: { canvas: Canvas; context: CanvasRenderingContext2D }, width: number, height: number): void { + target.canvas.width = Math.max(1, Math.floor(width)); + target.canvas.height = Math.max(1, Math.floor(height)); + } + + destroy(target: { canvas: Canvas; context: CanvasRenderingContext2D }): void { + target.canvas.width = 0; + target.canvas.height = 0; + // @ts-expect-error pdf.js expects these nulled on destroy + target.canvas = null; + // @ts-expect-error pdf.js expects these nulled on destroy + target.context = null; + } + }; +} + +export async function renderPage({ + pdfBytes, + pageNumber, + scale = 1.5, + targetWidth, + format = 'png', + jpegQuality = 82, +}: RenderInput): Promise<{ + width: number; + height: number; + image: Buffer; + contentType: 'image/png' | 'image/jpeg'; +}> { + // pdf.js may detach the provided ArrayBuffer. Work with an isolated copy so + // callers can safely reuse their original bytes across pages/calls. + const isolatedBytes = new Uint8Array(pdfBytes).slice(); + + const canvasRuntime = await loadCanvasRuntime(); + ensureNodeCanvasGlobals(canvasRuntime); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + configurePdfjsNodeRuntime(pdfjs); + + const standardFontDataUrl = resolvePdfjsStandardFontDataUrl(); + + const loadingTask = pdfjs.getDocument({ + data: isolatedBytes, + useWorkerFetch: false, + standardFontDataUrl, + isEvalSupported: false, + // Ensure pdf.js transport uses our canvas backend in Node/Next runtime. + CanvasFactory: createPdfjsCanvasFactory(canvasRuntime), + }); + + const pdf = await loadingTask.promise; + try { + const page = await pdf.getPage(pageNumber); + const baseViewport = page.getViewport({ scale: 1.0 }); + const effectiveScale = typeof targetWidth === 'number' && Number.isFinite(targetWidth) && targetWidth > 0 + ? (Math.max(1, Math.round(targetWidth)) / Math.max(1, baseViewport.width)) + : scale; + const viewport = page.getViewport({ scale: effectiveScale }); + const width = Math.max(1, Math.floor(viewport.width)); + const height = Math.max(1, Math.floor(viewport.height)); + const canvas = canvasRuntime.createCanvasFn(width, height); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, width, height); + + const renderTask = page.render({ + canvasContext: ctx as unknown as CanvasRenderingContext2D, + viewport, + intent: 'display', + }); + await renderTask.promise; + const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png'; + const image = format === 'jpeg' + ? canvas.toBuffer('image/jpeg', jpegQuality) + : canvas.toBuffer('image/png'); + return { + width, + height, + image, + contentType, + }; + } finally { + await pdf.destroy().catch(() => undefined); + await loadingTask.destroy().catch(() => undefined); + } +} diff --git a/src/lib/server/documents/previews-render.ts b/src/lib/server/documents/previews-render.ts index c32bbce..033e8fc 100644 --- a/src/lib/server/documents/previews-render.ts +++ b/src/lib/server/documents/previews-render.ts @@ -2,7 +2,7 @@ import path from 'path'; import { createCanvas, loadImage } from '@napi-rs/canvas'; import JSZip from 'jszip'; import { XMLParser } from 'fast-xml-parser'; -import { renderPage } from '@openreader/compute-core'; +import { renderPage } from '@/lib/server/documents/pdf-preview-renderer'; export type RenderedDocumentPreview = { bytes: Buffer; diff --git a/src/lib/server/documents/previews.ts b/src/lib/server/documents/previews.ts index 7c60c98..c127605 100644 --- a/src/lib/server/documents/previews.ts +++ b/src/lib/server/documents/previews.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'crypto'; import { and, eq, inArray, lt, or, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { documentPreviews } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documentPreviews } from '@openreader/database/schema'; import { serverLogger } from '@/lib/server/logger'; import { logServerError } from '@/lib/server/errors/logging'; import { diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts index a29feb8..073c048 100644 --- a/src/lib/server/documents/register-upload.ts +++ b/src/lib/server/documents/register-upload.ts @@ -1,5 +1,5 @@ -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { and, eq } from 'drizzle-orm'; import { enqueueDocumentPreview, diff --git a/src/lib/server/jobs/user-whisper-align-job.ts b/src/lib/server/jobs/user-whisper-align-job.ts index afe0127..022ee92 100644 --- a/src/lib/server/jobs/user-whisper-align-job.ts +++ b/src/lib/server/jobs/user-whisper-align-job.ts @@ -1,14 +1,13 @@ -import { getCompute } from '@/lib/server/compute'; -import type { WhisperAlignJobRequest } from '@openreader/compute-core/api-contracts'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; +import type { WhisperAlignRequest } from '@/lib/server/compute-worker/protocol'; import type { TTSSentenceAlignment } from '@/types/tts'; -export type UserWhisperAlignJobRequest = WhisperAlignJobRequest & { +export type UserWhisperAlignJobRequest = WhisperAlignRequest & { sentenceIndex?: number; }; export async function userWhisperAlignJob(input: UserWhisperAlignJobRequest): Promise { - const compute = await getCompute(); - const { alignments } = await compute.alignWords({ + const { alignments } = await getComputeWorkerClient().alignWords({ audioObjectKey: input.audioObjectKey, text: input.text, cacheKey: input.cacheKey, diff --git a/src/lib/server/pdf-parse/artifact.ts b/src/lib/server/pdf-parse/artifact.ts index d195480..7e49b92 100644 --- a/src/lib/server/pdf-parse/artifact.ts +++ b/src/lib/server/pdf-parse/artifact.ts @@ -1,9 +1,8 @@ -import { PDF_PARSER_VERSION } from '@openreader/compute-core/api-contracts'; import { - documentParsedKeyForVersion, getParsedDocumentBlobByKey, isMissingBlobError, } from '@/lib/server/documents/blobstore'; +import { resolveCurrentPdfParse } from '@/lib/server/pdf-parse/operation'; import type { ParsedPdfDocument } from '@/types/parsed-pdf'; export interface ParsedPdfArtifact { @@ -13,7 +12,25 @@ export interface ParsedPdfArtifact { } function parseArtifact(bytes: Buffer): ParsedPdfDocument { - return JSON.parse(bytes.toString('utf8')) as ParsedPdfDocument; + const parsed = JSON.parse(bytes.toString('utf8')) as Partial; + if ( + parsed.schemaVersion !== 1 + || typeof parsed.documentId !== 'string' + || typeof parsed.parserVersion !== 'string' + || typeof parsed.parsedAt !== 'number' + || !Array.isArray(parsed.pages) + || parsed.pages.some((page) => ( + !page + || typeof page !== 'object' + || typeof page.pageNumber !== 'number' + || typeof page.width !== 'number' + || typeof page.height !== 'number' + || !Array.isArray(page.blocks) + )) + ) { + throw new Error('Parsed PDF artifact envelope is invalid'); + } + return parsed as ParsedPdfDocument; } export async function readParsedPdfArtifactByKey(key: string): Promise { @@ -34,6 +51,11 @@ export async function readCurrentParsedPdfArtifact(input: { documentId: string; namespace: string | null; }): Promise { - const key = documentParsedKeyForVersion(input.documentId, input.namespace, PDF_PARSER_VERSION); - return readParsedPdfArtifactByKey(key); + const resolved = await resolveCurrentPdfParse(input); + if (!resolved.artifact) return null; + const artifact = await readParsedPdfArtifactByKey(resolved.artifact.objectKey); + if (artifact && artifact.parsed.documentId !== input.documentId) { + throw new Error('Parsed PDF artifact document identity mismatch'); + } + return artifact; } diff --git a/src/lib/server/pdf-parse/operation.ts b/src/lib/server/pdf-parse/operation.ts index f00aa19..a10136b 100644 --- a/src/lib/server/pdf-parse/operation.ts +++ b/src/lib/server/pdf-parse/operation.ts @@ -1,65 +1,66 @@ -import { buildPdfOpKey } from '@/lib/server/compute/worker'; -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; -import { - fetchWorkerOperationState, - fetchWorkerOperationStateByKey, -} from '@/lib/server/compute/worker-op-state'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { documentKey } from '@/lib/server/documents/blobstore'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { + PdfLayoutResolution, + PdfLayoutResult, + ComputeOperation, +} from '@/lib/server/compute-worker/protocol'; function currentPdfOperationInput(documentId: string, namespace: string | null, forceToken?: string): { documentId: string; namespace: string | null; documentObjectKey: string; - forceToken?: string; + replaceToken?: string; } { return { documentId, namespace, documentObjectKey: documentKey(documentId, namespace), - ...(forceToken ? { forceToken } : {}), + ...(forceToken ? { replaceToken: forceToken } : {}), }; } -export function buildCurrentPdfParseOpKeyPrefix(input: { +export async function resolveCurrentPdfParse(input: { documentId: string; namespace: string | null; -}): string { - return buildPdfOpKey(currentPdfOperationInput(input.documentId, input.namespace)); +}): Promise { + return getComputeWorkerClient().resolvePdfLayout(currentPdfOperationInput( + input.documentId, + input.namespace, + )); } export async function lookupCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; -}): Promise | null> { - return fetchWorkerOperationStateByKey( - buildCurrentPdfParseOpKeyPrefix(input), - ); +}): Promise | null> { + return (await resolveCurrentPdfParse(input)).operation; } export async function createOrReuseCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; forceToken?: string; -}): Promise> { - return createOrReusePdfWorkerOperation(currentPdfOperationInput( +}): Promise> { + return getComputeWorkerClient().createPdfLayoutOperation(currentPdfOperationInput( input.documentId, input.namespace, input.forceToken, )); } -export async function fetchPdfParseOperation(opId: string): Promise | null> { - return fetchWorkerOperationState(opId); +export async function fetchPdfParseOperation(opId: string): Promise | null> { + return getComputeWorkerClient().getOperation(opId); } export function isPdfParseOperationForDocument( - state: WorkerOperationState, + state: ComputeOperation, input: { documentId: string; namespace: string | null; }, ): boolean { - if (state.kind !== 'pdf_layout') return false; - return state.opKey.startsWith(buildCurrentPdfParseOpKeyPrefix(input)); + return state.subject.kind === 'pdf_layout' + && state.subject.documentId === input.documentId + && state.subject.namespace === input.namespace; } diff --git a/src/lib/server/pdf-parse/snapshot.ts b/src/lib/server/pdf-parse/snapshot.ts index 49c6f61..b4e8924 100644 --- a/src/lib/server/pdf-parse/snapshot.ts +++ b/src/lib/server/pdf-parse/snapshot.ts @@ -1,8 +1,8 @@ -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutResult, ComputeOperation } from '@/lib/server/compute-worker/protocol'; import type { PdfParseStatus } from '@/types/parsed-pdf'; import type { PdfParseSnapshot } from '@/lib/server/pdf-parse/types'; -function mapWorkerStatusToParseStatus(status: WorkerOperationState['status']): PdfParseStatus { +function mapWorkerStatusToParseStatus(status: ComputeOperation['status']): PdfParseStatus { switch (status) { case 'queued': return 'pending'; @@ -19,7 +19,7 @@ function mapWorkerStatusToParseStatus(status: WorkerOperationState['status']): P } export function parsedObjectKeyFromWorkerState( - state: WorkerOperationState, + state: ComputeOperation, ): string | null { const result = state.result; if (!result || typeof result !== 'object' || !('parsedObjectKey' in result)) return null; @@ -30,7 +30,7 @@ export function parsedObjectKeyFromWorkerState( } export function pdfParseSnapshotFromWorkerState( - state: WorkerOperationState, + state: ComputeOperation, ): PdfParseSnapshot { const parseStatus = mapWorkerStatusToParseStatus(state.status); return { diff --git a/src/lib/server/rate-limit/job-rate-limiter.ts b/src/lib/server/rate-limit/job-rate-limiter.ts index 9a886ce..962d46e 100644 --- a/src/lib/server/rate-limit/job-rate-limiter.ts +++ b/src/lib/server/rate-limit/job-rate-limiter.ts @@ -1,6 +1,6 @@ import { and, eq, gte, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { userJobEvents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userJobEvents } from '@openreader/database/schema'; import { nowTimestampMs } from '@/lib/shared/timestamps'; import type { RuntimeConfig } from '@/lib/server/admin/settings'; diff --git a/src/lib/server/rate-limit/rate-limiter.ts b/src/lib/server/rate-limit/rate-limiter.ts index a526d0d..cbbe335 100644 --- a/src/lib/server/rate-limit/rate-limiter.ts +++ b/src/lib/server/rate-limit/rate-limiter.ts @@ -1,6 +1,6 @@ -import { db } from '@/db'; -import { runInDbTransaction } from '@/db/run-in-transaction'; -import { userTtsChars } from '@/db/schema'; +import { db } from '@openreader/database'; +import { runInDbTransaction } from '@openreader/database/run-in-transaction'; +import { userTtsChars } from '@openreader/database/schema'; import { eq, and, lt, sql } from 'drizzle-orm'; import { nextUtcMidnightTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; diff --git a/src/lib/server/runtime-config.ts b/src/lib/server/runtime-config.ts index 87889d8..61bcfb6 100644 --- a/src/lib/server/runtime-config.ts +++ b/src/lib/server/runtime-config.ts @@ -6,7 +6,7 @@ import { type RuntimeConfigKey, type RuntimeConfigSource, } from '@/lib/server/admin/settings'; -import { isComputeAvailable } from '@/lib/server/compute'; +import { isComputeWorkerAvailable } from '@/lib/server/compute-worker/client'; export type ResolvedRuntimeConfig = RuntimeConfig & { computeAvailable: boolean; @@ -29,7 +29,7 @@ export async function getResolvedRuntimeConfig(): Promise const values = await getRuntimeConfig(); return { ...values, - computeAvailable: isComputeAvailable(), + computeAvailable: isComputeWorkerAvailable(), }; } diff --git a/src/lib/server/tasks/engine.ts b/src/lib/server/tasks/engine.ts index 93822df..24ffa62 100644 --- a/src/lib/server/tasks/engine.ts +++ b/src/lib/server/tasks/engine.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, eq, lt, ne, or, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { scheduledTasks } from '@/db/schema'; +import { db } from '@openreader/database'; +import { scheduledTasks } from '@openreader/database/schema'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; import { TASK_REGISTRY } from './registry'; diff --git a/src/lib/server/tasks/handlers/prune-job-events.ts b/src/lib/server/tasks/handlers/prune-job-events.ts index f0ddf18..03938dc 100644 --- a/src/lib/server/tasks/handlers/prune-job-events.ts +++ b/src/lib/server/tasks/handlers/prune-job-events.ts @@ -1,6 +1,6 @@ import { lt } from 'drizzle-orm'; -import { db } from '@/db'; -import { userJobEvents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userJobEvents } from '@openreader/database/schema'; import type { TaskResult } from '../types'; // Retention far exceeds the largest rate-limit window, so pruning never removes diff --git a/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts b/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts index a692370..0f88454 100644 --- a/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts +++ b/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts @@ -1,6 +1,6 @@ import { inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { isS3Configured } from '@/lib/server/storage/s3'; import { deleteDocumentBlob, listDocumentSourceBlobs } from '@/lib/server/documents/blobstore'; import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; diff --git a/src/lib/server/testing/test-namespace.ts b/src/lib/server/testing/test-namespace.ts index 044e46f..834d7cd 100644 --- a/src/lib/server/testing/test-namespace.ts +++ b/src/lib/server/testing/test-namespace.ts @@ -1,6 +1,6 @@ import path from 'path'; import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy'; -import { ensureSystemUserExists } from '@/db'; +import { ensureSystemUserExists } from '@openreader/database'; const TEST_NAMESPACE_HEADER = 'x-openreader-test-namespace'; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; diff --git a/src/lib/server/tts/segments-audio.ts b/src/lib/server/tts/segments-audio.ts index 779c79e..1959baa 100644 --- a/src/lib/server/tts/segments-audio.ts +++ b/src/lib/server/tts/segments-audio.ts @@ -1,8 +1,8 @@ import { and, eq } from 'drizzle-orm'; import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; -import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; export type ResolvedSegmentAudio = { documentId: string; diff --git a/src/lib/server/tts/segments-auth.ts b/src/lib/server/tts/segments-auth.ts index ff19de3..cd00f71 100644 --- a/src/lib/server/tts/segments-auth.ts +++ b/src/lib/server/tts/segments-auth.ts @@ -1,7 +1,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import type { NextRequest } from 'next/server'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import type { ReaderType } from '@/types/user-state'; diff --git a/src/lib/server/tts/segments-cache.ts b/src/lib/server/tts/segments-cache.ts index cc63791..f0f7c0a 100644 --- a/src/lib/server/tts/segments-cache.ts +++ b/src/lib/server/tts/segments-cache.ts @@ -1,6 +1,6 @@ import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; import { deleteTtsSegmentAudioObjects, deleteTtsSegmentPrefix, diff --git a/src/lib/server/user/claim-data.ts b/src/lib/server/user/claim-data.ts index 47a52c9..259425e 100644 --- a/src/lib/server/user/claim-data.ts +++ b/src/lib/server/user/claim-data.ts @@ -1,4 +1,4 @@ -import { db } from '@/db'; +import { db } from '@openreader/database'; import { documents, audiobooks, @@ -8,7 +8,7 @@ import { ttsSegmentVariants, userPreferences, userDocumentProgress, -} from '@/db/schema'; +} from '@openreader/database/schema'; import { eq, and, inArray } from 'drizzle-orm'; import { UNCLAIMED_USER_ID } from '../storage/docstore-legacy'; import { cleanupClaimedLegacyFsSources } from './legacy-fs-claim-cleanup'; diff --git a/src/lib/server/user/data-cleanup.ts b/src/lib/server/user/data-cleanup.ts index fd3845f..b096ab2 100644 --- a/src/lib/server/user/data-cleanup.ts +++ b/src/lib/server/user/data-cleanup.ts @@ -10,10 +10,10 @@ * and failures block the deletion. */ -import { db } from '@/db'; -import { documents, audiobooks, userJobEvents, userTtsChars } from '@/db/schema'; -import * as authSchemaSqlite from '@/db/schema_auth_sqlite'; -import * as authSchemaPostgres from '@/db/schema_auth_postgres'; +import { db } from '@openreader/database'; +import { documents, audiobooks, userJobEvents, userTtsChars } from '@openreader/database/schema'; +import * as authSchemaSqlite from '@openreader/database/schema-auth-sqlite'; +import * as authSchemaPostgres from '@openreader/database/schema-auth-postgres'; import { eq } from 'drizzle-orm'; import { getS3Config, isS3Configured } from '@/lib/server/storage/s3'; import { diff --git a/src/lib/shared/audio-text.ts b/src/lib/shared/audio-text.ts index 78d8797..74fdbae 100644 --- a/src/lib/shared/audio-text.ts +++ b/src/lib/shared/audio-text.ts @@ -9,9 +9,8 @@ * position-preserving variant that reuses the same patterns/glyph set so * viewer highlight offsets line up with the audio text. * - * The compute/worker package keeps its own mirror in - * `compute/core/src/whisper/alignment-map.ts` (a separate build target that - * cannot import from `@/lib`). Keep that copy in lock-step with these rules. + * Compute alignment receives this already-canonical text over the worker API, + * so its returned character offsets are measured against exactly this value. */ /** Matches a bare or explicit URL token (http(s):// or www.). Capture 1 = domain. */ diff --git a/src/types/parsed-pdf.ts b/src/types/parsed-pdf.ts index 2c7bdf7..7da5cd5 100644 --- a/src/types/parsed-pdf.ts +++ b/src/types/parsed-pdf.ts @@ -1,12 +1,12 @@ -import type { ParsedPdfBlockKind } from '@openreader/compute-core/types'; - export { type ParsedPdfBlock, type ParsedPdfBlockFragment, type ParsedPdfBlockKind, type ParsedPdfDocument, type ParsedPdfPage, -} from '@openreader/compute-core/types'; +} from '@/lib/server/compute-worker/protocol'; + +import type { ParsedPdfBlockKind } from '@/lib/server/compute-worker/protocol'; export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [ 'abstract', @@ -32,8 +32,11 @@ export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [ 'vision_footnote', ]; -export type { - PdfParsePhase, - PdfParseProgress, - PdfParseStatus, -} from '@openreader/compute-core/types'; +export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed'; +export type PdfParsePhase = 'infer' | 'merge'; +export interface PdfParseProgress { + totalPages: number; + pagesParsed: number; + currentPage?: number; + phase: PdfParsePhase; +} diff --git a/src/types/tts.ts b/src/types/tts.ts index c5f42d5..cc3073c 100644 --- a/src/types/tts.ts +++ b/src/types/tts.ts @@ -1,10 +1,11 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; export type { - TTSAudioBuffer, - TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from '@openreader/compute-core/types'; +} from '@/lib/server/compute-worker/protocol'; + +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; export type TTSLocation = string | number; diff --git a/tests/global-teardown.ts b/tests/global-teardown.ts index 826a85e..9a4d339 100644 --- a/tests/global-teardown.ts +++ b/tests/global-teardown.ts @@ -1,13 +1,13 @@ import { ListObjectsV2Command } from '@aws-sdk/client-s3'; import { and, eq, inArray, like, ne } from 'drizzle-orm'; -import { db } from '../src/db'; -import { audiobooks, audiobookChapters, documentPreviews, documents } from '../src/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters, documentPreviews, documents } from '@openreader/database/schema'; import { deleteDocumentPrefix } from '../src/lib/server/documents/blobstore'; import { deleteAudiobookPrefix } from '../src/lib/server/audiobooks/blobstore'; import { deleteTtsSegmentPrefix } from '../src/lib/server/tts/segments-blobstore'; import { getS3Client, getS3Config, isS3Configured } from '../src/lib/server/storage/s3'; -import * as authSchemaSqlite from '../src/db/schema_auth_sqlite'; -import * as authSchemaPostgres from '../src/db/schema_auth_postgres'; +import * as authSchemaSqlite from '@openreader/database/schema-auth-sqlite'; +import * as authSchemaPostgres from '@openreader/database/schema-auth-postgres'; function chunk(items: T[], size: number): T[][] { if (items.length === 0) return []; diff --git a/tests/unit/admin-providers-validation.vitest.spec.ts b/tests/unit/admin-providers-validation.vitest.spec.ts index 6511e0d..6638316 100644 --- a/tests/unit/admin-providers-validation.vitest.spec.ts +++ b/tests/unit/admin-providers-validation.vitest.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { inArray } from 'drizzle-orm'; -import { db } from '../../src/db'; -import { adminProviders } from '../../src/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders } from '@openreader/database/schema'; import { AdminProviderError, diff --git a/tests/unit/compute-worker-client-contract.vitest.spec.ts b/tests/unit/compute-worker-client-contract.vitest.spec.ts new file mode 100644 index 0000000..3eaa1e2 --- /dev/null +++ b/tests/unit/compute-worker-client-contract.vitest.spec.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, test } from 'vitest'; +import { createComputeWorkerApp, type ComputeWorkerApp } from '../../packages/compute-worker/src/api/app'; +import { FakeControlPlane } from '../../packages/compute-worker/tests/fixtures/fake-control-plane'; +import { ComputeWorkerClient } from '../../src/lib/server/compute-worker/client'; + +describe('ComputeWorkerClient contract', () => { + let runtime: ComputeWorkerApp | null = null; + + afterEach(async () => { + await runtime?.close(); + runtime = null; + }); + + test('exercises the real Fastify API through the app-owned HTTP client', async () => { + process.env.NATS_URL = 'nats://127.0.0.1:4222'; + const fake = new FakeControlPlane(); + runtime = await createComputeWorkerApp({ + host: '127.0.0.1', + port: 0, + workerToken: 'contract-token', + disableWorkers: true, + routeDeps: fake.deps, + }); + await runtime.start(); + + const address = runtime.app.server.address(); + if (!address || typeof address === 'string') { + throw new Error('Compute worker did not bind a TCP port'); + } + const client = new ComputeWorkerClient({ + baseUrl: `http://127.0.0.1:${address.port}`, + token: 'contract-token', + }); + const documentId = 'b'.repeat(64); + const request = { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }; + + const created = await client.createPdfLayoutOperation(request); + expect(created).toMatchObject({ + subject: { kind: 'pdf_layout', documentId, namespace: null }, + status: 'queued', + }); + expect(created).not.toHaveProperty('opKey'); + expect(created).not.toHaveProperty('jobId'); + + await expect(client.getOperation(created.opId)).resolves.toMatchObject({ + opId: created.opId, + subject: { kind: 'pdf_layout', documentId, namespace: null }, + }); + await expect(client.resolvePdfLayout(request)).resolves.toMatchObject({ + artifact: null, + operation: { opId: created.opId }, + }); + }); +}); diff --git a/tests/unit/document-blob-lease.vitest.spec.ts b/tests/unit/document-blob-lease.vitest.spec.ts index 684d75b..c6aede2 100644 --- a/tests/unit/document-blob-lease.vitest.spec.ts +++ b/tests/unit/document-blob-lease.vitest.spec.ts @@ -1,10 +1,10 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; -import * as sqliteSchema from '../../src/db/schema_sqlite'; +import * as sqliteSchema from '@openreader/database/schema-sqlite'; const holder = vi.hoisted(() => ({ db: null as unknown as ReturnType })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return holder.db; }, diff --git a/tests/unit/document-delete-cleanup.vitest.spec.ts b/tests/unit/document-delete-cleanup.vitest.spec.ts index 962905e..618f1fe 100644 --- a/tests/unit/document-delete-cleanup.vitest.spec.ts +++ b/tests/unit/document-delete-cleanup.vitest.spec.ts @@ -12,7 +12,7 @@ function resultBuilder(result: unknown[]) { }; } -vi.mock('@/db', () => { +vi.mock('@openreader/database', () => { const database = { delete: vi.fn(() => ({ where: vi.fn(() => ({ diff --git a/tests/unit/pdf-op-key.vitest.spec.ts b/tests/unit/pdf-op-key.vitest.spec.ts deleted file mode 100644 index fbe746e..0000000 --- a/tests/unit/pdf-op-key.vitest.spec.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core'; -import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; - -describe('pdf worker op key', () => { - test('includes parser version in the reuse boundary', () => { - expect(buildPdfOpKey({ - documentId: 'doc-1', - namespace: null, - documentObjectKey: 'documents/doc-1.pdf', - })).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`); - }); -}); diff --git a/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts b/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts new file mode 100644 index 0000000..c9276ea --- /dev/null +++ b/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const hoisted = vi.hoisted(() => ({ + getParsedDocumentBlobByKey: vi.fn(), + resolveCurrentPdfParse: vi.fn(), +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + getParsedDocumentBlobByKey: hoisted.getParsedDocumentBlobByKey, + isMissingBlobError: vi.fn(() => false), +})); + +vi.mock('@/lib/server/pdf-parse/operation', () => ({ + resolveCurrentPdfParse: hoisted.resolveCurrentPdfParse, +})); + +describe('parsed PDF artifact envelope validation', () => { + beforeEach(() => { + hoisted.resolveCurrentPdfParse.mockReset(); + hoisted.resolveCurrentPdfParse.mockResolvedValue({ + artifact: { objectKey: 'openreader/parsed.json' }, + operation: null, + }); + hoisted.getParsedDocumentBlobByKey.mockReset(); + }); + + test('accepts a valid current artifact envelope', async () => { + hoisted.getParsedDocumentBlobByKey.mockResolvedValue(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'doc-1', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [{ pageNumber: 1, width: 100, height: 200, blocks: [] }], + }))); + + const { readCurrentParsedPdfArtifact } = await import('../../src/lib/server/pdf-parse/artifact'); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .resolves.toMatchObject({ key: 'openreader/parsed.json' }); + }); + + test('rejects malformed envelopes and document identity mismatches', async () => { + const { readCurrentParsedPdfArtifact } = await import('../../src/lib/server/pdf-parse/artifact'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 2, + documentId: 'doc-1', + pages: [], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('envelope is invalid'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'other-doc', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('document identity mismatch'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'doc-1', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [{ pageNumber: 1, width: 100, height: 200 }], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('envelope is invalid'); + }); +}); diff --git a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts index 98ceb4d..a6d2822 100644 --- a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts +++ b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts @@ -12,10 +12,10 @@ const hoisted = vi.hoisted(() => ({ requireAuthContext: vi.fn(), fetchPdfParseOperation: vi.fn(), isPdfParseOperationForDocument: vi.fn(), - getWorkerClientConfigFromEnv: vi.fn(), + openOperationEvents: vi.fn(), })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return hoisted.db; }, @@ -30,8 +30,10 @@ vi.mock('@/lib/server/pdf-parse/operation', () => ({ isPdfParseOperationForDocument: hoisted.isPdfParseOperationForDocument, })); -vi.mock('@/lib/server/compute/worker', () => ({ - getWorkerClientConfigFromEnv: hoisted.getWorkerClientConfigFromEnv, +vi.mock('@/lib/server/compute-worker/client', () => ({ + getComputeWorkerClient: () => ({ + openOperationEvents: hoisted.openOperationEvents, + }), })); vi.mock('@/lib/server/documents/blobstore', () => ({ @@ -84,12 +86,8 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { }); hoisted.isPdfParseOperationForDocument.mockReset(); hoisted.isPdfParseOperationForDocument.mockReturnValue(true); - hoisted.getWorkerClientConfigFromEnv.mockReset(); - hoisted.getWorkerClientConfigFromEnv.mockReturnValue({ - baseUrl: 'http://worker.local', - token: 'worker-token', - }); - global.fetch = vi.fn(async () => new Response( + hoisted.openOperationEvents.mockReset(); + hoisted.openOperationEvents.mockResolvedValue(new Response( 'event: snapshot\ndata: {"eventId":1,"snapshot":{"opId":"op-1","status":"running"}}\n\n', { status: 200, @@ -97,7 +95,7 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { 'Content-Type': 'text/event-stream; charset=utf-8', }, }, - )) as typeof fetch; + )); }); test('requires an opId', async () => { @@ -123,10 +121,9 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { expect(text).toContain('"opId":"op-1"'); expect(hoisted.fetchPdfParseOperation).toHaveBeenCalledWith('op-1'); expect(hoisted.isPdfParseOperationForDocument).toHaveBeenCalled(); - expect(global.fetch).toHaveBeenCalledWith( - 'http://worker.local/ops/op-1/events', - expect.anything(), - ); + expect(hoisted.openOperationEvents).toHaveBeenCalledWith('op-1', expect.objectContaining({ + signal: expect.any(AbortSignal), + })); }); test('denies proxying when the op does not belong to the document', async () => { @@ -140,7 +137,7 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { expect(response.status).toBe(404); await expect(response.json()).resolves.toMatchObject({ error: 'Operation not found' }); expect(hoisted.fetchPdfParseOperation).toHaveBeenCalledWith('op-1'); - expect(global.fetch).not.toHaveBeenCalled(); + expect(hoisted.openOperationEvents).not.toHaveBeenCalled(); }); afterEach(() => { diff --git a/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts b/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts index 7160c92..055bd3c 100644 --- a/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts +++ b/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts @@ -20,7 +20,7 @@ const hoisted = vi.hoisted(() => ({ buildComputeRateLimitedResponse: vi.fn(), })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return hoisted.db; }, diff --git a/tests/unit/reap-orphaned-blobs.vitest.spec.ts b/tests/unit/reap-orphaned-blobs.vitest.spec.ts index 667a242..7b5b8b5 100644 --- a/tests/unit/reap-orphaned-blobs.vitest.spec.ts +++ b/tests/unit/reap-orphaned-blobs.vitest.spec.ts @@ -26,7 +26,7 @@ vi.mock('@/lib/server/documents/blob-lease', () => ({ })), })); vi.mock('@/lib/server/errors/logging', () => ({ logDegraded: vi.fn() })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ db: { select: () => ({ from: () => ({ diff --git a/tests/unit/runtime-seed-json.vitest.spec.ts b/tests/unit/runtime-seed-json.vitest.spec.ts index fd710bf..4dfc520 100644 --- a/tests/unit/runtime-seed-json.vitest.spec.ts +++ b/tests/unit/runtime-seed-json.vitest.spec.ts @@ -1,8 +1,8 @@ import { and, eq, inArray, not } from 'drizzle-orm'; import { describe, expect, test } from 'vitest'; -import { db } from '../../src/db'; -import { adminProviders, adminSettings } from '../../src/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders, adminSettings } from '@openreader/database/schema'; import { RUNTIME_KEYS, seedRuntimeConfigFromValues } from '../../src/lib/server/admin/settings'; import { __seedInternals } from '../../src/lib/server/admin/seed'; diff --git a/tests/unit/scheduled-tasks-engine.vitest.spec.ts b/tests/unit/scheduled-tasks-engine.vitest.spec.ts index 715bac4..124fd1e 100644 --- a/tests/unit/scheduled-tasks-engine.vitest.spec.ts +++ b/tests/unit/scheduled-tasks-engine.vitest.spec.ts @@ -2,12 +2,12 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; import { eq } from 'drizzle-orm'; -import * as sqliteSchema from '../../src/db/schema_sqlite'; +import * as sqliteSchema from '@openreader/database/schema-sqlite'; // Back the engine with a real in-memory SQLite so the CAS claim, due-detection, // and nextRunAt arithmetic are exercised against real SQL rather than mocks. const holder = vi.hoisted(() => ({ db: null as unknown as ReturnType })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return holder.db; }, diff --git a/tests/unit/transfer-user-documents.vitest.spec.ts b/tests/unit/transfer-user-documents.vitest.spec.ts index 1444d1d..11c93d7 100644 --- a/tests/unit/transfer-user-documents.vitest.spec.ts +++ b/tests/unit/transfer-user-documents.vitest.spec.ts @@ -7,7 +7,7 @@ import { documents, ttsSegmentEntries, ttsSegmentVariants, -} from '../../src/db/schema_sqlite'; +} from '@openreader/database/schema-sqlite'; import { transferUserDocumentSettings, transferUserDocuments } from '../../src/lib/server/user/claim-data'; describe('transferUserDocuments', () => { diff --git a/tests/unit/tts-segments-cache.vitest.spec.ts b/tests/unit/tts-segments-cache.vitest.spec.ts index 18ce89b..70fdf0c 100644 --- a/tests/unit/tts-segments-cache.vitest.spec.ts +++ b/tests/unit/tts-segments-cache.vitest.spec.ts @@ -15,7 +15,7 @@ function resultBuilder(result: unknown[]) { }; } -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ db: { select: vi.fn(() => ({ from: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])), diff --git a/tests/unit/user-data-cleanup.vitest.spec.ts b/tests/unit/user-data-cleanup.vitest.spec.ts index ff8496b..af1cdad 100644 --- a/tests/unit/user-data-cleanup.vitest.spec.ts +++ b/tests/unit/user-data-cleanup.vitest.spec.ts @@ -18,7 +18,7 @@ function resultBuilder(result: unknown[]) { }; } -vi.mock('@/db', () => { +vi.mock('@openreader/database', () => { const database = { select: vi.fn(() => ({ from: vi.fn(() => ({ diff --git a/tsconfig.json b/tsconfig.json index 2c7a43b..9cd3715 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,11 +20,6 @@ ], "paths": { "@/*": ["./src/*"], - "@openreader/compute-core": ["./compute/core/src/index.ts"], - "@openreader/compute-core/api-contracts": ["./compute/core/src/api-contracts/index.ts"], - "@openreader/compute-core/control-plane": ["./compute/core/src/control-plane/index.ts"], - "@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"], - "@openreader/compute-core/types": ["./compute/core/src/types/index.ts"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], @@ -32,7 +27,7 @@ "node_modules", "docs-site", "docs-site/**", - "compute/worker", - "compute/worker/**" + "compute-worker", + "compute-worker/**" ] } diff --git a/vitest.config.ts b/vitest.config.ts index 3489ba3..b04ccea 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,17 +10,7 @@ if (!/^https?:\/\//.test(process.env.BASE_URL ?? '')) { } const srcDir = fileURLToPath(new URL('./src/', import.meta.url)); -const computeCoreIndex = fileURLToPath(new URL('./compute/core/src/index.ts', import.meta.url)); -const computeCoreApiContracts = fileURLToPath(new URL('./compute/core/src/api-contracts/index.ts', import.meta.url)); -const computeCoreControlPlane = fileURLToPath(new URL('./compute/core/src/control-plane/index.ts', import.meta.url)); -const computeCoreLocalRuntime = fileURLToPath(new URL('./compute/core/src/local-runtime.ts', import.meta.url)); -const computeCoreTypes = fileURLToPath(new URL('./compute/core/src/types/index.ts', import.meta.url)); const alias = [ - { find: /^@openreader\/compute-core$/, replacement: computeCoreIndex }, - { find: /^@openreader\/compute-core\/api-contracts$/, replacement: computeCoreApiContracts }, - { find: /^@openreader\/compute-core\/control-plane$/, replacement: computeCoreControlPlane }, - { find: /^@openreader\/compute-core\/local-runtime$/, replacement: computeCoreLocalRuntime }, - { find: /^@openreader\/compute-core\/types$/, replacement: computeCoreTypes }, { find: /^@\//, replacement: `${srcDir}` }, { find: '@', replacement: srcDir }, ]; @@ -44,19 +34,12 @@ export default defineConfig({ setupFiles: ['tests/unit/setup-env.ts'], }, }, - { - test: { - name: 'compute-core', - environment: 'node', - include: ['compute/core/tests/control-plane/**/*.test.ts'], - }, - }, { test: { name: 'compute-worker', environment: 'node', - include: ['compute/worker/tests/{unit,api}/**/*.test.ts'], - setupFiles: ['compute/worker/tests/setup-env.ts'], + include: ['packages/compute-worker/tests/{unit,api,compute}/**/*.test.ts'], + setupFiles: ['packages/compute-worker/tests/setup-env.ts'], }, }, ],