From e1483ab80ac73eed20069b32d530ae1dcf231b4c Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 11:00:01 -0600 Subject: [PATCH] refactor(worker): modularize worker entrypoint and add vitest infra move worker startup logic from server.ts to runtime.ts for better modularity add worker-loop-policy.ts for loop control abstraction delete legacy unit tests under tests/unit/ related to compute worker and control-plane add vitest config and test directories for compute-core and compute-worker add vitest workflow for CI update package.json scripts for new test commands and add vitest as dev dependency BREAKING CHANGE: worker entrypoint is now compute/worker/src/runtime.ts instead of server.ts; legacy Playwright-based unit tests for compute worker are removed in favor of Vitest --- .github/workflows/vitest.yml | 23 + .../tests/control-plane/orchestrator.test.ts | 95 ++ compute/core/tests/control-plane/sse.test.ts | 26 + .../tests/control-plane/state-machine.test.ts | 76 + compute/worker/src/runtime.ts | 1428 +++++++++++++++++ compute/worker/src/server.ts | 1336 +-------------- compute/worker/src/worker-loop-policy.ts | 22 + compute/worker/tests/api/routes.test.ts | 122 ++ .../tests/fixtures/fake-control-plane.ts | 119 ++ compute/worker/tests/setup-env.ts | 8 + compute/worker/tests/tsconfig.json | 9 + .../tests/unit/jetstream-adapters.test.ts | 134 +- .../tests/unit/key-and-progress.test.ts | 37 + .../tests/unit/worker-loop-policy.test.ts | 18 + compute/worker/tsconfig.json | 2 +- package.json | 6 +- pnpm-lock.yaml | 636 +++++++- tests/unit/compute-abort-like-error.spec.ts | 18 - tests/unit/compute-control-plane.spec.ts | 201 --- tests/unit/compute-worker-op-key.spec.ts | 29 - .../unit/compute-worker-pdf-progress.spec.ts | 32 - vitest.config.ts | 24 + 22 files changed, 2701 insertions(+), 1700 deletions(-) create mode 100644 .github/workflows/vitest.yml create mode 100644 compute/core/tests/control-plane/orchestrator.test.ts create mode 100644 compute/core/tests/control-plane/sse.test.ts create mode 100644 compute/core/tests/control-plane/state-machine.test.ts create mode 100644 compute/worker/src/runtime.ts create mode 100644 compute/worker/src/worker-loop-policy.ts create mode 100644 compute/worker/tests/api/routes.test.ts create mode 100644 compute/worker/tests/fixtures/fake-control-plane.ts create mode 100644 compute/worker/tests/setup-env.ts create mode 100644 compute/worker/tests/tsconfig.json rename tests/unit/compute-worker-control-plane-jetstream.spec.ts => compute/worker/tests/unit/jetstream-adapters.test.ts (55%) create mode 100644 compute/worker/tests/unit/key-and-progress.test.ts create mode 100644 compute/worker/tests/unit/worker-loop-policy.test.ts delete mode 100644 tests/unit/compute-abort-like-error.spec.ts delete mode 100644 tests/unit/compute-control-plane.spec.ts delete mode 100644 tests/unit/compute-worker-op-key.spec.ts delete mode 100644 tests/unit/compute-worker-pdf-progress.spec.ts create mode 100644 vitest.config.ts diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml new file mode 100644 index 0000000..652663a --- /dev/null +++ b/.github/workflows/vitest.yml @@ -0,0 +1,23 @@ +name: Vitest Tests + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + vitest-testing: + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: lts/* + package-manager-cache: false + - uses: pnpm/action-setup@v6 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Run Vitest suites + run: pnpm test:vitest diff --git a/compute/core/tests/control-plane/orchestrator.test.ts b/compute/core/tests/control-plane/orchestrator.test.ts new file mode 100644 index 0000000..2a03adf --- /dev/null +++ b/compute/core/tests/control-plane/orchestrator.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, test } from 'vitest'; +import type { WorkerOperationRequest } from '../../src/api-contracts'; +import { + InMemoryOperationEventStream, + InMemoryOperationQueue, + InMemoryOperationStateStore, + OperationOrchestrator, +} from '../../src/control-plane'; + +function buildRequest(opKey: string): WorkerOperationRequest { + return { + kind: 'pdf_layout', + opKey, + payload: { + documentId: `doc-${opKey}`, + namespace: null, + documentObjectKey: `s3://bucket/${opKey}.pdf`, + }, + }; +} + +describe('operation orchestrator', () => { + test('reuses fresh operation and replaces stale operation', async () => { + let now = 1_000; + let nextId = 1; + const queue = new InMemoryOperationQueue(); + const stateStore = new InMemoryOperationStateStore(); + const eventStream = new InMemoryOperationEventStream(); + + const orchestrator = new OperationOrchestrator({ + queue, + stateStore, + eventStream, + config: { opStaleMs: 2_000, maxCasRetries: 5 }, + clock: { now: () => now }, + idFactory: { + opId: () => `op-${nextId}`, + jobId: () => `job-${nextId++}`, + }, + }); + + const request = buildRequest('shared-op'); + + const first = await orchestrator.enqueueOrReuse(request); + expect(first.opId).toBe('op-1'); + + now = 2_000; + const reused = await orchestrator.enqueueOrReuse(request); + expect(reused.opId).toBe('op-1'); + + await orchestrator.markRunning({ opId: first.opId, updatedAt: 2_100 }); + + now = 6_000; + const replaced = await orchestrator.enqueueOrReuse(request); + expect(replaced.opId).toBe('op-2'); + expect(await stateStore.getOpIndex('shared-op')).toEqual({ opId: 'op-2' }); + expect(queue.size('pdf_layout')).toBe(2); + }); + + test('survives transient CAS conflict and eventually creates operation', async () => { + const queue = new InMemoryOperationQueue(); + const eventStream = new InMemoryOperationEventStream(); + const store = new InMemoryOperationStateStore(); + + let firstAttempt = true; + const conflictStore = { + getOpState: store.getOpState.bind(store), + putOpState: store.putOpState.bind(store), + getOpIndex: store.getOpIndex.bind(store), + compareAndSetOpIndex: async (input: { opKey: string; newOpId: string; expectedOpId: string | null }) => { + if (firstAttempt && input.expectedOpId === null) { + firstAttempt = false; + return false; + } + return store.compareAndSetOpIndex(input); + }, + }; + + let id = 1; + const orchestrator = new OperationOrchestrator({ + queue, + stateStore: conflictStore, + eventStream, + config: { opStaleMs: 2_000, maxCasRetries: 4 }, + idFactory: { + opId: () => `op-${id}`, + jobId: () => `job-${id++}`, + }, + }); + + const created = await orchestrator.enqueueOrReuse(buildRequest('cas-key')); + expect(created.opId).toMatch(/^op-/); + expect(await store.getOpIndex('cas-key')).toEqual({ opId: created.opId }); + }); +}); diff --git a/compute/core/tests/control-plane/sse.test.ts b/compute/core/tests/control-plane/sse.test.ts new file mode 100644 index 0000000..e207950 --- /dev/null +++ b/compute/core/tests/control-plane/sse.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from 'vitest'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse'; + +describe('sse codec', () => { + test('encodes event id and payload and decodes both reliably', () => { + const frame = encodeSseFrame({ + id: 42, + event: 'snapshot', + data: { ok: true, opId: 'op-1' }, + }); + + expect(frame).toContain('event: snapshot'); + expect(parseSseEventId(frame)).toBe(42); + expect(parseSsePayload(frame)).toBe('{"ok":true,"opId":"op-1"}'); + }); + + test('supports multiline data payload', () => { + const frame = encodeSseFrame({ + id: 5, + data: 'line1\nline2', + }); + + expect(parseSseEventId(frame)).toBe(5); + expect(parseSsePayload(frame)).toBe('line1\nline2'); + }); +}); diff --git a/compute/core/tests/control-plane/state-machine.test.ts b/compute/core/tests/control-plane/state-machine.test.ts new file mode 100644 index 0000000..e9617cb --- /dev/null +++ b/compute/core/tests/control-plane/state-machine.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from 'vitest'; +import type { WorkerOperationState } from '../../src/api-contracts'; +import { + explainReplacementReason, + isInflightStatus, + isTerminalStatus, + shouldReuseExistingOperation, +} from '../../src/control-plane/state-machine'; + +function runningState(overrides: Partial = {}): WorkerOperationState { + return { + opId: 'op-1', + opKey: 'key-1', + kind: 'pdf_layout', + jobId: 'job-1', + status: 'running', + queuedAt: 1_000, + updatedAt: 2_000, + ...overrides, + }; +} + +describe('state-machine decisions', () => { + test('identifies terminal and inflight states', () => { + expect(isTerminalStatus('succeeded')).toBe(true); + expect(isTerminalStatus('failed')).toBe(true); + expect(isTerminalStatus('queued')).toBe(false); + + expect(isInflightStatus('queued')).toBe(true); + expect(isInflightStatus('running')).toBe(true); + expect(isInflightStatus('failed')).toBe(false); + }); + + test('reuses fresh inflight operation and rejects stale inflight operation', () => { + const current = runningState(); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 2_900, + opStaleMs: 1_000, + })).toBe(true); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 3_100, + opStaleMs: 1_000, + })).toBe(false); + + expect(explainReplacementReason({ + current, + requestKind: 'pdf_layout', + now: 3_100, + opStaleMs: 1_000, + })).toBe('stale_running'); + }); + + test('never reuses kind-mismatched operation', () => { + const current = runningState({ kind: 'whisper_align' }); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 2_100, + opStaleMs: 10_000, + })).toBe(false); + + expect(explainReplacementReason({ + current, + requestKind: 'pdf_layout', + now: 2_100, + opStaleMs: 10_000, + })).toBe('kind_mismatch'); + }); +}); diff --git a/compute/worker/src/runtime.ts b/compute/worker/src/runtime.ts new file mode 100644 index 0000000..1804855 --- /dev/null +++ b/compute/worker/src/runtime.ts @@ -0,0 +1,1428 @@ +import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import { + connect, + nanos, + credsAuthenticator, + type NatsConnection, +} from '@nats-io/transport-node'; +import { + AckPolicy, + DeliverPolicy, + ReplayPolicy, + RetentionPolicy, + StorageType, + jetstream, + jetstreamManager, + type Consumer, + type JetStreamClient, + type JetStreamManager, + type JsMsg, +} from '@nats-io/jetstream'; +import { Kvm } from '@nats-io/kv'; +import { + ensureComputeModels, + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, +} from '@openreader/compute-core/local-runtime'; +import { + getComputeTimeoutConfig, + getComputeOpStaleMs, + getAvailableCpuCores, + getOnnxThreadsPerJob, + withIdleTimeoutAndHardCap, + withTimeout, +} from '@openreader/compute-core'; +import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + WorkerOperationEvent, + WhisperAlignJobRequest, + WhisperAlignJobResult, + WorkerJobState, + WorkerJobTiming, + WorkerOperationKind, + WorkerOperationRequest, + WorkerOperationState, + PdfLayoutProgress, +} from '@openreader/compute-core/api-contracts'; +import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + JetStreamOperationEventStream, + OP_EVENTS_SUBJECT_WILDCARD, + JetStreamOperationQueue, + JetStreamOperationStateStore, + hashOpKey, +} from './control-plane/jetstream'; +import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; +import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; +import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; + +const JOBS_STREAM_NAME = 'compute_jobs'; +const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; +const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; +const WHISPER_CONSUMER_NAME = 'compute_whisper'; +const LAYOUT_CONSUMER_NAME = 'compute_layout'; +const EVENTS_STREAM_NAME = 'compute_events'; +const COMPUTE_STATE_BUCKET = 'compute_state'; +const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; +const LOOP_ERROR_BACKOFF_MS = 500; +const RUNNING_HEARTBEAT_MS = 5000; +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const WHISPER_MAX_DELIVER = 1; +const NATS_API_TIMEOUT_MS = 60_000; +// Disconnect from NATS after this much continuous idle so the worker stops +// generating outbound traffic (pull polling + keepalive PINGs) and Railway can +// put it to sleep. Reconnect happens lazily on the next inbound request. +const IDLE_DISCONNECT_MS = 120_000; +const IDLE_CHECK_INTERVAL_MS = 5_000; +// Bounded pull window so consumer loops yield periodically and can be stopped +// cleanly when going idle, instead of blocking on a long-lived pull. +const PULL_EXPIRES_MS = 5_000; +const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); +const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); +const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { + whisper_align: 15_000, + pdf_layout: 120_000, +}; + +interface QueuedJob { + jobId: string; + opId: string; + opKey: string; + kind: WorkerOperationKind; + queuedAt: number; + payload: TPayload; +} + +interface NatsSession { + nc: NatsConnection; + js: JetStreamClient; + jsm: JetStreamManager; + kv: Awaited>; + whisperConsumer: Consumer; + layoutConsumer: Consumer; +} + +type StreamedOperationState = WorkerOperationState; + +interface OperationEventStreamLike { + subscribe(input: { + opId: string; + sinceEventId?: number; + onEvent: (event: WorkerOperationEvent) => void | Promise; + onError?: (error: unknown) => void; + }): Promise<() => void>; +} + +interface OperationStateStoreLike { + getOpState(opId: string): Promise; +} + +interface OrchestratorLike { + enqueueOrReuse(request: WorkerOperationRequest): Promise; + markRunning(input: { + opId: string; + startedAt?: number; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markProgress(input: { + opId: string; + progress: WorkerOperationState['progress']; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { + opId: string; + result: unknown; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +export interface ComputeWorkerRouteDeps { + orchestrator: OrchestratorLike; + operationStateStore: OperationStateStoreLike; + operationEventStream: OperationEventStreamLike; +} + +export interface CreateComputeWorkerAppOptions { + host?: string; + port?: number; + workerToken?: string; + routeDeps?: ComputeWorkerRouteDeps; + disableWorkers?: boolean; +} + +export interface ComputeWorkerApp { + app: FastifyInstance; + host: string; + port: number; + start(options?: { registerSignalHandlers?: boolean }): Promise; + close(): Promise; +} + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function readIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +function normalizeNatsReplicas(value: number): number { + if (value === 3 || value === 5) return value; + return 1; +} + +function parseBoolEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const normalized = raw.toLowerCase(); + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; +} + +function buildLoggerConfig(): boolean | Record { + const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); + const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; + if (format === 'json') { + return { + level, + base: null, + }; + } + return { + level, + base: null, + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + }, + }; +} + +function normalizeS3Prefix(prefix: string | undefined): string { + const value = (prefix || 'openreader').trim(); + return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; +} + +function sanitizeNamespace(namespace: string | null): string | null { + if (!namespace) return null; + return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; +} + +function documentParsedKey(id: string, namespace: string | null, prefix: string): string { + if (!DOCUMENT_ID_REGEX.test(id)) { + throw new Error(`Invalid document id: ${id}`); + } + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + return `${prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; +} + +function buildS3Client(): S3Client { + const bucket = requireEnv('S3_BUCKET'); + const region = requireEnv('S3_REGION'); + const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); + const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); + const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; + const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); + + void bucket; + + return new S3Client({ + region, + endpoint, + forcePathStyle, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + if (typeof body === 'object' && body !== null && 'on' in body) { + const stream = body as NodeJS.ReadableStream; + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); + } + return Buffer.concat(chunks); + } + throw new Error('Unsupported S3 response body type'); +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function isAuthed(request: FastifyRequest, expectedToken: string): boolean { + const auth = request.headers.authorization; + if (!auth?.startsWith('Bearer ')) return false; + const token = auth.slice('Bearer '.length).trim(); + return token === expectedToken; +} + +function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { + if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; + return Math.max(0, Math.floor((end as number) - (start as number))); +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + return String(error); +} + +function requestPath(request: FastifyRequest): string { + return request.url.split('?')[0] ?? request.url; +} + +function isHealthPath(path: string): boolean { + return path === '/health/live' || path === '/health/ready'; +} + +function extractTraceId(request: FastifyRequest): string | null { + const header = request.headers['x-openreader-trace-id']; + if (Array.isArray(header)) return header[0] ?? null; + return typeof header === 'string' ? header : null; +} + +function extractOpId(request: FastifyRequest, path: string): string | null { + const params = request.params as { opId?: unknown } | undefined; + if (params && typeof params.opId === 'string' && params.opId.trim()) { + return params.opId.trim(); + } + const match = path.match(/^\/ops\/([^/]+)/); + if (!match?.[1]) return null; + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } +} + +function isAlreadyExistsError(error: unknown): boolean { + const message = toErrorMessage(error).toLowerCase(); + return message.includes('already in use') || message.includes('already exists'); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +class ConcurrencyGate { + private readonly maxInFlight: number; + private inFlight = 0; + private readonly queue: Array<() => void> = []; + + constructor(limit: number) { + this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; + } + + async acquire(): Promise { + if (this.inFlight < this.maxInFlight) { + this.inFlight += 1; + return; + } + + await new Promise((resolve) => { + this.queue.push(() => { + this.inFlight += 1; + resolve(); + }); + }); + } + + release(): void { + this.inFlight = Math.max(0, this.inFlight - 1); + const next = this.queue.shift(); + if (next) next(); + } +} + +function isTerminalStatus(status: WorkerJobState): boolean { + return status === 'succeeded' || status === 'failed'; +} + +function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { + if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; + const maybe = result as { parsedObjectKey?: unknown }; + return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; +} + +const alignSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +const layoutSchema = z.object({ + documentId: z.string().trim().min(1), + namespace: z.string().trim().min(1).max(128).nullable(), + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +const operationCreateSchema = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('whisper_align'), + opKey: z.string().trim().min(1).max(1024), + payload: alignSchema, + }), + z.object({ + kind: z.literal('pdf_layout'), + opKey: z.string().trim().min(1).max(1024), + payload: layoutSchema, + }), +]); + +async function ensureJetStreamResources( + jsm: JetStreamManager, + whisperTimeoutMs: number, + pdfTimeoutMs: number, + pdfAttempts: number, + jobsMaxBytes: number, + eventsMaxBytes: number, + natsReplicas: number, +): Promise { + const streamConfig = { + name: JOBS_STREAM_NAME, + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + retention: RetentionPolicy.Workqueue, + storage: StorageType.File, + max_bytes: jobsMaxBytes, + num_replicas: natsReplicas, + }; + + try { + await jsm.streams.add(streamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.streams.update(JOBS_STREAM_NAME, { + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + max_bytes: jobsMaxBytes, + num_replicas: natsReplicas, + }); + } + + const eventsStreamConfig = { + name: EVENTS_STREAM_NAME, + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + retention: RetentionPolicy.Limits, + storage: StorageType.File, + max_bytes: eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: natsReplicas, + }; + + try { + await jsm.streams.add(eventsStreamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.streams.update(EVENTS_STREAM_NAME, { + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + max_bytes: eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: natsReplicas, + }); + } + + const ensureConsumer = async ( + name: string, + subject: string, + ackWaitMs: number, + maxDeliver: number, + ): Promise => { + const config = { + durable_name: name, + ack_policy: AckPolicy.Explicit, + deliver_policy: DeliverPolicy.All, + replay_policy: ReplayPolicy.Instant, + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }; + + try { + await jsm.consumers.add(JOBS_STREAM_NAME, config); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.consumers.update(JOBS_STREAM_NAME, name, { + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }); + } + }; + + await Promise.all([ + ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), + ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), + ]); +} + +export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { + const port = options.port ?? readIntEnv('PORT', 8081); + const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); + const workerToken = options.workerToken ?? requireEnv('COMPUTE_WORKER_TOKEN'); + const disableWorkers = options.disableWorkers ?? false; + const natsUrl = requireEnv('NATS_URL'); + const timeoutConfig = getComputeTimeoutConfig(); + + const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); + const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; + const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; + const pdfHardCapMs = timeoutConfig.pdfHardCapMs; + const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); + const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); + const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); + const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); + const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); + const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); + const opStaleMs = getComputeOpStaleMs(); + + const connectOpts: Parameters[0] = { servers: natsUrl }; + const natsCreds = process.env.NATS_CREDS?.trim(); + const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); + + if (natsCreds) { + console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); + connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); + } else if (natsCredsFile) { + console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); + const { readFileSync } = await import('node:fs'); + const credsData = readFileSync(natsCredsFile); + connectOpts.authenticator = credsAuthenticator(credsData); + } + + // Lazy NATS connection lifecycle. The worker connects on demand (first request + // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle + // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is + // transparent: any inbound /ops request both wakes the container and re-establishes + // the session via ensureConnected(). + let session: NatsSession | null = null; + let connecting: Promise | null = null; + let workerLoops: Promise[] = []; + let idleTimer: NodeJS.Timeout | null = null; + let stopping = false; + let loopStopRequested = false; + + // Activity accounting feeding the idle detector. The worker is considered idle + // only when no HTTP request is in flight, no SSE stream is open, no job is + // processing, and nothing has happened for IDLE_DISCONNECT_MS. + let inFlightHttp = 0; + let activeSse = 0; + let inFlightJobs = 0; + let lastActivityAt = Date.now(); + const jobGate = new ConcurrencyGate(jobConcurrency); + + const markActivity = (): void => { + lastActivityAt = Date.now(); + }; + + function startIdleTimer(): void { + if (idleTimer) return; + idleTimer = setInterval(() => { + if (!session || stopping) return; + if (inFlightHttp > 0 || activeSse > 0 || inFlightJobs > 0) return; + if (Date.now() - lastActivityAt < IDLE_DISCONNECT_MS) return; + void disconnect('idle'); + }, IDLE_CHECK_INTERVAL_MS); + // Don't let the idle checker keep the process alive on its own. + idleTimer.unref?.(); + } + + async function disconnect(reason: string): Promise { + const current = session; + if (!current) return; + // Clear synchronously (before any await) so concurrent requests reconnect a + // fresh session instead of using the connection we're about to close. + session = null; + loopStopRequested = true; + if (idleTimer) { + clearInterval(idleTimer); + idleTimer = null; + } + try { + await current.nc.close(); + } catch { + // ignore close errors + } + await Promise.allSettled(workerLoops); + workerLoops = []; + loopStopRequested = false; + app.log.info({ reason }, 'nats disconnected'); + } + + async function ensureConnected(): Promise { + if (session) return session; + if (connecting) return connecting; + connecting = (async () => { + const nc: NatsConnection = await connect(connectOpts); + const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); + const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); + await ensureJetStreamResources( + jsm, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + jobsStreamMaxBytes, + eventsStreamMaxBytes, + natsReplicas, + ); + const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { + replicas: natsReplicas, + history: 1, + ttl: COMPUTE_STATE_TTL_MS, + max_bytes: jobStatesMaxBytes, + }); + const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); + const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); + const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; + session = next; + markActivity(); + startWorkerLoops(next); + startIdleTimer(); + // Safety net: if the connection closes for any reason (network drop after + // exhausting reconnects, or our own disconnect), drop the stale session so + // the next request reconnects cleanly. + void nc.closed().then(() => { + if (session?.nc === nc) session = null; + }); + app.log.info('nats connected'); + return next; + })(); + try { + return await connecting; + } finally { + connecting = null; + } + } + + const s3 = disableWorkers ? null : buildS3Client(); + const s3Bucket = disableWorkers ? '' : requireEnv('S3_BUCKET'); + const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); + + const ensureSafeKey = (key: string): string => { + const trimmed = key.trim(); + if (!trimmed.startsWith(`${s3Prefix}/`)) { + throw new Error('Object key prefix mismatch'); + } + return trimmed; + }; + + const readObjectByKey = async (key: string): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const safeKey = ensureSafeKey(key); + const response = await s3.send(new GetObjectCommand({ + Bucket: s3Bucket, + Key: safeKey, + })); + const bytes = await bodyToBuffer(response.Body); + return toArrayBuffer(new Uint8Array(bytes)); + }; + + const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const key = documentParsedKey(documentId, namespace, s3Prefix); + const body = Buffer.from(JSON.stringify(parsed)); + await s3.send(new PutObjectCommand({ + Bucket: s3Bucket, + Key: key, + Body: body, + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + })); + return key; + }; + + if (prewarmModels && !disableWorkers) { + await ensureComputeModels(); + } + + const app = Fastify({ + logger: buildLoggerConfig(), + disableRequestLogging: true, + }); + app.log.info({ + jobConcurrency, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + opStaleMs, + availableCpuCores: getAvailableCpuCores(), + onnxThreadsPerJob: getOnnxThreadsPerJob(), + natsApiTimeoutMs: NATS_API_TIMEOUT_MS, + natsReplicas, + eventsStreamMaxBytes, + pdfLayoutHardCapMs: pdfHardCapMs, + }, 'compute runtime config'); + + const whisperJobCodec = createJsonCodec>(); + const layoutJobCodec = createJsonCodec>(); + + const defaultOperationStateStore = new JetStreamOperationStateStore({ + getKv: async () => (await ensureConnected()).kv, + }); + + const defaultOperationEventStream = new JetStreamOperationEventStream({ + getJs: async () => (await ensureConnected()).js, + getJsm: async () => (await ensureConnected()).jsm, + eventsStreamName: EVENTS_STREAM_NAME, + }); + + const operationQueue = new JetStreamOperationQueue({ + getJs: async () => (await ensureConnected()).js, + whisperSubject: WHISPER_JOBS_SUBJECT, + layoutSubject: LAYOUT_JOBS_SUBJECT, + }); + + const defaultOrchestrator = new OperationOrchestrator({ + queue: operationQueue, + stateStore: defaultOperationStateStore, + eventStream: defaultOperationEventStream, + config: { + opStaleMs, + maxCasRetries: 10, + }, + }); + + const operationStateStore = options.routeDeps?.operationStateStore ?? defaultOperationStateStore; + const operationEventStream = options.routeDeps?.operationEventStream ?? defaultOperationEventStream; + const orchestrator = options.routeDeps?.orchestrator ?? defaultOrchestrator; + + const getOpState = async (opId: string): Promise => { + return await operationStateStore.getOpState(opId); + }; + + const releaseHttp = (request: FastifyRequest): void => { + const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; + if (!counted[REQUEST_COUNTED_KEY]) return; + counted[REQUEST_COUNTED_KEY] = false; + inFlightHttp = Math.max(0, inFlightHttp - 1); + markActivity(); + }; + + app.addHook('onRequest', async (request, reply) => { + const path = requestPath(request); + (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); + // Count every request as in-flight activity so the idle detector never + // disconnects mid-request. Released in onResponse, or manually after hijack + // for SSE streams (where onResponse does not fire). + (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; + inFlightHttp += 1; + markActivity(); + if (isHealthPath(path)) return; + if (!isAuthed(request, workerToken)) { + return reply.code(401).send({ error: 'Unauthorized' }); + } + return; + }); + + app.addHook('onResponse', async (request, reply) => { + releaseHttp(request); + const path = requestPath(request); + if (isHealthPath(path)) return; + if (reply.statusCode >= 500) { + const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; + const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; + app.log.error({ + reqId: request.id, + method: request.method, + path, + statusCode: reply.statusCode, + durationMs, + traceId: extractTraceId(request) ?? null, + opId: extractOpId(request, path), + }, 'http.error'); + } + }); + + app.get('/health/live', async () => ({ ok: true })); + + // Reports readiness without forcing a NATS round-trip. Probing NATS here would + // reconnect (and keep) the connection open, defeating idle sleep, so we only + // report the current connection state. The worker reconnects lazily on the next + // /ops request regardless of what this returns. + app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); + + app.post('/ops', async (request, reply) => { + const parsed = operationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { + error: 'Invalid request body', + issues: parsed.error.issues, + }; + } + + const requestOp = parsed.data as WorkerOperationRequest; + const op = await orchestrator.enqueueOrReuse(requestOp); + app.log.info({ + kind: requestOp.kind, + opId: op.opId, + jobId: op.jobId, + status: op.status, + opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), + }, 'op.accepted'); + reply.code(202); + return op; + }); + + app.get('/ops/:opId', async (request, reply) => { + const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const state = await getOpState(params.data.opId); + if (!state) { + reply.code(404); + return { error: 'Operation not found' }; + } + + return state; + }); + + app.get('/ops/:opId/events', async (request, reply) => { + const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const initial = await getOpState(params.data.opId); + if (!initial) { + reply.code(404); + return { error: 'Operation not found' }; + } + + const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; + const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); + const lastEventIdHeader = request.headers['last-event-id']; + const cursorFromHeader = Number( + Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), + ); + const sinceEventId = Math.max( + 0, + Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, + Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, + ); + + reply.hijack(); + // onResponse will not fire for a hijacked reply, so release the HTTP in-flight + // count here and track the long-lived stream via activeSse instead. + releaseHttp(request); + activeSse += 1; + markActivity(); + reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); + reply.raw.setHeader('Connection', 'keep-alive'); + reply.raw.setHeader('X-Accel-Buffering', 'no'); + + let closed = false; + let unsubscribe: (() => void) | null = null; + + const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { + if (closed || reply.raw.writableEnded) return; + const frameEvent: WorkerOperationEvent = { + eventId, + snapshot, + }; + reply.raw.write(encodeSseFrame({ + id: eventId, + event: 'snapshot', + data: frameEvent, + })); + }; + + const closeStream = (): void => { + if (closed) return; + closed = true; + if (unsubscribe) { + unsubscribe(); + unsubscribe = null; + } + activeSse = Math.max(0, activeSse - 1); + markActivity(); + if (!reply.raw.writableEnded) { + reply.raw.end(); + } + }; + + request.raw.on('close', () => { + closeStream(); + }); + + try { + let current = initial; + let signature = JSON.stringify(current); + writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); + if (isTerminalStatus(current.status)) { + return reply; + } + + unsubscribe = await operationEventStream.subscribe({ + opId: params.data.opId, + sinceEventId, + onEvent: (event) => { + if (closed) return; + if (event.snapshot.opId !== params.data.opId) return; + const nextSignature = JSON.stringify(event.snapshot); + if (nextSignature !== signature) { + current = event.snapshot; + signature = nextSignature; + writeSnapshot(current, event.eventId); + } + if (isTerminalStatus(event.snapshot.status)) { + closeStream(); + } + }, + onError: (error) => { + app.log.warn({ + opId: params.data.opId, + error: toErrorMessage(error), + }, 'op events stream loop error'); + closeStream(); + }, + }); + + await new Promise((resolve) => { + request.raw.once('close', () => resolve()); + }); + } catch (error) { + app.log.warn({ + opId: params.data.opId, + error: toErrorMessage(error), + }, 'op events stream loop error'); + } finally { + closeStream(); + } + + return reply; + }); + + const runWhisper = async ( + payload: WhisperAlignJobRequest, + queueWaitMs: number, + ): Promise => { + const parsed = alignSchema.parse(payload); + + const s3FetchStartedAt = Date.now(); + const audioBuffer = await withTimeout( + readObjectByKey(parsed.audioObjectKey), + whisperTimeoutMs, + 'whisper s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + + const computeStartedAt = Date.now(); + const result = await withTimeout( + runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: parsed.text, + cacheKey: parsed.cacheKey, + lang: parsed.lang, + }), + whisperTimeoutMs, + 'whisper alignment job', + ); + + const computeMs = Date.now() - computeStartedAt; + return { + ...result, + timing: { + queueWaitMs, + s3FetchMs, + computeMs, + }, + }; + }; + + const runLayout = async ( + payload: PdfLayoutJobRequest, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ): Promise => { + const parsed = layoutSchema.parse(payload); + + const s3FetchStartedAt = Date.now(); + const pdfBytes = await withTimeout( + readObjectByKey(parsed.documentObjectKey), + Math.max(pdfTimeoutMs, 1_000), + 'pdf s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + + let lastTotalPages = 0; + let lastPagesParsed = 0; + const computeStartedAt = Date.now(); + const result = await withIdleTimeoutAndHardCap({ + idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), + hardCapMs: pdfHardCapMs, + label: 'pdf layout job', + run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ + documentId: parsed.documentId, + pdfBytes, + onPageStarted: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + if (!hooks?.onProgress) return; + await hooks.onProgress(buildInferProgressForPageStart({ + pageNumber, + totalPages, + })); + }, + onPageParsed: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + lastPagesParsed = pageNumber; + if (!hooks?.onProgress) return; + await hooks.onProgress(buildInferProgressForPageParsed({ + pageNumber, + totalPages, + })); + }, + }), + }); + + const computeMs = Date.now() - computeStartedAt; + if (hooks?.onProgress && lastTotalPages > 0) { + await hooks.onProgress({ + totalPages: lastTotalPages, + pagesParsed: lastPagesParsed, + currentPage: lastPagesParsed || undefined, + phase: 'merge', + }); + } + const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); + return { + parsedObjectKey, + timing: { + queueWaitMs, + s3FetchMs, + computeMs, + }, + }; + }; + + type ProcessMessageInput = { + msg: JsMsg; + codec: JsonCodec>; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + workerLabel: string; + }; + + type ProcessMessageContext = { + decoded: QueuedJob; + workerLabel: string; + startedAt: number; + queueWaitTiming?: { queueWaitMs: number }; + latestProgress?: PdfLayoutProgress; + }; + + function extractTiming(result: unknown): WorkerJobTiming | undefined { + if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; + return (result as { timing?: WorkerJobTiming }).timing; + } + + async function markRunning( + context: ProcessMessageContext, + updatedAt: number, + options?: { includeStartedAt?: boolean }, + ): Promise { + if (context.latestProgress) { + await orchestrator.markProgress({ + opId: context.decoded.opId, + progress: context.latestProgress, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + return; + } + + await orchestrator.markRunning({ + opId: context.decoded.opId, + ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + } + + async function markProgress( + context: ProcessMessageContext, + progress: PdfLayoutProgress, + updatedAt: number, + ): Promise { + context.latestProgress = progress; + await markRunning(context, updatedAt); + } + + async function markTerminal(input: { + context: ProcessMessageContext; + status: 'succeeded' | 'failed'; + result?: TResult; + errorMessage?: string; + timing?: WorkerJobTiming; + updatedAt: number; + }): Promise { + if (input.status === 'succeeded') { + await orchestrator.markSucceeded({ + opId: input.context.decoded.opId, + result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, + updatedAt: input.updatedAt, + ...(input.timing ? { timing: input.timing } : {}), + }); + return; + } + + await orchestrator.markFailed({ + opId: input.context.decoded.opId, + error: { message: input.errorMessage ?? 'unknown worker failure' }, + updatedAt: input.updatedAt, + ...(input.timing ? { timing: input.timing } : {}), + }); + } + + async function handleRetry(input: { + context: ProcessMessageContext | null; + msg: JsMsg; + errorMessage: string; + }): Promise { + const deliveryCount = input.msg.info.deliveryCount; + const kind = input.context?.decoded.kind ?? 'pdf_layout'; + const retryAction = decideRetryAction({ + kind, + deliveryCount, + pdfAttempts, + whisperMaxDeliver: WHISPER_MAX_DELIVER, + }); + const hasRetriesLeft = retryAction === 'nak_retry'; + const maxAttempts = kind === 'whisper_align' ? WHISPER_MAX_DELIVER : pdfAttempts; + + if (input.context) { + const now = Date.now(); + const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); + const persistOpUpdate = hasRetriesLeft + ? (input.context.latestProgress + ? orchestrator.markProgress({ + opId: input.context.decoded.opId, + progress: input.context.latestProgress, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + }) + : orchestrator.markRunning({ + opId: input.context.decoded.opId, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + })) + : markTerminal({ + context: input.context, + status: 'failed', + errorMessage: input.errorMessage, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + }); + + await persistOpUpdate.catch((stateError) => { + app.log.error({ + worker: input.context?.workerLabel, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation state'); + }); + } + + if (hasRetriesLeft) { + input.msg.nak(); + app.log.error({ + worker: input.context?.workerLabel, + kind: input.context?.decoded.kind, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + status: 'running', + error: input.errorMessage, + deliveryCount, + maxAttempts, + retryAction: 'nack_retry', + }, 'job.terminal'); + return; + } + + input.msg.term(input.errorMessage); + app.log.error({ + worker: input.context?.workerLabel, + kind: input.context?.decoded.kind, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + status: 'failed', + error: input.errorMessage, + deliveryCount, + maxAttempts, + retrySuppressed: kind === 'whisper_align' ? 'whisper_align' : undefined, + retryAction: 'term', + }, 'job.terminal'); + } + + async function processMessage(input: ProcessMessageInput): Promise { + let context: ProcessMessageContext | null = null; + let heartbeat: NodeJS.Timeout | null = null; + try { + const decoded = input.codec.decode(input.msg.data); + const startedAt = Date.now(); + context = { + decoded, + workerLabel: input.workerLabel, + startedAt, + queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), + }; + + await markRunning(context, startedAt); + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, + deliveryCount: input.msg.info.deliveryCount, + }, 'job.started'); + + heartbeat = setInterval(() => { + const now = Date.now(); + void markRunning(context!, now).catch((stateError) => { + app.log.error({ + worker: input.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation heartbeat state'); + }); + }, RUNNING_HEARTBEAT_MS); + + const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { + onProgress: async (progress) => { + await markProgress(context!, progress, Date.now()); + }, + }); + const resultTiming = extractTiming(result); + const now = Date.now(); + + await markTerminal({ + context, + status: 'succeeded', + result, + timing: resultTiming, + updatedAt: now, + }); + + input.msg.ack(); + const terminalDurationMs = safeDurationMs(context.startedAt, now); + const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; + if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + durationMs: terminalDurationMs ?? null, + timing: resultTiming ?? null, + }, 'job.stage'); + } + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + status: 'succeeded', + durationMs: terminalDurationMs ?? null, + resultRef: extractResultRef(decoded.kind, result), + timing: resultTiming ?? null, + }, 'job.terminal'); + } catch (error) { + await handleRetry({ + context, + msg: input.msg, + errorMessage: toErrorMessage(error), + }); + } finally { + if (heartbeat) clearInterval(heartbeat); + } + } + + async function createWorkerLoop(input: { + owner: NatsSession; + consumer: Consumer; + codec: JsonCodec>; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + workerLabel: string; + }): Promise { + // Exit when the loop's connection is no longer the active session (idle + // disconnect, unexpected close, or replaced by a reconnect). + const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; + while (!detached()) { + let msg: JsMsg | null = null; + try { + try { + // Bounded pull so the loop yields periodically and exits promptly when + // the session is torn down for idle (nc.close() rejects the pending pull). + msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); + } catch (error) { + if (detached()) return; + app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); + await sleep(LOOP_ERROR_BACKOFF_MS); + continue; + } + + // An empty pull is not activity; let the idle window advance. + if (!msg) continue; + markActivity(); + inFlightJobs += 1; + await jobGate.acquire(); + if (detached()) { + return; + } + await processMessage({ + msg, + codec: input.codec, + run: input.run, + workerLabel: input.workerLabel, + }); + } finally { + if (msg) { + jobGate.release(); + inFlightJobs = Math.max(0, inFlightJobs - 1); + markActivity(); + } + } + } + } + + function startWorkerLoops(active: NatsSession): void { + if (disableWorkers) return; + // Always starts a fresh set bound to the new session. Any loops from a prior + // session self-terminate once they observe session !== their owner. + workerLoops = []; + loopStopRequested = false; + for (let i = 0; i < jobConcurrency; i += 1) { + workerLoops.push(createWorkerLoop({ + owner: active, + consumer: active.whisperConsumer, + codec: whisperJobCodec, + run: runWhisper, + workerLabel: `whisper-${i + 1}`, + })); + } + for (let i = 0; i < jobConcurrency; i += 1) { + workerLoops.push(createWorkerLoop({ + owner: active, + consumer: active.layoutConsumer, + codec: layoutJobCodec, + run: runLayout, + workerLabel: `layout-${i + 1}`, + })); + } + } + + const close = async (): Promise => { + if (stopping) return; + stopping = true; + if (idleTimer) { + clearInterval(idleTimer); + idleTimer = null; + } + await app.close(); + await Promise.allSettled(workerLoops); + const current = session; + session = null; + if (current) { + try { + await current.nc.drain(); + } catch { + try { + await current.nc.close(); + } catch { + // ignore close errors + } + } + } + }; + + const registerSignalHandlers = (): void => { + process.once('SIGINT', () => { + void close().finally(() => process.exit(0)); + }); + + process.once('SIGTERM', () => { + void close().finally(() => process.exit(0)); + }); + }; + + return { + app, + host, + port, + async start(startOptions) { + if (startOptions?.registerSignalHandlers) { + registerSignalHandlers(); + } + await app.listen({ host, port }); + app.log.info({ host, port }, 'compute worker listening'); + }, + close, + }; +} + +export async function startComputeWorkerFromEnv(): Promise { + const runtime = await createComputeWorkerApp(); + await runtime.start({ registerSignalHandlers: true }); +} diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts index 8c62d03..1bf176c 100644 --- a/compute/worker/src/server.ts +++ b/compute/worker/src/server.ts @@ -1,1338 +1,6 @@ -import Fastify, { type FastifyRequest } from 'fastify'; -import { z } from 'zod'; -import { - connect, - nanos, - credsAuthenticator, - type NatsConnection, -} from '@nats-io/transport-node'; -import { - AckPolicy, - DeliverPolicy, - ReplayPolicy, - RetentionPolicy, - StorageType, - jetstream, - jetstreamManager, - type Consumer, - type JetStreamClient, - type JetStreamManager, - type JsMsg, -} from '@nats-io/jetstream'; -import { Kvm } from '@nats-io/kv'; -import { - ensureComputeModels, - runPdfLayoutFromPdfBuffer, - runWhisperAlignmentFromAudioBuffer, -} from '@openreader/compute-core/local-runtime'; -import { - getComputeTimeoutConfig, - getComputeOpStaleMs, - getAvailableCpuCores, - getOnnxThreadsPerJob, - withIdleTimeoutAndHardCap, - withTimeout, -} from '@openreader/compute-core'; -import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerJobState, - WorkerJobTiming, - WorkerOperationKind, - WorkerOperationRequest, - WorkerOperationState, - PdfLayoutProgress, -} from '@openreader/compute-core/api-contracts'; -import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { - JetStreamOperationEventStream, - OP_EVENTS_SUBJECT_WILDCARD, - JetStreamOperationQueue, - JetStreamOperationStateStore, - hashOpKey, -} from './control-plane/jetstream'; -import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; -import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; +import { startComputeWorkerFromEnv } from './runtime'; -const JOBS_STREAM_NAME = 'compute_jobs'; -const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; -const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; -const WHISPER_CONSUMER_NAME = 'compute_whisper'; -const LAYOUT_CONSUMER_NAME = 'compute_layout'; -const EVENTS_STREAM_NAME = 'compute_events'; -const COMPUTE_STATE_BUCKET = 'compute_state'; -const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; -const LOOP_ERROR_BACKOFF_MS = 500; -const RUNNING_HEARTBEAT_MS = 5000; -const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; -const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; -const WHISPER_MAX_DELIVER = 1; -const NATS_API_TIMEOUT_MS = 60_000; -// Disconnect from NATS after this much continuous idle so the worker stops -// generating outbound traffic (pull polling + keepalive PINGs) and Railway can -// put it to sleep. Reconnect happens lazily on the next inbound request. -const IDLE_DISCONNECT_MS = 120_000; -const IDLE_CHECK_INTERVAL_MS = 5_000; -// Bounded pull window so consumer loops yield periodically and can be stopped -// cleanly when going idle, instead of blocking on a long-lived pull. -const PULL_EXPIRES_MS = 5_000; -const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); -const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); -const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { - whisper_align: 15_000, - pdf_layout: 120_000, -}; - -interface QueuedJob { - jobId: string; - opId: string; - opKey: string; - kind: WorkerOperationKind; - queuedAt: number; - payload: TPayload; -} - -interface NatsSession { - nc: NatsConnection; - js: JetStreamClient; - jsm: JetStreamManager; - kv: Awaited>; - whisperConsumer: Consumer; - layoutConsumer: Consumer; -} - -type StreamedOperationState = WorkerOperationState; - -function requireEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} - -function readIntEnv(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -function normalizeNatsReplicas(value: number): number { - if (value === 3 || value === 5) return value; - return 1; -} - -function parseBoolEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const normalized = raw.toLowerCase(); - return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; -} - -function buildLoggerConfig(): boolean | Record { - const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); - const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; - if (format === 'json') { - return { - level, - base: null, - }; - } - return { - level, - base: null, - transport: { - target: 'pino-pretty', - options: { - colorize: true, - translateTime: 'SYS:standard', - ignore: 'pid,hostname', - }, - }, - }; -} - -function normalizeS3Prefix(prefix: string | undefined): string { - const value = (prefix || 'openreader').trim(); - return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; -} - -function sanitizeNamespace(namespace: string | null): string | null { - if (!namespace) return null; - return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; -} - -function documentParsedKey(id: string, namespace: string | null, prefix: string): string { - if (!DOCUMENT_ID_REGEX.test(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; -} - -function buildS3Client(): S3Client { - const bucket = requireEnv('S3_BUCKET'); - const region = requireEnv('S3_REGION'); - const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); - const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); - const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; - const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); - - void bucket; - - return new S3Client({ - region, - endpoint, - forcePathStyle, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - credentials: { - accessKeyId, - secretAccessKey, - }, - }); -} - -async function bodyToBuffer(body: unknown): Promise { - if (!body) return Buffer.alloc(0); - if (body instanceof Uint8Array) return Buffer.from(body); - if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); - if (body instanceof ArrayBuffer) return Buffer.from(body); - if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { - const maybe = body as { transformToByteArray?: () => Promise }; - if (typeof maybe.transformToByteArray === 'function') { - return Buffer.from(await maybe.transformToByteArray()); - } - } - if (typeof body === 'object' && body !== null && 'on' in body) { - const stream = body as NodeJS.ReadableStream; - const chunks: Buffer[] = []; - for await (const chunk of stream) { - if (Buffer.isBuffer(chunk)) chunks.push(chunk); - else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); - else chunks.push(Buffer.from(chunk as Uint8Array)); - } - return Buffer.concat(chunks); - } - throw new Error('Unsupported S3 response body type'); -} - -function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { - const copy = new Uint8Array(bytes.byteLength); - copy.set(bytes); - return copy.buffer; -} - -function isAuthed(request: FastifyRequest, expectedToken: string): boolean { - const auth = request.headers.authorization; - if (!auth?.startsWith('Bearer ')) return false; - const token = auth.slice('Bearer '.length).trim(); - return token === expectedToken; -} - -function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { - if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; - return Math.max(0, Math.floor((end as number) - (start as number))); -} - -function toErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - return String(error); -} - -function requestPath(request: FastifyRequest): string { - return request.url.split('?')[0] ?? request.url; -} - -function isHealthPath(path: string): boolean { - return path === '/health/live' || path === '/health/ready'; -} - -function extractTraceId(request: FastifyRequest): string | null { - const header = request.headers['x-openreader-trace-id']; - if (Array.isArray(header)) return header[0] ?? null; - return typeof header === 'string' ? header : null; -} - -function extractOpId(request: FastifyRequest, path: string): string | null { - const params = request.params as { opId?: unknown } | undefined; - if (params && typeof params.opId === 'string' && params.opId.trim()) { - return params.opId.trim(); - } - const match = path.match(/^\/ops\/([^/]+)/); - if (!match?.[1]) return null; - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; - } -} - -function isAlreadyExistsError(error: unknown): boolean { - const message = toErrorMessage(error).toLowerCase(); - return message.includes('already in use') || message.includes('already exists'); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -class ConcurrencyGate { - private readonly maxInFlight: number; - private inFlight = 0; - private readonly queue: Array<() => void> = []; - - constructor(limit: number) { - this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; - } - - async acquire(): Promise { - if (this.inFlight < this.maxInFlight) { - this.inFlight += 1; - return; - } - - await new Promise((resolve) => { - this.queue.push(() => { - this.inFlight += 1; - resolve(); - }); - }); - } - - release(): void { - this.inFlight = Math.max(0, this.inFlight - 1); - const next = this.queue.shift(); - if (next) next(); - } -} - -function isTerminalStatus(status: WorkerJobState): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { - if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; - const maybe = result as { parsedObjectKey?: unknown }; - return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; -} - -const alignSchema = z.object({ - text: z.string().trim().min(1), - lang: z.string().trim().min(1).max(16).optional(), - cacheKey: z.string().trim().min(1).max(256).optional(), - audioObjectKey: z.string().trim().min(1).max(2048), -}); - -const layoutSchema = z.object({ - documentId: z.string().trim().min(1), - namespace: z.string().trim().min(1).max(128).nullable(), - documentObjectKey: z.string().trim().min(1).max(2048), -}); - -const operationCreateSchema = z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('whisper_align'), - opKey: z.string().trim().min(1).max(1024), - payload: alignSchema, - }), - z.object({ - kind: z.literal('pdf_layout'), - opKey: z.string().trim().min(1).max(1024), - payload: layoutSchema, - }), -]); - -async function ensureJetStreamResources( - jsm: JetStreamManager, - whisperTimeoutMs: number, - pdfTimeoutMs: number, - pdfAttempts: number, - jobsMaxBytes: number, - eventsMaxBytes: number, - natsReplicas: number, -): Promise { - const streamConfig = { - name: JOBS_STREAM_NAME, - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - retention: RetentionPolicy.Workqueue, - storage: StorageType.File, - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(streamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(JOBS_STREAM_NAME, { - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }); - } - - const eventsStreamConfig = { - name: EVENTS_STREAM_NAME, - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - retention: RetentionPolicy.Limits, - storage: StorageType.File, - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(eventsStreamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(EVENTS_STREAM_NAME, { - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }); - } - - const ensureConsumer = async ( - name: string, - subject: string, - ackWaitMs: number, - maxDeliver: number, - ): Promise => { - const config = { - durable_name: name, - ack_policy: AckPolicy.Explicit, - deliver_policy: DeliverPolicy.All, - replay_policy: ReplayPolicy.Instant, - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }; - - try { - await jsm.consumers.add(JOBS_STREAM_NAME, config); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.consumers.update(JOBS_STREAM_NAME, name, { - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }); - } - }; - - await Promise.all([ - ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), - ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), - ]); -} - -async function main(): Promise { - const port = readIntEnv('PORT', 8081); - const host = process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'; - const workerToken = requireEnv('COMPUTE_WORKER_TOKEN'); - const natsUrl = requireEnv('NATS_URL'); - const timeoutConfig = getComputeTimeoutConfig(); - - const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); - const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; - const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; - const pdfHardCapMs = timeoutConfig.pdfHardCapMs; - const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); - const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); - const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); - const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); - const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); - const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); - const opStaleMs = getComputeOpStaleMs(); - - const connectOpts: Parameters[0] = { servers: natsUrl }; - const natsCreds = process.env.NATS_CREDS?.trim(); - const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); - - if (natsCreds) { - console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); - connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); - } else if (natsCredsFile) { - console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); - const { readFileSync } = await import('node:fs'); - const credsData = readFileSync(natsCredsFile); - connectOpts.authenticator = credsAuthenticator(credsData); - } - - // Lazy NATS connection lifecycle. The worker connects on demand (first request - // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle - // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound /ops request both wakes the container and re-establishes - // the session via ensureConnected(). - let session: NatsSession | null = null; - let connecting: Promise | null = null; - let workerLoops: Promise[] = []; - let idleTimer: NodeJS.Timeout | null = null; - let stopping = false; - let loopStopRequested = false; - - // Activity accounting feeding the idle detector. The worker is considered idle - // only when no HTTP request is in flight, no SSE stream is open, no job is - // processing, and nothing has happened for IDLE_DISCONNECT_MS. - let inFlightHttp = 0; - let activeSse = 0; - let inFlightJobs = 0; - let lastActivityAt = Date.now(); - const jobGate = new ConcurrencyGate(jobConcurrency); - - const markActivity = (): void => { - lastActivityAt = Date.now(); - }; - - function startIdleTimer(): void { - if (idleTimer) return; - idleTimer = setInterval(() => { - if (!session || stopping) return; - if (inFlightHttp > 0 || activeSse > 0 || inFlightJobs > 0) return; - if (Date.now() - lastActivityAt < IDLE_DISCONNECT_MS) return; - void disconnect('idle'); - }, IDLE_CHECK_INTERVAL_MS); - // Don't let the idle checker keep the process alive on its own. - idleTimer.unref?.(); - } - - async function disconnect(reason: string): Promise { - const current = session; - if (!current) return; - // Clear synchronously (before any await) so concurrent requests reconnect a - // fresh session instead of using the connection we're about to close. - session = null; - loopStopRequested = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - try { - await current.nc.close(); - } catch { - // ignore close errors - } - await Promise.allSettled(workerLoops); - workerLoops = []; - loopStopRequested = false; - app.log.info({ reason }, 'nats disconnected'); - } - - async function ensureConnected(): Promise { - if (session) return session; - if (connecting) return connecting; - connecting = (async () => { - const nc: NatsConnection = await connect(connectOpts); - const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); - const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); - await ensureJetStreamResources( - jsm, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - jobsStreamMaxBytes, - eventsStreamMaxBytes, - natsReplicas, - ); - const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { - replicas: natsReplicas, - history: 1, - ttl: COMPUTE_STATE_TTL_MS, - max_bytes: jobStatesMaxBytes, - }); - const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); - const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); - const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; - session = next; - markActivity(); - startWorkerLoops(next); - startIdleTimer(); - // Safety net: if the connection closes for any reason (network drop after - // exhausting reconnects, or our own disconnect), drop the stale session so - // the next request reconnects cleanly. - void nc.closed().then(() => { - if (session?.nc === nc) session = null; - }); - app.log.info('nats connected'); - return next; - })(); - try { - return await connecting; - } finally { - connecting = null; - } - } - - const s3 = buildS3Client(); - const s3Bucket = requireEnv('S3_BUCKET'); - const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); - - const ensureSafeKey = (key: string): string => { - const trimmed = key.trim(); - if (!trimmed.startsWith(`${s3Prefix}/`)) { - throw new Error('Object key prefix mismatch'); - } - return trimmed; - }; - - const readObjectByKey = async (key: string): Promise => { - const safeKey = ensureSafeKey(key); - const response = await s3.send(new GetObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - const bytes = await bodyToBuffer(response.Body); - return toArrayBuffer(new Uint8Array(bytes)); - }; - - const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { - const key = documentParsedKey(documentId, namespace, s3Prefix); - const body = Buffer.from(JSON.stringify(parsed)); - await s3.send(new PutObjectCommand({ - Bucket: s3Bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - })); - return key; - }; - - if (prewarmModels) { - await ensureComputeModels(); - } - - const app = Fastify({ - logger: buildLoggerConfig(), - disableRequestLogging: true, - }); - app.log.info({ - jobConcurrency, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - opStaleMs, - availableCpuCores: getAvailableCpuCores(), - onnxThreadsPerJob: getOnnxThreadsPerJob(), - natsApiTimeoutMs: NATS_API_TIMEOUT_MS, - natsReplicas, - eventsStreamMaxBytes, - pdfLayoutHardCapMs: pdfHardCapMs, - }, 'compute runtime config'); - - const whisperJobCodec = createJsonCodec>(); - const layoutJobCodec = createJsonCodec>(); - - const operationStateStore = new JetStreamOperationStateStore({ - getKv: async () => (await ensureConnected()).kv, - }); - - const operationEventStream = new JetStreamOperationEventStream({ - getJs: async () => (await ensureConnected()).js, - getJsm: async () => (await ensureConnected()).jsm, - eventsStreamName: EVENTS_STREAM_NAME, - }); - - const operationQueue = new JetStreamOperationQueue({ - getJs: async () => (await ensureConnected()).js, - whisperSubject: WHISPER_JOBS_SUBJECT, - layoutSubject: LAYOUT_JOBS_SUBJECT, - }); - - const orchestrator = new OperationOrchestrator({ - queue: operationQueue, - stateStore: operationStateStore, - eventStream: operationEventStream, - config: { - opStaleMs, - maxCasRetries: 10, - }, - }); - - const getOpState = async (opId: string): Promise => { - return await operationStateStore.getOpState(opId); - }; - - const releaseHttp = (request: FastifyRequest): void => { - const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; - if (!counted[REQUEST_COUNTED_KEY]) return; - counted[REQUEST_COUNTED_KEY] = false; - inFlightHttp = Math.max(0, inFlightHttp - 1); - markActivity(); - }; - - app.addHook('onRequest', async (request, reply) => { - const path = requestPath(request); - (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); - // Count every request as in-flight activity so the idle detector never - // disconnects mid-request. Released in onResponse, or manually after hijack - // for SSE streams (where onResponse does not fire). - (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; - inFlightHttp += 1; - markActivity(); - if (isHealthPath(path)) return; - if (!isAuthed(request, workerToken)) { - return reply.code(401).send({ error: 'Unauthorized' }); - } - return; - }); - - app.addHook('onResponse', async (request, reply) => { - releaseHttp(request); - const path = requestPath(request); - if (isHealthPath(path)) return; - if (reply.statusCode >= 500) { - const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; - const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; - app.log.error({ - reqId: request.id, - method: request.method, - path, - statusCode: reply.statusCode, - durationMs, - traceId: extractTraceId(request) ?? null, - opId: extractOpId(request, path), - }, 'http.error'); - } - }); - - app.get('/health/live', async () => ({ ok: true })); - - // Reports readiness without forcing a NATS round-trip. Probing NATS here would - // reconnect (and keep) the connection open, defeating idle sleep, so we only - // report the current connection state. The worker reconnects lazily on the next - // /ops request regardless of what this returns. - app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); - - app.post('/ops', async (request, reply) => { - const parsed = operationCreateSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - const requestOp = parsed.data as WorkerOperationRequest; - const op = await orchestrator.enqueueOrReuse(requestOp); - app.log.info({ - kind: requestOp.kind, - opId: op.opId, - jobId: op.jobId, - status: op.status, - opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), - }, 'op.accepted'); - reply.code(202); - return op; - }); - - app.get('/ops/:opId', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const state = await getOpState(params.data.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; - }); - - app.get('/ops/:opId/events', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const initial = await getOpState(params.data.opId); - if (!initial) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; - const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); - const lastEventIdHeader = request.headers['last-event-id']; - const cursorFromHeader = Number( - Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), - ); - const sinceEventId = Math.max( - 0, - Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, - Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, - ); - - reply.hijack(); - // onResponse will not fire for a hijacked reply, so release the HTTP in-flight - // count here and track the long-lived stream via activeSse instead. - releaseHttp(request); - activeSse += 1; - markActivity(); - reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); - reply.raw.setHeader('Connection', 'keep-alive'); - reply.raw.setHeader('X-Accel-Buffering', 'no'); - - let closed = false; - let unsubscribe: (() => void) | null = null; - - const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { - if (closed || reply.raw.writableEnded) return; - const frameEvent: WorkerOperationEvent = { - eventId, - snapshot, - }; - reply.raw.write(encodeSseFrame({ - id: eventId, - event: 'snapshot', - data: frameEvent, - })); - }; - - const closeStream = (): void => { - if (closed) return; - closed = true; - if (unsubscribe) { - unsubscribe(); - unsubscribe = null; - } - activeSse = Math.max(0, activeSse - 1); - markActivity(); - if (!reply.raw.writableEnded) { - reply.raw.end(); - } - }; - - request.raw.on('close', () => { - closeStream(); - }); - - try { - let current = initial; - let signature = JSON.stringify(current); - writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); - if (isTerminalStatus(current.status)) { - return reply; - } - - unsubscribe = await operationEventStream.subscribe({ - opId: params.data.opId, - sinceEventId, - onEvent: (event) => { - if (closed) return; - if (event.snapshot.opId !== params.data.opId) return; - const nextSignature = JSON.stringify(event.snapshot); - if (nextSignature !== signature) { - current = event.snapshot; - signature = nextSignature; - writeSnapshot(current, event.eventId); - } - if (isTerminalStatus(event.snapshot.status)) { - closeStream(); - } - }, - onError: (error) => { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - closeStream(); - }, - }); - - await new Promise((resolve) => { - request.raw.once('close', () => resolve()); - }); - } catch (error) { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - } finally { - closeStream(); - } - - return reply; - }); - - const runWhisper = async ( - payload: WhisperAlignJobRequest, - queueWaitMs: number, - ): Promise => { - const parsed = alignSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const audioBuffer = await withTimeout( - readObjectByKey(parsed.audioObjectKey), - whisperTimeoutMs, - 'whisper s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - const computeStartedAt = Date.now(); - const result = await withTimeout( - runWhisperAlignmentFromAudioBuffer({ - audioBuffer, - text: parsed.text, - cacheKey: parsed.cacheKey, - lang: parsed.lang, - }), - whisperTimeoutMs, - 'whisper alignment job', - ); - - const computeMs = Date.now() - computeStartedAt; - return { - ...result, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - const runLayout = async ( - payload: PdfLayoutJobRequest, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ): Promise => { - const parsed = layoutSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const pdfBytes = await withTimeout( - readObjectByKey(parsed.documentObjectKey), - Math.max(pdfTimeoutMs, 1_000), - 'pdf s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - let lastTotalPages = 0; - let lastPagesParsed = 0; - const computeStartedAt = Date.now(); - const result = await withIdleTimeoutAndHardCap({ - idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), - hardCapMs: pdfHardCapMs, - label: 'pdf layout job', - run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ - documentId: parsed.documentId, - pdfBytes, - onPageStarted: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageStart({ - pageNumber, - totalPages, - })); - }, - onPageParsed: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - lastPagesParsed = pageNumber; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageParsed({ - pageNumber, - totalPages, - })); - }, - }), - }); - - const computeMs = Date.now() - computeStartedAt; - if (hooks?.onProgress && lastTotalPages > 0) { - await hooks.onProgress({ - totalPages: lastTotalPages, - pagesParsed: lastPagesParsed, - currentPage: lastPagesParsed || undefined, - phase: 'merge', - }); - } - const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); - return { - parsedObjectKey, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - type ProcessMessageInput = { - msg: JsMsg; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }; - - type ProcessMessageContext = { - decoded: QueuedJob; - workerLabel: string; - startedAt: number; - queueWaitTiming?: { queueWaitMs: number }; - latestProgress?: PdfLayoutProgress; - }; - - function buildQueueWaitTiming(queuedAt: number, now: number): { queueWaitMs: number } | undefined { - const queueWaitMs = safeDurationMs(queuedAt, now); - return typeof queueWaitMs === 'number' ? { queueWaitMs } : undefined; - } - - function extractTiming(result: unknown): WorkerJobTiming | undefined { - if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; - return (result as { timing?: WorkerJobTiming }).timing; - } - - async function markRunning( - context: ProcessMessageContext, - updatedAt: number, - options?: { includeStartedAt?: boolean }, - ): Promise { - if (context.latestProgress) { - await orchestrator.markProgress({ - opId: context.decoded.opId, - progress: context.latestProgress, - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - return; - } - - await orchestrator.markRunning({ - opId: context.decoded.opId, - ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - } - - async function markProgress( - context: ProcessMessageContext, - progress: PdfLayoutProgress, - updatedAt: number, - ): Promise { - context.latestProgress = progress; - await markRunning(context, updatedAt); - } - - async function markTerminal(input: { - context: ProcessMessageContext; - status: 'succeeded' | 'failed'; - result?: TResult; - errorMessage?: string; - timing?: WorkerJobTiming; - updatedAt: number; - }): Promise { - if (input.status === 'succeeded') { - await orchestrator.markSucceeded({ - opId: input.context.decoded.opId, - result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - return; - } - - await orchestrator.markFailed({ - opId: input.context.decoded.opId, - error: { message: input.errorMessage ?? 'unknown worker failure' }, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - } - - async function handleRetry(input: { - context: ProcessMessageContext | null; - msg: JsMsg; - errorMessage: string; - }): Promise { - const deliveryCount = input.msg.info.deliveryCount; - const isWhisperAlign = input.context?.decoded.kind === 'whisper_align'; - const maxAttempts = isWhisperAlign ? WHISPER_MAX_DELIVER : pdfAttempts; - const hasRetriesLeft = !isWhisperAlign && deliveryCount < maxAttempts; - - if (input.context) { - const now = Date.now(); - const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); - const persistOpUpdate = hasRetriesLeft - ? (input.context.latestProgress - ? orchestrator.markProgress({ - opId: input.context.decoded.opId, - progress: input.context.latestProgress, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }) - : orchestrator.markRunning({ - opId: input.context.decoded.opId, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - })) - : markTerminal({ - context: input.context, - status: 'failed', - errorMessage: input.errorMessage, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }); - - await persistOpUpdate.catch((stateError) => { - app.log.error({ - worker: input.context?.workerLabel, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation state'); - }); - } - - if (hasRetriesLeft) { - input.msg.nak(); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'running', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retryAction: 'nack_retry', - }, 'job.terminal'); - return; - } - - input.msg.term(input.errorMessage); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'failed', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retrySuppressed: isWhisperAlign ? 'whisper_align' : undefined, - retryAction: 'term', - }, 'job.terminal'); - } - - async function processMessage(input: ProcessMessageInput): Promise { - let context: ProcessMessageContext | null = null; - let heartbeat: NodeJS.Timeout | null = null; - try { - const decoded = input.codec.decode(input.msg.data); - const startedAt = Date.now(); - context = { - decoded, - workerLabel: input.workerLabel, - startedAt, - queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), - }; - - await markRunning(context, startedAt); - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, - deliveryCount: input.msg.info.deliveryCount, - }, 'job.started'); - - heartbeat = setInterval(() => { - const now = Date.now(); - void markRunning(context!, now).catch((stateError) => { - app.log.error({ - worker: input.workerLabel, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation heartbeat state'); - }); - }, RUNNING_HEARTBEAT_MS); - - const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { - onProgress: async (progress) => { - await markProgress(context!, progress, Date.now()); - }, - }); - const resultTiming = extractTiming(result); - const now = Date.now(); - - await markTerminal({ - context, - status: 'succeeded', - result, - timing: resultTiming, - updatedAt: now, - }); - - input.msg.ack(); - const terminalDurationMs = safeDurationMs(context.startedAt, now); - const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; - if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - durationMs: terminalDurationMs ?? null, - timing: resultTiming ?? null, - }, 'job.stage'); - } - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - status: 'succeeded', - durationMs: terminalDurationMs ?? null, - resultRef: extractResultRef(decoded.kind, result), - timing: resultTiming ?? null, - }, 'job.terminal'); - } catch (error) { - await handleRetry({ - context, - msg: input.msg, - errorMessage: toErrorMessage(error), - }); - } finally { - if (heartbeat) clearInterval(heartbeat); - } - } - - async function createWorkerLoop(input: { - owner: NatsSession; - consumer: Consumer; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }): Promise { - // Exit when the loop's connection is no longer the active session (idle - // disconnect, unexpected close, or replaced by a reconnect). - const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; - while (!detached()) { - let msg: JsMsg | null = null; - try { - try { - // Bounded pull so the loop yields periodically and exits promptly when - // the session is torn down for idle (nc.close() rejects the pending pull). - msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); - } catch (error) { - if (detached()) return; - app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); - await sleep(LOOP_ERROR_BACKOFF_MS); - continue; - } - - // An empty pull is not activity; let the idle window advance. - if (!msg) continue; - markActivity(); - inFlightJobs += 1; - await jobGate.acquire(); - if (detached()) { - return; - } - await processMessage({ - msg, - codec: input.codec, - run: input.run, - workerLabel: input.workerLabel, - }); - } finally { - if (msg) { - jobGate.release(); - inFlightJobs = Math.max(0, inFlightJobs - 1); - markActivity(); - } - } - } - } - - function startWorkerLoops(active: NatsSession): void { - // Always starts a fresh set bound to the new session. Any loops from a prior - // session self-terminate once they observe session !== their owner. - workerLoops = []; - loopStopRequested = false; - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.whisperConsumer, - codec: whisperJobCodec, - run: runWhisper, - workerLabel: `whisper-${i + 1}`, - })); - } - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.layoutConsumer, - codec: layoutJobCodec, - run: runLayout, - workerLabel: `layout-${i + 1}`, - })); - } - } - - const close = async (): Promise => { - if (stopping) return; - stopping = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - await app.close(); - await Promise.allSettled(workerLoops); - const current = session; - session = null; - if (current) { - try { - await current.nc.drain(); - } catch { - try { - await current.nc.close(); - } catch { - // ignore close errors - } - } - } - }; - - process.once('SIGINT', () => { - void close().finally(() => process.exit(0)); - }); - - process.once('SIGTERM', () => { - void close().finally(() => process.exit(0)); - }); - - await app.listen({ host, port }); - app.log.info({ host, port }, 'compute worker listening'); -} - -void main().catch((error) => { +void startComputeWorkerFromEnv().catch((error) => { console.error('[compute-worker] fatal startup error', error); process.exit(1); }); diff --git a/compute/worker/src/worker-loop-policy.ts b/compute/worker/src/worker-loop-policy.ts new file mode 100644 index 0000000..7462b4c --- /dev/null +++ b/compute/worker/src/worker-loop-policy.ts @@ -0,0 +1,22 @@ +import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts'; + +export type RetryAction = 'nak_retry' | 'term_fail'; + +export function buildQueueWaitTiming(queuedAt: number, now: number): { queueWaitMs: number } | undefined { + if (!Number.isFinite(queuedAt) || !Number.isFinite(now)) return undefined; + return { queueWaitMs: Math.max(0, Math.floor(now - queuedAt)) }; +} + +export function decideRetryAction(input: { + kind: WorkerOperationKind; + deliveryCount: number; + pdfAttempts: number; + whisperMaxDeliver?: number; +}): RetryAction { + const whisperMaxDeliver = input.whisperMaxDeliver ?? 1; + if (input.kind === 'whisper_align') { + return input.deliveryCount < whisperMaxDeliver ? 'nak_retry' : 'term_fail'; + } + + return input.deliveryCount < input.pdfAttempts ? 'nak_retry' : 'term_fail'; +} diff --git a/compute/worker/tests/api/routes.test.ts b/compute/worker/tests/api/routes.test.ts new file mode 100644 index 0000000..77c4d05 --- /dev/null +++ b/compute/worker/tests/api/routes.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { createComputeWorkerApp } from '../../src/runtime'; +import { FakeControlPlane } from '../fixtures/fake-control-plane'; + +const AUTH = { authorization: 'Bearer test-token' }; + +describe('compute worker API routes', () => { + let fake: FakeControlPlane; + let runtime: Awaited>; + + beforeEach(async () => { + fake = new FakeControlPlane(); + runtime = await createComputeWorkerApp({ + workerToken: 'test-token', + disableWorkers: true, + routeDeps: fake.deps, + }); + }); + + afterEach(async () => { + await runtime.close(); + }); + + test('allows unauthenticated health checks but protects operation routes', async () => { + const live = await runtime.app.inject({ method: 'GET', url: '/health/live' }); + expect(live.statusCode).toBe(200); + + const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' }); + expect(protectedRoute.statusCode).toBe(401); + }); + + test('validates operation creation body and returns 400 for invalid payload', async () => { + const response = await runtime.app.inject({ + method: 'POST', + url: '/ops', + headers: AUTH, + payload: { + kind: 'pdf_layout', + opKey: '', + payload: { + documentId: 'd1', + namespace: null, + documentObjectKey: 's3://bucket/doc.pdf', + }, + }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json()).toMatchObject({ error: 'Invalid request body' }); + }); + + test('creates operation and fetches state by op id', async () => { + const create = await runtime.app.inject({ + method: 'POST', + url: '/ops', + headers: AUTH, + payload: { + kind: 'pdf_layout', + opKey: 'doc-1:layout', + payload: { + documentId: 'doc-1', + namespace: null, + documentObjectKey: 'openreader/doc-1.pdf', + }, + }, + }); + + expect(create.statusCode).toBe(202); + const created = create.json(); + expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' }); + + const fetch = await runtime.app.inject({ + method: 'GET', + url: `/ops/${created.opId}`, + headers: AUTH, + }); + + expect(fetch.statusCode).toBe(200); + expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' }); + }); + + test('returns not found for unknown operation and event stream lookups', async () => { + const opResponse = await runtime.app.inject({ + method: 'GET', + url: '/ops/missing', + headers: AUTH, + }); + expect(opResponse.statusCode).toBe(404); + + const eventsResponse = await runtime.app.inject({ + method: 'GET', + url: '/ops/missing/events', + headers: AUTH, + }); + expect(eventsResponse.statusCode).toBe(404); + }); + + test('streams initial SSE snapshot for terminal operation and honors cursor id', async () => { + fake.seedState({ + opId: 'op-terminal', + opKey: 'k-terminal', + kind: 'pdf_layout', + jobId: 'job-op-terminal', + status: 'succeeded', + queuedAt: 1000, + updatedAt: 2000, + result: { parsedObjectKey: 'openreader/parsed.json' }, + }); + + const stream = await runtime.app.inject({ + method: 'GET', + url: '/ops/op-terminal/events?sinceEventId=7', + headers: AUTH, + }); + + expect(stream.statusCode).toBe(200); + expect(stream.headers['content-type']).toContain('text/event-stream'); + expect(stream.body).toContain('event: snapshot'); + expect(stream.body).toContain('id: 7'); + expect(stream.body).toContain('"status":"succeeded"'); + }); +}); diff --git a/compute/worker/tests/fixtures/fake-control-plane.ts b/compute/worker/tests/fixtures/fake-control-plane.ts new file mode 100644 index 0000000..2d18f2b --- /dev/null +++ b/compute/worker/tests/fixtures/fake-control-plane.ts @@ -0,0 +1,119 @@ +import type { + PdfLayoutJobResult, + WhisperAlignJobResult, + WorkerOperationEvent, + WorkerOperationRequest, + WorkerOperationState, +} from '@openreader/compute-core/api-contracts'; +import type { ComputeWorkerRouteDeps } from '../../src/runtime'; + +type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; +type ComputeState = WorkerOperationState; +type ComputeEvent = WorkerOperationEvent; + +export class FakeControlPlane { + private readonly stateByOpId = new Map(); + private readonly opIdByOpKey = new Map(); + private readonly eventsByOpId = new Map(); + private nextOpId = 1; + + readonly deps: ComputeWorkerRouteDeps = { + orchestrator: { + enqueueOrReuse: async (request) => this.enqueueOrReuse(request), + markRunning: async (input) => this.updateState(input.opId, { + status: 'running', + startedAt: input.startedAt, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markProgress: async (input) => this.updateState(input.opId, { + status: 'running', + progress: input.progress, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markSucceeded: async (input) => this.updateState(input.opId, { + status: 'succeeded', + result: input.result as ComputeResult, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markFailed: async (input) => this.updateState(input.opId, { + status: 'failed', + error: typeof input.error === 'string' ? { message: input.error } : input.error, + updatedAt: input.updatedAt, + timing: input.timing, + }), + }, + operationStateStore: { + getOpState: async (opId) => this.stateByOpId.get(opId) ?? null, + }, + operationEventStream: { + subscribe: async ({ opId, sinceEventId, onEvent }) => { + const since = Math.max(0, Math.floor(sinceEventId ?? 0)); + const replay = (this.eventsByOpId.get(opId) ?? []).filter((event) => event.eventId > since); + for (const event of replay) { + await onEvent(event); + } + return () => undefined; + }, + }, + }; + + seedState(state: ComputeState): void { + this.stateByOpId.set(state.opId, state); + this.opIdByOpKey.set(state.opKey, state.opId); + } + + seedEvent(opId: string, event: ComputeEvent): void { + const list = this.eventsByOpId.get(opId) ?? []; + list.push(event); + this.eventsByOpId.set(opId, list); + } + + private async enqueueOrReuse(request: WorkerOperationRequest): Promise { + const existingId = this.opIdByOpKey.get(request.opKey); + if (existingId) { + const existing = this.stateByOpId.get(existingId); + if (existing) return existing; + } + + const now = Date.now(); + const opId = `op-${this.nextOpId++}`; + const state: ComputeState = { + opId, + opKey: request.opKey, + kind: request.kind, + jobId: `job-${opId}`, + status: 'queued', + queuedAt: now, + updatedAt: now, + }; + + this.stateByOpId.set(opId, state); + this.opIdByOpKey.set(request.opKey, opId); + this.seedEvent(opId, { eventId: 1, snapshot: state }); + return state; + } + + private async updateState( + opId: string, + patch: Partial, + ): Promise { + const current = this.stateByOpId.get(opId); + if (!current) { + throw new Error(`Unknown opId: ${opId}`); + } + const next: ComputeState = { + ...current, + ...patch, + updatedAt: patch.updatedAt ?? Date.now(), + }; + this.stateByOpId.set(opId, next); + const currentEvents = this.eventsByOpId.get(opId) ?? []; + const nextEventId = (currentEvents.at(-1)?.eventId ?? 0) + 1; + currentEvents.push({ eventId: nextEventId, snapshot: next }); + this.eventsByOpId.set(opId, currentEvents); + return next; + } +} diff --git a/compute/worker/tests/setup-env.ts b/compute/worker/tests/setup-env.ts new file mode 100644 index 0000000..d013982 --- /dev/null +++ b/compute/worker/tests/setup-env.ts @@ -0,0 +1,8 @@ +process.env.COMPUTE_WORKER_TOKEN = process.env.COMPUTE_WORKER_TOKEN || 'test-token'; +process.env.NATS_URL = process.env.NATS_URL || 'nats://127.0.0.1:4222'; +process.env.COMPUTE_PREWARM_MODELS = 'false'; +process.env.S3_BUCKET = process.env.S3_BUCKET || 'test-bucket'; +process.env.S3_REGION = process.env.S3_REGION || 'us-east-1'; +process.env.S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID || 'test'; +process.env.S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY || 'test'; +process.env.S3_PREFIX = process.env.S3_PREFIX || 'openreader'; diff --git a/compute/worker/tests/tsconfig.json b/compute/worker/tests/tsconfig.json new file mode 100644 index 0000000..f969c4c --- /dev/null +++ b/compute/worker/tests/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["./**/*.ts"], + "exclude": [] +} diff --git a/tests/unit/compute-worker-control-plane-jetstream.spec.ts b/compute/worker/tests/unit/jetstream-adapters.test.ts similarity index 55% rename from tests/unit/compute-worker-control-plane-jetstream.spec.ts rename to compute/worker/tests/unit/jetstream-adapters.test.ts index fb3f3e3..9f30ccc 100644 --- a/tests/unit/compute-worker-control-plane-jetstream.spec.ts +++ b/compute/worker/tests/unit/jetstream-adapters.test.ts @@ -1,16 +1,17 @@ -import { expect, test } from '@playwright/test'; -import { OperationOrchestrator } from '../../compute/core/src/control-plane'; -import type { WorkerOperationRequest } from '../../compute/core/src/api-contracts'; +import { describe, expect, test } from 'vitest'; +import { OperationOrchestrator } from '@openreader/compute-core/control-plane'; +import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, JetStreamOperationStateStore, + hashOpKey, opEventsSubject, opIndexKvKey, opStateKvKey, type KvEntryLike, type KvStoreLike, -} from '../../compute/worker/src/control-plane/jetstream'; +} from '../../src/control-plane/jetstream'; class FakeKvStore implements KvStoreLike { private readonly data = new Map(); @@ -18,46 +19,31 @@ class FakeKvStore implements KvStoreLike { async get(key: string): Promise { const value = this.data.get(key); - if (!value) return null; - return { - operation: value.operation, - value: value.value.slice(), - revision: value.revision, - }; + return value + ? { + operation: value.operation, + value: value.value.slice(), + revision: value.revision, + } + : null; } async put(key: string, data: Uint8Array): Promise { this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } async create(key: string, data: Uint8Array): Promise { - if (this.data.has(key)) { - throw new Error('key exists'); - } + if (this.data.has(key)) throw new Error('key exists'); this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } async update(key: string, data: Uint8Array, version: number): Promise { const current = this.data.get(key); - if (!current || current.revision !== version) { - throw new Error('wrong last sequence'); - } + if (!current || current.revision !== version) throw new Error('wrong last sequence'); this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } } @@ -72,7 +58,7 @@ class FakeJetStream { async publish(subject: string, data: Uint8Array): Promise<{ seq: number; stream: string; duplicate: boolean }> { this.seq += 1; - const payload = JSON.parse(new TextDecoder().decode(data)) as unknown; + const payload = JSON.parse(new TextDecoder().decode(data)); this.published.push({ subject, payload, seq: this.seq }); return { seq: this.seq, stream: 'fake', duplicate: false }; } @@ -90,97 +76,81 @@ function buildPdfRequest(opKey: string): WorkerOperationRequest { }; } -test.describe('worker jetstream control-plane adapters', () => { - test('state store compareAndSet handles create/update semantics', async () => { +describe('jetstream adapters', () => { + test('state store compareAndSet enforces create/update semantics', async () => { const kv = new FakeKvStore(); const store = new JetStreamOperationStateStore({ getKv: async () => kv }); - const created = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-1', - expectedOpId: null, - }); - const failedCreate = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: null, - }); - const wrongExpected = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: 'op-x', - }); - const updated = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: 'op-1', - }); + const created = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-1', expectedOpId: null }); + const duplicateCreate = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: null }); + const wrongExpected = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: 'op-x' }); + const updated = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: 'op-1' }); - expect(created).toBeTruthy(); - expect(failedCreate).toBeFalsy(); - expect(wrongExpected).toBeFalsy(); - expect(updated).toBeTruthy(); + expect(created).toBe(true); + expect(duplicateCreate).toBe(false); + expect(wrongExpected).toBe(false); + expect(updated).toBe(true); expect(await store.getOpIndex('k1')).toEqual({ opId: 'op-2' }); }); - test('queue and event adapters publish expected JetStream subjects', async () => { + test('queue routes layout jobs to the expected subject and publishes events', async () => { const js = new FakeJetStream(); const queue = new JetStreamOperationQueue({ - getJs: async () => js as any, + getJs: async () => js as never, whisperSubject: 'jobs.whisper', layoutSubject: 'jobs.layout', }); const events = new JetStreamOperationEventStream({ - getJs: async () => js as any, + getJs: async () => js as never, getJsm: async () => ({ consumers: { add: async () => ({ name: 'noop' }), delete: async () => true, }, - }) as any, + }) as never, eventsStreamName: 'compute_events', }); await queue.enqueue({ - jobId: 'j1', - opId: 'o1', - opKey: 'k1', + jobId: 'job-1', + opId: 'op-1', + opKey: 'k-1', kind: 'pdf_layout', queuedAt: 1000, payload: { documentId: 'd1', namespace: null, documentObjectKey: 'obj' }, }); - const appended = await events.append('o1', { - opId: 'o1', - opKey: 'k1', + const appended = await events.append('op-1', { + opId: 'op-1', + opKey: 'k-1', kind: 'pdf_layout', - jobId: 'j1', + jobId: 'job-1', status: 'queued', queuedAt: 1000, updatedAt: 1000, }); - expect(js.published.map((entry) => entry.subject)).toEqual(['jobs.layout', opEventsSubject('o1')]); + expect(js.published.map((entry) => entry.subject)).toEqual(['jobs.layout', opEventsSubject('op-1')]); expect(appended.eventId).toBe(2); }); - test('orchestrator integration writes index/state and reuses active op', async () => { + test('orchestrator writes expected index/state keys', async () => { const kv = new FakeKvStore(); const js = new FakeJetStream(); - const store = new JetStreamOperationStateStore({ getKv: async () => kv }); - const events = new JetStreamOperationEventStream({ - getJs: async () => js as any, + const stateStore = new JetStreamOperationStateStore({ getKv: async () => kv }); + const eventStream = new JetStreamOperationEventStream({ + getJs: async () => js as never, getJsm: async () => ({ consumers: { add: async () => ({ name: 'noop' }), delete: async () => true, }, - }) as any, + }) as never, eventsStreamName: 'compute_events', }); const queue = new JetStreamOperationQueue({ - getJs: async () => js as any, + getJs: async () => js as never, whisperSubject: 'jobs.whisper', layoutSubject: 'jobs.layout', }); @@ -189,8 +159,8 @@ test.describe('worker jetstream control-plane adapters', () => { let nextId = 1; const orchestrator = new OperationOrchestrator({ queue, - stateStore: store, - eventStream: events, + stateStore, + eventStream, config: { opStaleMs: 10_000, maxCasRetries: 3 }, clock: { now: () => now }, idFactory: { @@ -199,18 +169,18 @@ test.describe('worker jetstream control-plane adapters', () => { }, }); - const req = buildPdfRequest('k-integration'); - const first = await orchestrator.enqueueOrReuse(req); + const first = await orchestrator.enqueueOrReuse(buildPdfRequest('k-integration')); now = 2_000; - const reused = await orchestrator.enqueueOrReuse(req); + const reused = await orchestrator.enqueueOrReuse(buildPdfRequest('k-integration')); expect(first.opId).toBe('op-1'); expect(reused.opId).toBe('op-1'); const indexEntry = await kv.get(opIndexKvKey('k-integration')); const stateEntry = await kv.get(opStateKvKey('op-1')); + expect(indexEntry?.operation).toBe('PUT'); expect(stateEntry?.operation).toBe('PUT'); - expect(js.published.map((entry) => entry.subject)).toEqual(['ops.events.op-1', 'jobs.layout']); + expect(hashOpKey('k-integration')).toHaveLength(64); }); }); diff --git a/compute/worker/tests/unit/key-and-progress.test.ts b/compute/worker/tests/unit/key-and-progress.test.ts new file mode 100644 index 0000000..6887cad --- /dev/null +++ b/compute/worker/tests/unit/key-and-progress.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from 'vitest'; +import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/control-plane/jetstream'; +import { + buildInferProgressForPageParsed, + buildInferProgressForPageStart, +} from '../../src/pdf-progress'; + +describe('compute worker helpers', () => { + test('hash and kv key helpers are stable and deterministic', () => { + const hashA = hashOpKey('doc-123|layout|v1'); + const hashB = hashOpKey('doc-123|layout|v1'); + const hashC = hashOpKey('doc-123|layout|v2'); + + expect(hashA).toBe(hashB); + expect(hashA).toHaveLength(64); + expect(hashC).not.toBe(hashA); + + expect(opIndexKvKey('doc-123|layout|v1')).toBe(`op_index.${hashA}`); + expect(opStateKvKey('op-1')).toBe('op_state.op-1'); + }); + + test('progress helpers infer page counters correctly', () => { + expect(buildInferProgressForPageStart({ pageNumber: 1, totalPages: 12 })).toEqual({ + totalPages: 12, + pagesParsed: 0, + currentPage: 1, + phase: 'infer', + }); + + expect(buildInferProgressForPageParsed({ pageNumber: 5, totalPages: 12 })).toEqual({ + totalPages: 12, + pagesParsed: 5, + currentPage: 5, + phase: 'infer', + }); + }); +}); diff --git a/compute/worker/tests/unit/worker-loop-policy.test.ts b/compute/worker/tests/unit/worker-loop-policy.test.ts new file mode 100644 index 0000000..818e459 --- /dev/null +++ b/compute/worker/tests/unit/worker-loop-policy.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from 'vitest'; +import { buildQueueWaitTiming, decideRetryAction } from '../../src/worker-loop-policy'; + +describe('worker loop policy', () => { + test('returns queue wait timing with non-negative clamped duration', () => { + expect(buildQueueWaitTiming(1000, 1300)).toEqual({ queueWaitMs: 300 }); + expect(buildQueueWaitTiming(1500, 1300)).toEqual({ queueWaitMs: 0 }); + }); + + test('retry policy: layout jobs can retry until max attempts', () => { + expect(decideRetryAction({ kind: 'pdf_layout', deliveryCount: 1, pdfAttempts: 3 })).toBe('nak_retry'); + expect(decideRetryAction({ kind: 'pdf_layout', deliveryCount: 3, pdfAttempts: 3 })).toBe('term_fail'); + }); + + test('retry policy: whisper jobs terminate immediately by default', () => { + expect(decideRetryAction({ kind: 'whisper_align', deliveryCount: 1, pdfAttempts: 10 })).toBe('term_fail'); + }); +}); diff --git a/compute/worker/tsconfig.json b/compute/worker/tsconfig.json index 5912d1a..428e252 100644 --- a/compute/worker/tsconfig.json +++ b/compute/worker/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "noEmit": true + "noEmit": true, }, "include": ["src/**/*.ts"], "exclude": [] diff --git a/package.json b/package.json index eb62569..a7aa4d5 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,9 @@ "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", + "test:e2e": "playwright test", + "test:unit": "vitest run", + "test:compute": "vitest run --project compute-core --project compute-worker", "test:ci-env": "CI=true playwright test", "migrate": "node drizzle/scripts/migrate.mjs", "migrate-fs": "node scripts/migrate-fs-v2.mjs", @@ -90,6 +93,7 @@ "eslint-config-next": "^15.5.18", "postcss": "^8.5.14", "tailwindcss": "^3.4.19", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^4.1.7" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f15c27b..9d3aedd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,7 +42,7 @@ importers: version: 7.0.1 better-auth: specifier: ^1.6.11 - version: 1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + version: 1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))) better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -194,6 +194,9 @@ importers: typescript: specifier: ^5.9.3 version: 5.9.3 + vitest: + specifier: ^4.1.7 + version: 4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) compute/core: dependencies: @@ -1569,6 +1572,12 @@ packages: '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@napi-rs/wasm-runtime@1.1.4': + resolution: {integrity: sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + '@nats-io/jetstream@3.4.0': resolution: {integrity: sha512-GzHQodNJ942+R5LRb8PuZ5ugVWVWMRiufxUYLLVWkXKfwDXYN+Owo0d7L/b9O7BPyrbYD7jQWAC6+ZVuXa9Gyw==} @@ -1679,6 +1688,9 @@ packages: resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} + '@oxc-project/types@0.132.0': + resolution: {integrity: sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -1717,6 +1729,104 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1 + '@rolldown/binding-android-arm64@1.0.2': + resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.0.2': + resolution: {integrity: sha512-vdFA9+C/rekyGce7WqHs/xoT0ioZEWaOFyZLIV1mEeNFaFDUQrPIo8Vs2GvJ6eetb3rzDUtUBgzto3ExpXJB3w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.0.2': + resolution: {integrity: sha512-BewSOwTHazv77DTYiAZXSqqKZ4KP/KonFisDMVU7PImxoWfB2aepnPhd2E4SWz3zDzYgDNbs6jBmTdgNnF02GA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.0.2': + resolution: {integrity: sha512-m41o7M0YWtUdqk61Tb+jnKb2rN++iRdIASlExkUoKfIAH30DOHCB8fVLzSUpbWHHU8esmEioY62PxzexE8MBuA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-jcojB9H7W/jS29pMKWAK1N+fU99vXodHDTatS3b3y/XSOCiHo0kkA74pL3jJmkoQtYpOCxDvaKs1fo2Ij/1X5w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-1jn6qDU5iiOgFgygDzKUuKP0maTi0/f1+sBLgvij/76C77Nm3ts6ufz9Bjg5q5dduxiUIxtq86JIoBvo1xQ4Ig==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-QVLO/czFMdoMFSqlX3bcswcJNm/23r+qoa/jgtmFc/qEp6/jXmIkDjF/XIo8dPfGaiwy1xfQn8o77L79GeXFgw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + resolution: {integrity: sha512-hgO5Abm0w5UL6FEa2iFnZqo2KlK7TQ5QhV5x09hujBf7t5KzHQ1VmfPuTpqRy/rNlSxua3eWH374xxiVrP+lcA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.0.2': + resolution: {integrity: sha512-fy8rXxuYEu602abC8MUNaPjYLIFzReOaEIEMKMUa0rFEUxNpVXhs15KSSQ4qlqSaM7B6rcj9rDZgADh/IGDzLQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-0+bOkiQ779+r1WpoHOWHqncvyySci0vKph+myNDYb+im6meJAzHQXay6oEgnkHuUGouM1LKTZwqKpBow6Kj7CQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-mjSkrzZK5Qsl0a9d1JgILOiuZOSDTVdKENcSXBoqbzSrspLR/4/IRVDo5wd2GgZjNss/viBFJdeq+j7qH2nypw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.0.2': + resolution: {integrity: sha512-1v5vHasdfQAZoEHakBV72LIFAC9JjnymsiKxp+GEr/ma3+NJCPSaYK+qavInOovJkgwFrs7GccX2d6IgDA3Z5w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.0.2': + resolution: {integrity: sha512-mb1VobWn6NheziTk5/WEaR6AKVbrwT5sOi6C7zk3gy/pD1qtJfU1j4PgTo2NJnOtbL9Dl3Aeei8w9jJ7qC2jZQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@rolldown/binding-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-SqKonF56vA/L2yHwHYcEp2P34URpOZ7d1fS635cTkpDnUtEGdUbhI6NzsPdqeSWvAAeGDrxjWjNmibDIdFf9/A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-v7qRI7gXLRINcOGXt+7YmAZ6iFuyZVMIoXAxhd8oP+DR9dLfL9GfNIx7PLMxmhZdvq8waUJBQiWN9EKNy+TRBQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} @@ -1960,9 +2070,15 @@ packages: '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/debug@4.1.13': resolution: {integrity: sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==} + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -2207,6 +2323,35 @@ packages: vue-router: optional: true + '@vitest/expect@4.1.7': + resolution: {integrity: sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==} + + '@vitest/mocker@4.1.7': + resolution: {integrity: sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.1.7': + resolution: {integrity: sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==} + + '@vitest/runner@4.1.7': + resolution: {integrity: sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==} + + '@vitest/snapshot@4.1.7': + resolution: {integrity: sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==} + + '@vitest/spy@4.1.7': + resolution: {integrity: sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==} + + '@vitest/utils@4.1.7': + resolution: {integrity: sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==} + '@xmldom/xmldom@0.9.10': resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} engines: {node: '>=14.6'} @@ -2327,6 +2472,10 @@ packages: resolution: {integrity: sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==} engines: {node: '>= 0.4'} + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} @@ -2569,6 +2718,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -2635,6 +2788,9 @@ packages: resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} engines: {'0': node >= 6.0} + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + cookie@1.1.1: resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} engines: {node: '>=18'} @@ -2909,6 +3065,9 @@ packages: resolution: {integrity: sha512-HVLACW1TppGYjJ8H6/jqH/pqOtKRw6wMlrB23xfExmFWxFquAIWCmwoLsOyN96K4a5KbmOf5At9ZUO3GZbetAw==} engines: {node: '>= 0.4'} + es-module-lexer@2.1.0: + resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} @@ -3087,6 +3246,9 @@ packages: estree-util-is-identifier-name@3.0.0: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} @@ -3109,6 +3271,10 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + ext@1.7.0: resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} @@ -3624,6 +3790,80 @@ packages: light-my-request@6.6.0: resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -3658,6 +3898,9 @@ packages: resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} engines: {node: 20 || >=22} + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + make-cancellable-promise@1.3.2: resolution: {integrity: sha512-GCXh3bq/WuMbS+Ky4JBPW1hYTOU+znU+Q5m9Pu+pI8EoUqIHk9+tviOKC6/qhHh8C4/As3tzJ69IF32kdz85ww==} @@ -3953,6 +4196,9 @@ packages: resolution: {integrity: sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==} engines: {node: '>= 0.4'} + obug@2.1.1: + resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + on-exit-leak-free@2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} @@ -4037,6 +4283,9 @@ packages: resolution: {integrity: sha512-+vnG6S4dYcYxZd+CZxzXCNKdELYZSKfohrk98yajCo1PtRoDgCTrrwOvK1GT0UoAdVszagDVllQc0U1vaX4NUQ==} engines: {node: '>=6'} + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdfjs-dist@4.8.69: resolution: {integrity: sha512-IHZsA4T7YElCKNNXtiLgqScw4zPd3pG9do8UrznC757gMd7UPeHSL2qwNNMJo4r79fl8oj1Xx+1nh2YkzdMpLQ==} engines: {node: '>=18'} @@ -4177,6 +4426,10 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} + postcss@8.5.15: + resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} + engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} @@ -4411,6 +4664,11 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + rolldown@1.0.2: + resolution: {integrity: sha512-oZx5zVDtVB44AW3eaifgDml1gWRDZGvjcfdxonE4swNPG98PrrXjaO/KrnUjzlMnztCCRVlUueA1kCXhARGk6g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + rou3@0.7.12: resolution: {integrity: sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg==} @@ -4511,6 +4769,9 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -4545,6 +4806,12 @@ packages: stable-hash@0.0.5: resolution: {integrity: sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA==} + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@4.1.0: + resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + stop-iteration-iterator@1.1.0: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} @@ -4692,10 +4959,21 @@ packages: tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@1.2.3: + resolution: {integrity: sha512-g62dB+w1/OEFnPvmX0yd/HnetYITOL+1nJW7kitOycOeAvmbWC/nu0fwmmQ/kupNojqExzyC/T++pST/jRJ2mQ==} + engines: {node: '>=18'} + tinyglobby@0.2.16: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} + engines: {node: '>=14.0.0'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -4825,6 +5103,90 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite@8.0.14: + resolution: {integrity: sha512-s4BJJ+5y1pYL6Otw51FHhVJQhPnuRinKig64g/1+EUNaJsd3gCKdD31IPFvswUgW9/60QT9oFHbZHbQK5imcxw==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.1.18 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.1.7: + resolution: {integrity: sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.7 + '@vitest/browser-preview': 4.1.7 + '@vitest/browser-webdriverio': 4.1.7 + '@vitest/coverage-istanbul': 4.1.7 + '@vitest/coverage-v8': 4.1.7 + '@vitest/ui': 4.1.7 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + warning@4.0.3: resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} @@ -4849,6 +5211,11 @@ packages: engines: {node: '>= 8'} hasBin: true + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} @@ -6248,6 +6615,13 @@ snapshots: '@tybys/wasm-util': 0.10.2 optional: true + '@napi-rs/wasm-runtime@1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@tybys/wasm-util': 0.10.2 + optional: true + '@nats-io/jetstream@3.4.0': dependencies: '@nats-io/nats-core': 3.4.0 @@ -6326,6 +6700,8 @@ snapshots: '@opentelemetry/semantic-conventions@1.41.1': {} + '@oxc-project/types@0.132.0': {} + '@pinojs/redact@0.4.0': {} '@pkgjs/parseargs@0.11.0': @@ -6360,6 +6736,57 @@ snapshots: dependencies: react: 19.2.6 + '@rolldown/binding-android-arm64@1.0.2': + optional: true + + '@rolldown/binding-darwin-arm64@1.0.2': + optional: true + + '@rolldown/binding-darwin-x64@1.0.2': + optional: true + + '@rolldown/binding-freebsd-x64@1.0.2': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.0.2': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-x64-musl@1.0.2': + optional: true + + '@rolldown/binding-openharmony-arm64@1.0.2': + optional: true + + '@rolldown/binding-wasm32-wasi@1.0.2': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.0.2': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.0.2': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + '@rtsao/scc@1.1.0': {} '@rushstack/eslint-patch@1.16.1': {} @@ -6660,10 +7087,17 @@ snapshots: dependencies: '@types/node': 20.19.41 + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + '@types/debug@4.1.13': dependencies: '@types/ms': 2.1.0 + '@types/deep-eql@4.0.2': {} + '@types/estree-jsx@1.0.5': dependencies: '@types/estree': 1.0.9 @@ -6873,6 +7307,47 @@ snapshots: next: 15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 + '@vitest/expect@4.1.7': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 + chai: 6.2.2 + tinyrainbow: 3.1.0 + + '@vitest/mocker@4.1.7(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))': + dependencies: + '@vitest/spy': 4.1.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3) + + '@vitest/pretty-format@4.1.7': + dependencies: + tinyrainbow: 3.1.0 + + '@vitest/runner@4.1.7': + dependencies: + '@vitest/utils': 4.1.7 + pathe: 2.0.3 + + '@vitest/snapshot@4.1.7': + dependencies: + '@vitest/pretty-format': 4.1.7 + '@vitest/utils': 4.1.7 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.1.7': {} + + '@vitest/utils@4.1.7': + dependencies: + '@vitest/pretty-format': 4.1.7 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + '@xmldom/xmldom@0.9.10': {} abort-controller@3.0.0: @@ -7031,6 +7506,8 @@ snapshots: get-intrinsic: 1.3.0 is-array-buffer: 3.0.5 + assertion-error@2.0.1: {} + ast-types-flow@0.0.8: {} async-function@1.0.0: {} @@ -7096,7 +7573,7 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6): + better-auth@1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))): dependencies: '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0)) @@ -7123,6 +7600,7 @@ snapshots: pg: 8.20.0 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) + vitest: 4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) transitivePeerDependencies: - '@cloudflare/workers-types' - '@opentelemetry/api' @@ -7219,6 +7697,8 @@ snapshots: ccount@2.0.1: {} + chai@6.2.2: {} + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -7287,6 +7767,8 @@ snapshots: readable-stream: 3.6.2 typedarray: 0.0.6 + convert-source-map@2.0.0: {} + cookie@1.1.1: {} core-js@3.49.0: {} @@ -7531,6 +8013,8 @@ snapshots: iterator.prototype: 1.1.5 math-intrinsics: 1.1.0 + es-module-lexer@2.1.0: {} + es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 @@ -7891,6 +8375,10 @@ snapshots: estree-util-is-identifier-name@3.0.0: {} + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + esutils@2.0.3: {} event-emitter@0.3.5: @@ -7910,6 +8398,8 @@ snapshots: expand-template@2.0.3: {} + expect-type@1.3.0: {} + ext@1.7.0: dependencies: type: 2.7.3 @@ -8485,6 +8975,55 @@ snapshots: process-warning: 4.0.1 set-cookie-parser: 2.7.2 + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} @@ -8511,6 +9050,10 @@ snapshots: lru-cache@11.3.6: {} + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + make-cancellable-promise@1.3.2: {} make-event-props@1.6.2: {} @@ -9008,6 +9551,8 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 + obug@2.1.1: {} + on-exit-leak-free@2.1.2: {} once@1.4.0: @@ -9087,6 +9632,8 @@ snapshots: path2d@0.2.2: optional: true + pathe@2.0.3: {} + pdfjs-dist@4.8.69: optionalDependencies: canvas: 3.2.3 @@ -9232,6 +9779,12 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postcss@8.5.15: + dependencies: + nanoid: 3.3.12 + picocolors: 1.1.1 + source-map-js: 1.2.1 + postgres-array@2.0.0: {} postgres-bytea@1.0.1: {} @@ -9537,6 +10090,27 @@ snapshots: rfdc@1.4.1: {} + rolldown@1.0.2: + dependencies: + '@oxc-project/types': 0.132.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.0.2 + '@rolldown/binding-darwin-arm64': 1.0.2 + '@rolldown/binding-darwin-x64': 1.0.2 + '@rolldown/binding-freebsd-x64': 1.0.2 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.2 + '@rolldown/binding-linux-arm64-gnu': 1.0.2 + '@rolldown/binding-linux-arm64-musl': 1.0.2 + '@rolldown/binding-linux-ppc64-gnu': 1.0.2 + '@rolldown/binding-linux-s390x-gnu': 1.0.2 + '@rolldown/binding-linux-x64-gnu': 1.0.2 + '@rolldown/binding-linux-x64-musl': 1.0.2 + '@rolldown/binding-openharmony-arm64': 1.0.2 + '@rolldown/binding-wasm32-wasi': 1.0.2 + '@rolldown/binding-win32-arm64-msvc': 1.0.2 + '@rolldown/binding-win32-x64-msvc': 1.0.2 + rou3@0.7.12: {} run-parallel@1.2.0: @@ -9678,6 +10252,8 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} + signal-exit@4.1.0: {} simple-concat@1.0.1: {} @@ -9707,6 +10283,10 @@ snapshots: stable-hash@0.0.5: {} + stackback@0.0.2: {} + + std-env@4.1.0: {} + stop-iteration-iterator@1.1.0: dependencies: es-errors: 1.3.0 @@ -9928,11 +10508,17 @@ snapshots: tiny-invariant@1.3.3: {} + tinybench@2.9.0: {} + + tinyexec@1.2.3: {} + tinyglobby@0.2.16: dependencies: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 + tinyrainbow@3.1.0: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -10110,6 +10696,47 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 + vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.4 + postcss: 8.5.15 + rolldown: 1.0.2 + tinyglobby: 0.2.16 + optionalDependencies: + '@types/node': 20.19.41 + esbuild: 0.28.0 + fsevents: 2.3.3 + jiti: 1.21.7 + tsx: 4.22.3 + + vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)): + dependencies: + '@vitest/expect': 4.1.7 + '@vitest/mocker': 4.1.7(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) + '@vitest/pretty-format': 4.1.7 + '@vitest/runner': 4.1.7 + '@vitest/snapshot': 4.1.7 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.2.3 + tinyglobby: 0.2.16 + tinyrainbow: 3.1.0 + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.19.41 + transitivePeerDependencies: + - msw + warning@4.0.3: dependencies: loose-envify: 1.4.0 @@ -10159,6 +10786,11 @@ snapshots: dependencies: isexe: 2.0.0 + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + word-wrap@1.2.5: {} wrap-ansi@7.0.0: diff --git a/tests/unit/compute-abort-like-error.spec.ts b/tests/unit/compute-abort-like-error.spec.ts deleted file mode 100644 index e31287e..0000000 --- a/tests/unit/compute-abort-like-error.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { isAbortLikeError } from '../../src/lib/server/compute/abort-like-error'; - -test.describe('isAbortLikeError', () => { - test('matches abort-shaped errors', () => { - expect(isAbortLikeError(new DOMException('This operation was aborted', 'AbortError'))).toBe(true); - expect(isAbortLikeError(Object.assign(new Error('random'), { name: 'AbortError' }))).toBe(true); - expect(isAbortLikeError(new Error('This operation was aborted'))).toBe(true); - expect(isAbortLikeError({ name: 'AbortError' })).toBe(true); - }); - - test('does not match non-abort errors', () => { - expect(isAbortLikeError(new Error('boom'))).toBe(false); - expect(isAbortLikeError({ name: 'TypeError' })).toBe(false); - expect(isAbortLikeError(null)).toBe(false); - expect(isAbortLikeError(undefined)).toBe(false); - }); -}); diff --git a/tests/unit/compute-control-plane.spec.ts b/tests/unit/compute-control-plane.spec.ts deleted file mode 100644 index 7a9e338..0000000 --- a/tests/unit/compute-control-plane.spec.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { expect, test } from '@playwright/test'; -import type { WorkerOperationRequest, WorkerOperationState } from '../../compute/core/src/api-contracts'; -import { - encodeSseFrame, - explainReplacementReason, - InMemoryOperationEventStream, - InMemoryOperationQueue, - InMemoryOperationStateStore, - OperationOrchestrator, - parseSseEventId, - parseSsePayload, - shouldReuseExistingOperation, -} from '../../compute/core/src/control-plane'; - -function buildPdfLayoutRequest(opKey: string): WorkerOperationRequest { - return { - kind: 'pdf_layout', - opKey, - payload: { - documentId: `doc-${opKey}`, - documentObjectKey: `s3://bucket/${opKey}.pdf`, - namespace: null, - }, - }; -} - -test.describe('compute control-plane', () => { - test('in-memory queue and state store support enqueue/claim/CAS', async () => { - const queue = new InMemoryOperationQueue(); - const store = new InMemoryOperationStateStore(); - - await queue.enqueue({ - jobId: 'job-1', - opId: 'op-1', - opKey: 'k-1', - kind: 'pdf_layout', - queuedAt: 1000, - payload: { documentId: 'd1', documentObjectKey: 'obj1', namespace: null }, - }); - await queue.enqueue({ - jobId: 'job-2', - opId: 'op-2', - opKey: 'k-2', - kind: 'whisper_align', - queuedAt: 1100, - payload: { text: 'hello', audioObjectKey: 'obj2' }, - }); - - expect(queue.size()).toBe(2); - expect(queue.size('pdf_layout')).toBe(1); - expect(queue.size('whisper_align')).toBe(1); - - const claimedLayout = await queue.claimNext('pdf_layout'); - expect(claimedLayout?.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(0); - - const firstCas = await store.compareAndSetOpIndex({ - opKey: 'k-1', - newOpId: 'op-1', - expectedOpId: null, - }); - const secondCas = await store.compareAndSetOpIndex({ - opKey: 'k-1', - newOpId: 'op-2', - expectedOpId: null, - }); - - expect(firstCas).toBeTruthy(); - expect(secondCas).toBeFalsy(); - expect(await store.getOpIndex('k-1')).toEqual({ opId: 'op-1' }); - }); - - test('in-memory event stream replays from sinceEventId and streams live events', async () => { - const stream = new InMemoryOperationEventStream(); - - const queued: WorkerOperationState = { - opId: 'op-1', - opKey: 'k-1', - kind: 'pdf_layout', - jobId: 'job-1', - status: 'queued', - queuedAt: 1000, - updatedAt: 1000, - }; - const running: WorkerOperationState = { - ...queued, - status: 'running', - startedAt: 1200, - updatedAt: 1200, - }; - const succeeded: WorkerOperationState = { - ...running, - status: 'succeeded', - updatedAt: 1400, - result: { ok: true }, - }; - - await stream.append('op-1', queued); - await stream.append('op-1', running); - - const receivedEventIds: number[] = []; - const unsubscribe = await stream.subscribe({ - opId: 'op-1', - sinceEventId: 1, - onEvent: (event) => { - receivedEventIds.push(event.eventId); - }, - }); - - await stream.append('op-1', succeeded); - unsubscribe(); - - expect(receivedEventIds).toEqual([2, 3]); - }); - - test('orchestrator reuses fresh inflight operations and replaces stale ones', async () => { - let now = 1_000; - let nextId = 1; - const queue = new InMemoryOperationQueue(); - const stateStore = new InMemoryOperationStateStore(); - const eventStream = new InMemoryOperationEventStream(); - const orchestrator = new OperationOrchestrator({ - queue, - stateStore, - eventStream, - config: { opStaleMs: 2_000, maxCasRetries: 5 }, - clock: { now: () => now }, - idFactory: { - opId: () => `op-${nextId}`, - jobId: () => `job-${nextId++}`, - }, - }); - - const request = buildPdfLayoutRequest('same-op-key'); - - const first = await orchestrator.enqueueOrReuse(request); - expect(first.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(1); - - now = 2_000; - const reused = await orchestrator.enqueueOrReuse(request); - expect(reused.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(1); - - await orchestrator.markRunning({ opId: first.opId, updatedAt: 2_100 }); - - now = 6_000; - const replaced = await orchestrator.enqueueOrReuse(request); - expect(replaced.opId).toBe('op-2'); - expect(queue.size('pdf_layout')).toBe(2); - expect(await stateStore.getOpIndex('same-op-key')).toEqual({ opId: 'op-2' }); - - const op1Events = await eventStream.listSince('op-1', 0); - const op2Events = await eventStream.listSince('op-2', 0); - expect(op1Events.map((event) => event.eventId)).toEqual([1, 2]); - expect(op2Events.map((event) => event.eventId)).toEqual([1]); - }); - - test('state-machine helpers return consistent reuse/replacement decisions', () => { - const current: WorkerOperationState = { - opId: 'op-1', - opKey: 'same-op-key', - kind: 'pdf_layout', - jobId: 'job-1', - status: 'running', - queuedAt: 1000, - updatedAt: 2000, - }; - - expect(shouldReuseExistingOperation({ - current, - requestKind: 'pdf_layout', - now: 2500, - opStaleMs: 1_000, - })).toBeTruthy(); - - expect(shouldReuseExistingOperation({ - current, - requestKind: 'pdf_layout', - now: 4005, - opStaleMs: 1_000, - })).toBeFalsy(); - - expect(explainReplacementReason({ - current, - requestKind: 'pdf_layout', - now: 4005, - opStaleMs: 1_000, - })).toBe('stale_running'); - }); - - test('sse helpers encode and decode frame payload and id', () => { - const frame = encodeSseFrame({ - id: 42, - event: 'snapshot', - data: { ok: true }, - }); - expect(parseSseEventId(frame)).toBe(42); - expect(parseSsePayload(frame)).toBe('{"ok":true}'); - }); -}); diff --git a/tests/unit/compute-worker-op-key.spec.ts b/tests/unit/compute-worker-op-key.spec.ts deleted file mode 100644 index 2101af6..0000000 --- a/tests/unit/compute-worker-op-key.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; - -test.describe('compute worker pdf opKey', () => { - test('keeps stable key when no force token is provided', () => { - const base = { - documentId: 'doc-123', - namespace: 'ns-1', - documentObjectKey: 'docs/ns-1/doc-123', - }; - expect(buildPdfOpKey(base)).toBe('pdf_layout|v1|doc-123|ns-1|docs/ns-1/doc-123|'); - expect(buildPdfOpKey(base)).toBe(buildPdfOpKey(base)); - }); - - test('cache-busts key when force token is provided', () => { - const base = { - documentId: 'doc-123', - namespace: 'ns-1', - documentObjectKey: 'docs/ns-1/doc-123', - }; - const opKeyA = buildPdfOpKey({ ...base, forceToken: 'force-a' }); - const opKeyB = buildPdfOpKey({ ...base, forceToken: 'force-b' }); - const normal = buildPdfOpKey(base); - - expect(opKeyA).not.toBe(opKeyB); - expect(opKeyA).not.toBe(normal); - expect(opKeyB).not.toBe(normal); - }); -}); diff --git a/tests/unit/compute-worker-pdf-progress.spec.ts b/tests/unit/compute-worker-pdf-progress.spec.ts deleted file mode 100644 index 284fd04..0000000 --- a/tests/unit/compute-worker-pdf-progress.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { - buildInferProgressForPageParsed, - buildInferProgressForPageStart, -} from '../../compute/worker/src/pdf-progress'; - -test.describe('compute worker pdf progress helpers', () => { - test('page-start progress keeps current page but does not count it as parsed yet', () => { - expect(buildInferProgressForPageStart({ pageNumber: 1, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 0, - currentPage: 1, - phase: 'infer', - }); - - expect(buildInferProgressForPageStart({ pageNumber: 5, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 4, - currentPage: 5, - phase: 'infer', - }); - }); - - test('page-parsed progress counts the current page as parsed', () => { - expect(buildInferProgressForPageParsed({ pageNumber: 5, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 5, - currentPage: 5, - phase: 'infer', - }); - }); -}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..fb32f4f --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + reporters: process.env.CI ? ['default', 'github'] : ['default'], + projects: [ + { + test: { + name: 'compute-core', + environment: 'node', + include: ['compute/core/tests/control-plane/**/*.test.ts'], + }, + }, + { + test: { + name: 'compute-worker', + environment: 'node', + include: ['compute/worker/tests/{unit,api}/**/*.test.ts'], + setupFiles: ['compute/worker/tests/setup-env.ts'], + }, + }, + ], + }, +});