diff --git a/.env.example b/.env.example index 9de5c4d..69c8ed0 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,7 @@ API_BASE=http://localhost:8880/v1 API_KEY=api_key_optional -# (Optional) TTS request/cache tuning (leave unset to use defaults) +# (Optional) TTS request/cache tuning (defaults shown below; leave unset to use defaults) # TTS_CACHE_MAX_SIZE_BYTES=268435456 # 256MB # TTS_CACHE_TTL_MS=1800000 # 30 minutes # TTS_MAX_RETRIES=2 @@ -24,27 +24,24 @@ API_KEY=api_key_optional # TTS_RETRY_BACKOFF=2 # TTS_UPSTREAM_TIMEOUT_MS=285000 # 285 seconds -# (Optional) Enable TTS character rate limiting (default is `false`) -# TTS_ENABLE_RATE_LIMIT=true -# (Optional) TTS per-user daily limits (leave unset to use defaults) -# TTS_DAILY_LIMIT_ANONYMOUS=50000 -# TTS_DAILY_LIMIT_AUTHENTICATED=500000 -# (Optional) TTS IP backstop daily limits (leave unset to use defaults) -# TTS_IP_DAILY_LIMIT_ANONYMOUS=100000 -# TTS_IP_DAILY_LIMIT_AUTHENTICATED=1000000 - -# Auth configuration (recommended for contributors and public instances) +# Auth configuration (recommended; required for admin features) # (Optional) Auth is only enabled when AUTH_SECRET and BASE_URL are set BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network) AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -hex 32` AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional trusted origins (BASE_URL is always trusted) # (Optional) Allow anonymous auth sessions when auth is enabled (default: `false`) -USE_ANONYMOUS_AUTH_SESSIONS= +# USE_ANONYMOUS_AUTH_SESSIONS=false # (Optional) Sign in w/ GitHub Configuration -GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= -# (Optional) Disable Better Auth built-in rate limiting (useful for testing) -DISABLE_AUTH_RATE_LIMIT= +# GITHUB_CLIENT_ID= +# GITHUB_CLIENT_SECRET= +# (Optional) Disable Better Auth built-in rate limiting (useful for testing; default: `false`) +# DISABLE_AUTH_RATE_LIMIT=false + +# (Optional) Honor the x-openreader-test-namespace header on a production build. +# This is test/CI scaffolding and MUST stay unset on real deployments; it is set +# automatically by the Playwright web server. Non-production builds honor it +# without this flag. +# ENABLE_TEST_NAMESPACE=false # (Optional) Comma-separated list of emails that are auto-promoted to admin. # Admins see the "Admin" tab in Settings (TTS shared providers + site features). @@ -54,34 +51,28 @@ ADMIN_EMAILS= # (Optional) Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables. # Defaults to SQLite at docstore/sqlite3.db when not set. -POSTGRES_URL= +# POSTGRES_URL= # Embedded SeaweedFS weed mini config -# (Optional) Enable embedded weed mini for local S3-compatible storage (default: `true`) -USE_EMBEDDED_WEED_MINI= -WEED_MINI_DIR= -WEED_MINI_WAIT_SEC= +# (Optional) Enable embedded weed mini for local S3-compatible storage (defaults shown below) +# USE_EMBEDDED_WEED_MINI=true +# WEED_MINI_DIR=docstore/seaweedfs +# WEED_MINI_WAIT_SEC=20 # S3 storage config (use with embedded weed mini or external S3-compatible storage) # (Optional) For embedded weed mini, set explicit keys if you want stable credentials across restarts. S3_ACCESS_KEY_ID= S3_SECRET_ACCESS_KEY= S3_BUCKET= -S3_REGION= +# S3_REGION= # (Optional) If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host. -S3_ENDPOINT= -S3_FORCE_PATH_STYLE= -S3_PREFIX= +# S3_ENDPOINT= +# S3_FORCE_PATH_STYLE= +# S3_PREFIX= -# Migrations configuration -# (Optional) Skip automatic startup migrations when set to `false` (default: `true`) -RUN_DRIZZLE_MIGRATIONS= -# (Optional) Skip automatic filesystem->S3/DB migration pass when set to `false` (default: `true`) -RUN_FS_MIGRATIONS= - -# (Optional) Server library import roots (uses /docstore/library by default) -IMPORT_LIBRARY_DIR= -IMPORT_LIBRARY_DIRS= +# (Optional) Library import mode directory (uses /docstore/library by default) +# IMPORT_LIBRARY_DIR= +# IMPORT_LIBRARY_DIRS= # Compute # Embedded/local (default): leave COMPUTE_WORKER_URL empty. @@ -118,7 +109,7 @@ IMPORT_LIBRARY_DIRS= # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main # (Optional) Override ffmpeg binary path used for audiobook processing -FFMPEG_BIN= +# FFMPEG_BIN= # (Optional) Client feature flags — seeded into the admin-managed runtime # config on first boot, then ignored. Edit values from Settings → Admin → @@ -132,3 +123,11 @@ FFMPEG_BIN= # RUNTIME_SEED_DEFAULT_TTS_PROVIDER=custom-openai # RUNTIME_SEED_CHANGELOG_FEED_URL=https://docs.openreader.richardr.dev/changelog/manifest.json # RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT=true +# RUNTIME_SEED_DISABLE_TTS_LIMIT=true +# RUNTIME_SEED_DISABLE_COMPUTE_LIMIT=true + +# Migrations configuration +# (Optional) Skip automatic startup migrations when set to `false` (default: `true`) +# RUN_DRIZZLE_MIGRATIONS=true +# (Optional) Skip automatic filesystem->S3/DB migration pass when set to `false` (default: `true`) +# RUN_FS_MIGRATIONS=true diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml new file mode 100644 index 0000000..27c2102 --- /dev/null +++ b/.github/workflows/vitest.yml @@ -0,0 +1,27 @@ +name: Vitest Tests + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + vitest-testing: + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: lts/* + package-manager-cache: false + - uses: pnpm/action-setup@v6 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Apply drizzle migrations (SQLite) + run: pnpm migrate + - name: Run Vitest suites + run: pnpm test:unit diff --git a/compute/core/tests/control-plane/orchestrator.test.ts b/compute/core/tests/control-plane/orchestrator.test.ts new file mode 100644 index 0000000..2a03adf --- /dev/null +++ b/compute/core/tests/control-plane/orchestrator.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, test } from 'vitest'; +import type { WorkerOperationRequest } from '../../src/api-contracts'; +import { + InMemoryOperationEventStream, + InMemoryOperationQueue, + InMemoryOperationStateStore, + OperationOrchestrator, +} from '../../src/control-plane'; + +function buildRequest(opKey: string): WorkerOperationRequest { + return { + kind: 'pdf_layout', + opKey, + payload: { + documentId: `doc-${opKey}`, + namespace: null, + documentObjectKey: `s3://bucket/${opKey}.pdf`, + }, + }; +} + +describe('operation orchestrator', () => { + test('reuses fresh operation and replaces stale operation', async () => { + let now = 1_000; + let nextId = 1; + const queue = new InMemoryOperationQueue(); + const stateStore = new InMemoryOperationStateStore(); + const eventStream = new InMemoryOperationEventStream(); + + const orchestrator = new OperationOrchestrator({ + queue, + stateStore, + eventStream, + config: { opStaleMs: 2_000, maxCasRetries: 5 }, + clock: { now: () => now }, + idFactory: { + opId: () => `op-${nextId}`, + jobId: () => `job-${nextId++}`, + }, + }); + + const request = buildRequest('shared-op'); + + const first = await orchestrator.enqueueOrReuse(request); + expect(first.opId).toBe('op-1'); + + now = 2_000; + const reused = await orchestrator.enqueueOrReuse(request); + expect(reused.opId).toBe('op-1'); + + await orchestrator.markRunning({ opId: first.opId, updatedAt: 2_100 }); + + now = 6_000; + const replaced = await orchestrator.enqueueOrReuse(request); + expect(replaced.opId).toBe('op-2'); + expect(await stateStore.getOpIndex('shared-op')).toEqual({ opId: 'op-2' }); + expect(queue.size('pdf_layout')).toBe(2); + }); + + test('survives transient CAS conflict and eventually creates operation', async () => { + const queue = new InMemoryOperationQueue(); + const eventStream = new InMemoryOperationEventStream(); + const store = new InMemoryOperationStateStore(); + + let firstAttempt = true; + const conflictStore = { + getOpState: store.getOpState.bind(store), + putOpState: store.putOpState.bind(store), + getOpIndex: store.getOpIndex.bind(store), + compareAndSetOpIndex: async (input: { opKey: string; newOpId: string; expectedOpId: string | null }) => { + if (firstAttempt && input.expectedOpId === null) { + firstAttempt = false; + return false; + } + return store.compareAndSetOpIndex(input); + }, + }; + + let id = 1; + const orchestrator = new OperationOrchestrator({ + queue, + stateStore: conflictStore, + eventStream, + config: { opStaleMs: 2_000, maxCasRetries: 4 }, + idFactory: { + opId: () => `op-${id}`, + jobId: () => `job-${id++}`, + }, + }); + + const created = await orchestrator.enqueueOrReuse(buildRequest('cas-key')); + expect(created.opId).toMatch(/^op-/); + expect(await store.getOpIndex('cas-key')).toEqual({ opId: created.opId }); + }); +}); diff --git a/compute/core/tests/control-plane/sse.test.ts b/compute/core/tests/control-plane/sse.test.ts new file mode 100644 index 0000000..e207950 --- /dev/null +++ b/compute/core/tests/control-plane/sse.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from 'vitest'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse'; + +describe('sse codec', () => { + test('encodes event id and payload and decodes both reliably', () => { + const frame = encodeSseFrame({ + id: 42, + event: 'snapshot', + data: { ok: true, opId: 'op-1' }, + }); + + expect(frame).toContain('event: snapshot'); + expect(parseSseEventId(frame)).toBe(42); + expect(parseSsePayload(frame)).toBe('{"ok":true,"opId":"op-1"}'); + }); + + test('supports multiline data payload', () => { + const frame = encodeSseFrame({ + id: 5, + data: 'line1\nline2', + }); + + expect(parseSseEventId(frame)).toBe(5); + expect(parseSsePayload(frame)).toBe('line1\nline2'); + }); +}); diff --git a/compute/core/tests/control-plane/state-machine.test.ts b/compute/core/tests/control-plane/state-machine.test.ts new file mode 100644 index 0000000..e9617cb --- /dev/null +++ b/compute/core/tests/control-plane/state-machine.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from 'vitest'; +import type { WorkerOperationState } from '../../src/api-contracts'; +import { + explainReplacementReason, + isInflightStatus, + isTerminalStatus, + shouldReuseExistingOperation, +} from '../../src/control-plane/state-machine'; + +function runningState(overrides: Partial = {}): WorkerOperationState { + return { + opId: 'op-1', + opKey: 'key-1', + kind: 'pdf_layout', + jobId: 'job-1', + status: 'running', + queuedAt: 1_000, + updatedAt: 2_000, + ...overrides, + }; +} + +describe('state-machine decisions', () => { + test('identifies terminal and inflight states', () => { + expect(isTerminalStatus('succeeded')).toBe(true); + expect(isTerminalStatus('failed')).toBe(true); + expect(isTerminalStatus('queued')).toBe(false); + + expect(isInflightStatus('queued')).toBe(true); + expect(isInflightStatus('running')).toBe(true); + expect(isInflightStatus('failed')).toBe(false); + }); + + test('reuses fresh inflight operation and rejects stale inflight operation', () => { + const current = runningState(); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 2_900, + opStaleMs: 1_000, + })).toBe(true); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 3_100, + opStaleMs: 1_000, + })).toBe(false); + + expect(explainReplacementReason({ + current, + requestKind: 'pdf_layout', + now: 3_100, + opStaleMs: 1_000, + })).toBe('stale_running'); + }); + + test('never reuses kind-mismatched operation', () => { + const current = runningState({ kind: 'whisper_align' }); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 2_100, + opStaleMs: 10_000, + })).toBe(false); + + expect(explainReplacementReason({ + current, + requestKind: 'pdf_layout', + now: 2_100, + opStaleMs: 10_000, + })).toBe('kind_mismatch'); + }); +}); diff --git a/compute/worker/src/runtime.ts b/compute/worker/src/runtime.ts new file mode 100644 index 0000000..1804855 --- /dev/null +++ b/compute/worker/src/runtime.ts @@ -0,0 +1,1428 @@ +import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import { + connect, + nanos, + credsAuthenticator, + type NatsConnection, +} from '@nats-io/transport-node'; +import { + AckPolicy, + DeliverPolicy, + ReplayPolicy, + RetentionPolicy, + StorageType, + jetstream, + jetstreamManager, + type Consumer, + type JetStreamClient, + type JetStreamManager, + type JsMsg, +} from '@nats-io/jetstream'; +import { Kvm } from '@nats-io/kv'; +import { + ensureComputeModels, + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, +} from '@openreader/compute-core/local-runtime'; +import { + getComputeTimeoutConfig, + getComputeOpStaleMs, + getAvailableCpuCores, + getOnnxThreadsPerJob, + withIdleTimeoutAndHardCap, + withTimeout, +} from '@openreader/compute-core'; +import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + WorkerOperationEvent, + WhisperAlignJobRequest, + WhisperAlignJobResult, + WorkerJobState, + WorkerJobTiming, + WorkerOperationKind, + WorkerOperationRequest, + WorkerOperationState, + PdfLayoutProgress, +} from '@openreader/compute-core/api-contracts'; +import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + JetStreamOperationEventStream, + OP_EVENTS_SUBJECT_WILDCARD, + JetStreamOperationQueue, + JetStreamOperationStateStore, + hashOpKey, +} from './control-plane/jetstream'; +import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; +import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; +import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; + +const JOBS_STREAM_NAME = 'compute_jobs'; +const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; +const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; +const WHISPER_CONSUMER_NAME = 'compute_whisper'; +const LAYOUT_CONSUMER_NAME = 'compute_layout'; +const EVENTS_STREAM_NAME = 'compute_events'; +const COMPUTE_STATE_BUCKET = 'compute_state'; +const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; +const LOOP_ERROR_BACKOFF_MS = 500; +const RUNNING_HEARTBEAT_MS = 5000; +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const WHISPER_MAX_DELIVER = 1; +const NATS_API_TIMEOUT_MS = 60_000; +// Disconnect from NATS after this much continuous idle so the worker stops +// generating outbound traffic (pull polling + keepalive PINGs) and Railway can +// put it to sleep. Reconnect happens lazily on the next inbound request. +const IDLE_DISCONNECT_MS = 120_000; +const IDLE_CHECK_INTERVAL_MS = 5_000; +// Bounded pull window so consumer loops yield periodically and can be stopped +// cleanly when going idle, instead of blocking on a long-lived pull. +const PULL_EXPIRES_MS = 5_000; +const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); +const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); +const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { + whisper_align: 15_000, + pdf_layout: 120_000, +}; + +interface QueuedJob { + jobId: string; + opId: string; + opKey: string; + kind: WorkerOperationKind; + queuedAt: number; + payload: TPayload; +} + +interface NatsSession { + nc: NatsConnection; + js: JetStreamClient; + jsm: JetStreamManager; + kv: Awaited>; + whisperConsumer: Consumer; + layoutConsumer: Consumer; +} + +type StreamedOperationState = WorkerOperationState; + +interface OperationEventStreamLike { + subscribe(input: { + opId: string; + sinceEventId?: number; + onEvent: (event: WorkerOperationEvent) => void | Promise; + onError?: (error: unknown) => void; + }): Promise<() => void>; +} + +interface OperationStateStoreLike { + getOpState(opId: string): Promise; +} + +interface OrchestratorLike { + enqueueOrReuse(request: WorkerOperationRequest): Promise; + markRunning(input: { + opId: string; + startedAt?: number; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markProgress(input: { + opId: string; + progress: WorkerOperationState['progress']; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { + opId: string; + result: unknown; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +export interface ComputeWorkerRouteDeps { + orchestrator: OrchestratorLike; + operationStateStore: OperationStateStoreLike; + operationEventStream: OperationEventStreamLike; +} + +export interface CreateComputeWorkerAppOptions { + host?: string; + port?: number; + workerToken?: string; + routeDeps?: ComputeWorkerRouteDeps; + disableWorkers?: boolean; +} + +export interface ComputeWorkerApp { + app: FastifyInstance; + host: string; + port: number; + start(options?: { registerSignalHandlers?: boolean }): Promise; + close(): Promise; +} + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function readIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +function normalizeNatsReplicas(value: number): number { + if (value === 3 || value === 5) return value; + return 1; +} + +function parseBoolEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const normalized = raw.toLowerCase(); + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; +} + +function buildLoggerConfig(): boolean | Record { + const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); + const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; + if (format === 'json') { + return { + level, + base: null, + }; + } + return { + level, + base: null, + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + }, + }; +} + +function normalizeS3Prefix(prefix: string | undefined): string { + const value = (prefix || 'openreader').trim(); + return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; +} + +function sanitizeNamespace(namespace: string | null): string | null { + if (!namespace) return null; + return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; +} + +function documentParsedKey(id: string, namespace: string | null, prefix: string): string { + if (!DOCUMENT_ID_REGEX.test(id)) { + throw new Error(`Invalid document id: ${id}`); + } + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + return `${prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; +} + +function buildS3Client(): S3Client { + const bucket = requireEnv('S3_BUCKET'); + const region = requireEnv('S3_REGION'); + const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); + const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); + const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; + const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); + + void bucket; + + return new S3Client({ + region, + endpoint, + forcePathStyle, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + if (typeof body === 'object' && body !== null && 'on' in body) { + const stream = body as NodeJS.ReadableStream; + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); + } + return Buffer.concat(chunks); + } + throw new Error('Unsupported S3 response body type'); +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function isAuthed(request: FastifyRequest, expectedToken: string): boolean { + const auth = request.headers.authorization; + if (!auth?.startsWith('Bearer ')) return false; + const token = auth.slice('Bearer '.length).trim(); + return token === expectedToken; +} + +function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { + if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; + return Math.max(0, Math.floor((end as number) - (start as number))); +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + return String(error); +} + +function requestPath(request: FastifyRequest): string { + return request.url.split('?')[0] ?? request.url; +} + +function isHealthPath(path: string): boolean { + return path === '/health/live' || path === '/health/ready'; +} + +function extractTraceId(request: FastifyRequest): string | null { + const header = request.headers['x-openreader-trace-id']; + if (Array.isArray(header)) return header[0] ?? null; + return typeof header === 'string' ? header : null; +} + +function extractOpId(request: FastifyRequest, path: string): string | null { + const params = request.params as { opId?: unknown } | undefined; + if (params && typeof params.opId === 'string' && params.opId.trim()) { + return params.opId.trim(); + } + const match = path.match(/^\/ops\/([^/]+)/); + if (!match?.[1]) return null; + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } +} + +function isAlreadyExistsError(error: unknown): boolean { + const message = toErrorMessage(error).toLowerCase(); + return message.includes('already in use') || message.includes('already exists'); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +class ConcurrencyGate { + private readonly maxInFlight: number; + private inFlight = 0; + private readonly queue: Array<() => void> = []; + + constructor(limit: number) { + this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; + } + + async acquire(): Promise { + if (this.inFlight < this.maxInFlight) { + this.inFlight += 1; + return; + } + + await new Promise((resolve) => { + this.queue.push(() => { + this.inFlight += 1; + resolve(); + }); + }); + } + + release(): void { + this.inFlight = Math.max(0, this.inFlight - 1); + const next = this.queue.shift(); + if (next) next(); + } +} + +function isTerminalStatus(status: WorkerJobState): boolean { + return status === 'succeeded' || status === 'failed'; +} + +function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { + if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; + const maybe = result as { parsedObjectKey?: unknown }; + return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; +} + +const alignSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +const layoutSchema = z.object({ + documentId: z.string().trim().min(1), + namespace: z.string().trim().min(1).max(128).nullable(), + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +const operationCreateSchema = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('whisper_align'), + opKey: z.string().trim().min(1).max(1024), + payload: alignSchema, + }), + z.object({ + kind: z.literal('pdf_layout'), + opKey: z.string().trim().min(1).max(1024), + payload: layoutSchema, + }), +]); + +async function ensureJetStreamResources( + jsm: JetStreamManager, + whisperTimeoutMs: number, + pdfTimeoutMs: number, + pdfAttempts: number, + jobsMaxBytes: number, + eventsMaxBytes: number, + natsReplicas: number, +): Promise { + const streamConfig = { + name: JOBS_STREAM_NAME, + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + retention: RetentionPolicy.Workqueue, + storage: StorageType.File, + max_bytes: jobsMaxBytes, + num_replicas: natsReplicas, + }; + + try { + await jsm.streams.add(streamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.streams.update(JOBS_STREAM_NAME, { + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + max_bytes: jobsMaxBytes, + num_replicas: natsReplicas, + }); + } + + const eventsStreamConfig = { + name: EVENTS_STREAM_NAME, + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + retention: RetentionPolicy.Limits, + storage: StorageType.File, + max_bytes: eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: natsReplicas, + }; + + try { + await jsm.streams.add(eventsStreamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.streams.update(EVENTS_STREAM_NAME, { + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + max_bytes: eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: natsReplicas, + }); + } + + const ensureConsumer = async ( + name: string, + subject: string, + ackWaitMs: number, + maxDeliver: number, + ): Promise => { + const config = { + durable_name: name, + ack_policy: AckPolicy.Explicit, + deliver_policy: DeliverPolicy.All, + replay_policy: ReplayPolicy.Instant, + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }; + + try { + await jsm.consumers.add(JOBS_STREAM_NAME, config); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.consumers.update(JOBS_STREAM_NAME, name, { + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }); + } + }; + + await Promise.all([ + ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), + ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), + ]); +} + +export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { + const port = options.port ?? readIntEnv('PORT', 8081); + const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); + const workerToken = options.workerToken ?? requireEnv('COMPUTE_WORKER_TOKEN'); + const disableWorkers = options.disableWorkers ?? false; + const natsUrl = requireEnv('NATS_URL'); + const timeoutConfig = getComputeTimeoutConfig(); + + const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); + const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; + const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; + const pdfHardCapMs = timeoutConfig.pdfHardCapMs; + const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); + const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); + const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); + const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); + const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); + const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); + const opStaleMs = getComputeOpStaleMs(); + + const connectOpts: Parameters[0] = { servers: natsUrl }; + const natsCreds = process.env.NATS_CREDS?.trim(); + const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); + + if (natsCreds) { + console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); + connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); + } else if (natsCredsFile) { + console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); + const { readFileSync } = await import('node:fs'); + const credsData = readFileSync(natsCredsFile); + connectOpts.authenticator = credsAuthenticator(credsData); + } + + // Lazy NATS connection lifecycle. The worker connects on demand (first request + // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle + // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is + // transparent: any inbound /ops request both wakes the container and re-establishes + // the session via ensureConnected(). + let session: NatsSession | null = null; + let connecting: Promise | null = null; + let workerLoops: Promise[] = []; + let idleTimer: NodeJS.Timeout | null = null; + let stopping = false; + let loopStopRequested = false; + + // Activity accounting feeding the idle detector. The worker is considered idle + // only when no HTTP request is in flight, no SSE stream is open, no job is + // processing, and nothing has happened for IDLE_DISCONNECT_MS. + let inFlightHttp = 0; + let activeSse = 0; + let inFlightJobs = 0; + let lastActivityAt = Date.now(); + const jobGate = new ConcurrencyGate(jobConcurrency); + + const markActivity = (): void => { + lastActivityAt = Date.now(); + }; + + function startIdleTimer(): void { + if (idleTimer) return; + idleTimer = setInterval(() => { + if (!session || stopping) return; + if (inFlightHttp > 0 || activeSse > 0 || inFlightJobs > 0) return; + if (Date.now() - lastActivityAt < IDLE_DISCONNECT_MS) return; + void disconnect('idle'); + }, IDLE_CHECK_INTERVAL_MS); + // Don't let the idle checker keep the process alive on its own. + idleTimer.unref?.(); + } + + async function disconnect(reason: string): Promise { + const current = session; + if (!current) return; + // Clear synchronously (before any await) so concurrent requests reconnect a + // fresh session instead of using the connection we're about to close. + session = null; + loopStopRequested = true; + if (idleTimer) { + clearInterval(idleTimer); + idleTimer = null; + } + try { + await current.nc.close(); + } catch { + // ignore close errors + } + await Promise.allSettled(workerLoops); + workerLoops = []; + loopStopRequested = false; + app.log.info({ reason }, 'nats disconnected'); + } + + async function ensureConnected(): Promise { + if (session) return session; + if (connecting) return connecting; + connecting = (async () => { + const nc: NatsConnection = await connect(connectOpts); + const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); + const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); + await ensureJetStreamResources( + jsm, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + jobsStreamMaxBytes, + eventsStreamMaxBytes, + natsReplicas, + ); + const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { + replicas: natsReplicas, + history: 1, + ttl: COMPUTE_STATE_TTL_MS, + max_bytes: jobStatesMaxBytes, + }); + const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); + const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); + const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; + session = next; + markActivity(); + startWorkerLoops(next); + startIdleTimer(); + // Safety net: if the connection closes for any reason (network drop after + // exhausting reconnects, or our own disconnect), drop the stale session so + // the next request reconnects cleanly. + void nc.closed().then(() => { + if (session?.nc === nc) session = null; + }); + app.log.info('nats connected'); + return next; + })(); + try { + return await connecting; + } finally { + connecting = null; + } + } + + const s3 = disableWorkers ? null : buildS3Client(); + const s3Bucket = disableWorkers ? '' : requireEnv('S3_BUCKET'); + const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); + + const ensureSafeKey = (key: string): string => { + const trimmed = key.trim(); + if (!trimmed.startsWith(`${s3Prefix}/`)) { + throw new Error('Object key prefix mismatch'); + } + return trimmed; + }; + + const readObjectByKey = async (key: string): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const safeKey = ensureSafeKey(key); + const response = await s3.send(new GetObjectCommand({ + Bucket: s3Bucket, + Key: safeKey, + })); + const bytes = await bodyToBuffer(response.Body); + return toArrayBuffer(new Uint8Array(bytes)); + }; + + const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const key = documentParsedKey(documentId, namespace, s3Prefix); + const body = Buffer.from(JSON.stringify(parsed)); + await s3.send(new PutObjectCommand({ + Bucket: s3Bucket, + Key: key, + Body: body, + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + })); + return key; + }; + + if (prewarmModels && !disableWorkers) { + await ensureComputeModels(); + } + + const app = Fastify({ + logger: buildLoggerConfig(), + disableRequestLogging: true, + }); + app.log.info({ + jobConcurrency, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + opStaleMs, + availableCpuCores: getAvailableCpuCores(), + onnxThreadsPerJob: getOnnxThreadsPerJob(), + natsApiTimeoutMs: NATS_API_TIMEOUT_MS, + natsReplicas, + eventsStreamMaxBytes, + pdfLayoutHardCapMs: pdfHardCapMs, + }, 'compute runtime config'); + + const whisperJobCodec = createJsonCodec>(); + const layoutJobCodec = createJsonCodec>(); + + const defaultOperationStateStore = new JetStreamOperationStateStore({ + getKv: async () => (await ensureConnected()).kv, + }); + + const defaultOperationEventStream = new JetStreamOperationEventStream({ + getJs: async () => (await ensureConnected()).js, + getJsm: async () => (await ensureConnected()).jsm, + eventsStreamName: EVENTS_STREAM_NAME, + }); + + const operationQueue = new JetStreamOperationQueue({ + getJs: async () => (await ensureConnected()).js, + whisperSubject: WHISPER_JOBS_SUBJECT, + layoutSubject: LAYOUT_JOBS_SUBJECT, + }); + + const defaultOrchestrator = new OperationOrchestrator({ + queue: operationQueue, + stateStore: defaultOperationStateStore, + eventStream: defaultOperationEventStream, + config: { + opStaleMs, + maxCasRetries: 10, + }, + }); + + const operationStateStore = options.routeDeps?.operationStateStore ?? defaultOperationStateStore; + const operationEventStream = options.routeDeps?.operationEventStream ?? defaultOperationEventStream; + const orchestrator = options.routeDeps?.orchestrator ?? defaultOrchestrator; + + const getOpState = async (opId: string): Promise => { + return await operationStateStore.getOpState(opId); + }; + + const releaseHttp = (request: FastifyRequest): void => { + const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; + if (!counted[REQUEST_COUNTED_KEY]) return; + counted[REQUEST_COUNTED_KEY] = false; + inFlightHttp = Math.max(0, inFlightHttp - 1); + markActivity(); + }; + + app.addHook('onRequest', async (request, reply) => { + const path = requestPath(request); + (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); + // Count every request as in-flight activity so the idle detector never + // disconnects mid-request. Released in onResponse, or manually after hijack + // for SSE streams (where onResponse does not fire). + (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; + inFlightHttp += 1; + markActivity(); + if (isHealthPath(path)) return; + if (!isAuthed(request, workerToken)) { + return reply.code(401).send({ error: 'Unauthorized' }); + } + return; + }); + + app.addHook('onResponse', async (request, reply) => { + releaseHttp(request); + const path = requestPath(request); + if (isHealthPath(path)) return; + if (reply.statusCode >= 500) { + const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; + const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; + app.log.error({ + reqId: request.id, + method: request.method, + path, + statusCode: reply.statusCode, + durationMs, + traceId: extractTraceId(request) ?? null, + opId: extractOpId(request, path), + }, 'http.error'); + } + }); + + app.get('/health/live', async () => ({ ok: true })); + + // Reports readiness without forcing a NATS round-trip. Probing NATS here would + // reconnect (and keep) the connection open, defeating idle sleep, so we only + // report the current connection state. The worker reconnects lazily on the next + // /ops request regardless of what this returns. + app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); + + app.post('/ops', async (request, reply) => { + const parsed = operationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { + error: 'Invalid request body', + issues: parsed.error.issues, + }; + } + + const requestOp = parsed.data as WorkerOperationRequest; + const op = await orchestrator.enqueueOrReuse(requestOp); + app.log.info({ + kind: requestOp.kind, + opId: op.opId, + jobId: op.jobId, + status: op.status, + opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), + }, 'op.accepted'); + reply.code(202); + return op; + }); + + app.get('/ops/:opId', async (request, reply) => { + const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const state = await getOpState(params.data.opId); + if (!state) { + reply.code(404); + return { error: 'Operation not found' }; + } + + return state; + }); + + app.get('/ops/:opId/events', async (request, reply) => { + const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const initial = await getOpState(params.data.opId); + if (!initial) { + reply.code(404); + return { error: 'Operation not found' }; + } + + const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; + const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); + const lastEventIdHeader = request.headers['last-event-id']; + const cursorFromHeader = Number( + Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), + ); + const sinceEventId = Math.max( + 0, + Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, + Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, + ); + + reply.hijack(); + // onResponse will not fire for a hijacked reply, so release the HTTP in-flight + // count here and track the long-lived stream via activeSse instead. + releaseHttp(request); + activeSse += 1; + markActivity(); + reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); + reply.raw.setHeader('Connection', 'keep-alive'); + reply.raw.setHeader('X-Accel-Buffering', 'no'); + + let closed = false; + let unsubscribe: (() => void) | null = null; + + const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { + if (closed || reply.raw.writableEnded) return; + const frameEvent: WorkerOperationEvent = { + eventId, + snapshot, + }; + reply.raw.write(encodeSseFrame({ + id: eventId, + event: 'snapshot', + data: frameEvent, + })); + }; + + const closeStream = (): void => { + if (closed) return; + closed = true; + if (unsubscribe) { + unsubscribe(); + unsubscribe = null; + } + activeSse = Math.max(0, activeSse - 1); + markActivity(); + if (!reply.raw.writableEnded) { + reply.raw.end(); + } + }; + + request.raw.on('close', () => { + closeStream(); + }); + + try { + let current = initial; + let signature = JSON.stringify(current); + writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); + if (isTerminalStatus(current.status)) { + return reply; + } + + unsubscribe = await operationEventStream.subscribe({ + opId: params.data.opId, + sinceEventId, + onEvent: (event) => { + if (closed) return; + if (event.snapshot.opId !== params.data.opId) return; + const nextSignature = JSON.stringify(event.snapshot); + if (nextSignature !== signature) { + current = event.snapshot; + signature = nextSignature; + writeSnapshot(current, event.eventId); + } + if (isTerminalStatus(event.snapshot.status)) { + closeStream(); + } + }, + onError: (error) => { + app.log.warn({ + opId: params.data.opId, + error: toErrorMessage(error), + }, 'op events stream loop error'); + closeStream(); + }, + }); + + await new Promise((resolve) => { + request.raw.once('close', () => resolve()); + }); + } catch (error) { + app.log.warn({ + opId: params.data.opId, + error: toErrorMessage(error), + }, 'op events stream loop error'); + } finally { + closeStream(); + } + + return reply; + }); + + const runWhisper = async ( + payload: WhisperAlignJobRequest, + queueWaitMs: number, + ): Promise => { + const parsed = alignSchema.parse(payload); + + const s3FetchStartedAt = Date.now(); + const audioBuffer = await withTimeout( + readObjectByKey(parsed.audioObjectKey), + whisperTimeoutMs, + 'whisper s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + + const computeStartedAt = Date.now(); + const result = await withTimeout( + runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: parsed.text, + cacheKey: parsed.cacheKey, + lang: parsed.lang, + }), + whisperTimeoutMs, + 'whisper alignment job', + ); + + const computeMs = Date.now() - computeStartedAt; + return { + ...result, + timing: { + queueWaitMs, + s3FetchMs, + computeMs, + }, + }; + }; + + const runLayout = async ( + payload: PdfLayoutJobRequest, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ): Promise => { + const parsed = layoutSchema.parse(payload); + + const s3FetchStartedAt = Date.now(); + const pdfBytes = await withTimeout( + readObjectByKey(parsed.documentObjectKey), + Math.max(pdfTimeoutMs, 1_000), + 'pdf s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + + let lastTotalPages = 0; + let lastPagesParsed = 0; + const computeStartedAt = Date.now(); + const result = await withIdleTimeoutAndHardCap({ + idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), + hardCapMs: pdfHardCapMs, + label: 'pdf layout job', + run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ + documentId: parsed.documentId, + pdfBytes, + onPageStarted: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + if (!hooks?.onProgress) return; + await hooks.onProgress(buildInferProgressForPageStart({ + pageNumber, + totalPages, + })); + }, + onPageParsed: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + lastPagesParsed = pageNumber; + if (!hooks?.onProgress) return; + await hooks.onProgress(buildInferProgressForPageParsed({ + pageNumber, + totalPages, + })); + }, + }), + }); + + const computeMs = Date.now() - computeStartedAt; + if (hooks?.onProgress && lastTotalPages > 0) { + await hooks.onProgress({ + totalPages: lastTotalPages, + pagesParsed: lastPagesParsed, + currentPage: lastPagesParsed || undefined, + phase: 'merge', + }); + } + const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); + return { + parsedObjectKey, + timing: { + queueWaitMs, + s3FetchMs, + computeMs, + }, + }; + }; + + type ProcessMessageInput = { + msg: JsMsg; + codec: JsonCodec>; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + workerLabel: string; + }; + + type ProcessMessageContext = { + decoded: QueuedJob; + workerLabel: string; + startedAt: number; + queueWaitTiming?: { queueWaitMs: number }; + latestProgress?: PdfLayoutProgress; + }; + + function extractTiming(result: unknown): WorkerJobTiming | undefined { + if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; + return (result as { timing?: WorkerJobTiming }).timing; + } + + async function markRunning( + context: ProcessMessageContext, + updatedAt: number, + options?: { includeStartedAt?: boolean }, + ): Promise { + if (context.latestProgress) { + await orchestrator.markProgress({ + opId: context.decoded.opId, + progress: context.latestProgress, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + return; + } + + await orchestrator.markRunning({ + opId: context.decoded.opId, + ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + } + + async function markProgress( + context: ProcessMessageContext, + progress: PdfLayoutProgress, + updatedAt: number, + ): Promise { + context.latestProgress = progress; + await markRunning(context, updatedAt); + } + + async function markTerminal(input: { + context: ProcessMessageContext; + status: 'succeeded' | 'failed'; + result?: TResult; + errorMessage?: string; + timing?: WorkerJobTiming; + updatedAt: number; + }): Promise { + if (input.status === 'succeeded') { + await orchestrator.markSucceeded({ + opId: input.context.decoded.opId, + result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, + updatedAt: input.updatedAt, + ...(input.timing ? { timing: input.timing } : {}), + }); + return; + } + + await orchestrator.markFailed({ + opId: input.context.decoded.opId, + error: { message: input.errorMessage ?? 'unknown worker failure' }, + updatedAt: input.updatedAt, + ...(input.timing ? { timing: input.timing } : {}), + }); + } + + async function handleRetry(input: { + context: ProcessMessageContext | null; + msg: JsMsg; + errorMessage: string; + }): Promise { + const deliveryCount = input.msg.info.deliveryCount; + const kind = input.context?.decoded.kind ?? 'pdf_layout'; + const retryAction = decideRetryAction({ + kind, + deliveryCount, + pdfAttempts, + whisperMaxDeliver: WHISPER_MAX_DELIVER, + }); + const hasRetriesLeft = retryAction === 'nak_retry'; + const maxAttempts = kind === 'whisper_align' ? WHISPER_MAX_DELIVER : pdfAttempts; + + if (input.context) { + const now = Date.now(); + const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); + const persistOpUpdate = hasRetriesLeft + ? (input.context.latestProgress + ? orchestrator.markProgress({ + opId: input.context.decoded.opId, + progress: input.context.latestProgress, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + }) + : orchestrator.markRunning({ + opId: input.context.decoded.opId, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + })) + : markTerminal({ + context: input.context, + status: 'failed', + errorMessage: input.errorMessage, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + }); + + await persistOpUpdate.catch((stateError) => { + app.log.error({ + worker: input.context?.workerLabel, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation state'); + }); + } + + if (hasRetriesLeft) { + input.msg.nak(); + app.log.error({ + worker: input.context?.workerLabel, + kind: input.context?.decoded.kind, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + status: 'running', + error: input.errorMessage, + deliveryCount, + maxAttempts, + retryAction: 'nack_retry', + }, 'job.terminal'); + return; + } + + input.msg.term(input.errorMessage); + app.log.error({ + worker: input.context?.workerLabel, + kind: input.context?.decoded.kind, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + status: 'failed', + error: input.errorMessage, + deliveryCount, + maxAttempts, + retrySuppressed: kind === 'whisper_align' ? 'whisper_align' : undefined, + retryAction: 'term', + }, 'job.terminal'); + } + + async function processMessage(input: ProcessMessageInput): Promise { + let context: ProcessMessageContext | null = null; + let heartbeat: NodeJS.Timeout | null = null; + try { + const decoded = input.codec.decode(input.msg.data); + const startedAt = Date.now(); + context = { + decoded, + workerLabel: input.workerLabel, + startedAt, + queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), + }; + + await markRunning(context, startedAt); + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, + deliveryCount: input.msg.info.deliveryCount, + }, 'job.started'); + + heartbeat = setInterval(() => { + const now = Date.now(); + void markRunning(context!, now).catch((stateError) => { + app.log.error({ + worker: input.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation heartbeat state'); + }); + }, RUNNING_HEARTBEAT_MS); + + const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { + onProgress: async (progress) => { + await markProgress(context!, progress, Date.now()); + }, + }); + const resultTiming = extractTiming(result); + const now = Date.now(); + + await markTerminal({ + context, + status: 'succeeded', + result, + timing: resultTiming, + updatedAt: now, + }); + + input.msg.ack(); + const terminalDurationMs = safeDurationMs(context.startedAt, now); + const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; + if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + durationMs: terminalDurationMs ?? null, + timing: resultTiming ?? null, + }, 'job.stage'); + } + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + status: 'succeeded', + durationMs: terminalDurationMs ?? null, + resultRef: extractResultRef(decoded.kind, result), + timing: resultTiming ?? null, + }, 'job.terminal'); + } catch (error) { + await handleRetry({ + context, + msg: input.msg, + errorMessage: toErrorMessage(error), + }); + } finally { + if (heartbeat) clearInterval(heartbeat); + } + } + + async function createWorkerLoop(input: { + owner: NatsSession; + consumer: Consumer; + codec: JsonCodec>; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + workerLabel: string; + }): Promise { + // Exit when the loop's connection is no longer the active session (idle + // disconnect, unexpected close, or replaced by a reconnect). + const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; + while (!detached()) { + let msg: JsMsg | null = null; + try { + try { + // Bounded pull so the loop yields periodically and exits promptly when + // the session is torn down for idle (nc.close() rejects the pending pull). + msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); + } catch (error) { + if (detached()) return; + app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); + await sleep(LOOP_ERROR_BACKOFF_MS); + continue; + } + + // An empty pull is not activity; let the idle window advance. + if (!msg) continue; + markActivity(); + inFlightJobs += 1; + await jobGate.acquire(); + if (detached()) { + return; + } + await processMessage({ + msg, + codec: input.codec, + run: input.run, + workerLabel: input.workerLabel, + }); + } finally { + if (msg) { + jobGate.release(); + inFlightJobs = Math.max(0, inFlightJobs - 1); + markActivity(); + } + } + } + } + + function startWorkerLoops(active: NatsSession): void { + if (disableWorkers) return; + // Always starts a fresh set bound to the new session. Any loops from a prior + // session self-terminate once they observe session !== their owner. + workerLoops = []; + loopStopRequested = false; + for (let i = 0; i < jobConcurrency; i += 1) { + workerLoops.push(createWorkerLoop({ + owner: active, + consumer: active.whisperConsumer, + codec: whisperJobCodec, + run: runWhisper, + workerLabel: `whisper-${i + 1}`, + })); + } + for (let i = 0; i < jobConcurrency; i += 1) { + workerLoops.push(createWorkerLoop({ + owner: active, + consumer: active.layoutConsumer, + codec: layoutJobCodec, + run: runLayout, + workerLabel: `layout-${i + 1}`, + })); + } + } + + const close = async (): Promise => { + if (stopping) return; + stopping = true; + if (idleTimer) { + clearInterval(idleTimer); + idleTimer = null; + } + await app.close(); + await Promise.allSettled(workerLoops); + const current = session; + session = null; + if (current) { + try { + await current.nc.drain(); + } catch { + try { + await current.nc.close(); + } catch { + // ignore close errors + } + } + } + }; + + const registerSignalHandlers = (): void => { + process.once('SIGINT', () => { + void close().finally(() => process.exit(0)); + }); + + process.once('SIGTERM', () => { + void close().finally(() => process.exit(0)); + }); + }; + + return { + app, + host, + port, + async start(startOptions) { + if (startOptions?.registerSignalHandlers) { + registerSignalHandlers(); + } + await app.listen({ host, port }); + app.log.info({ host, port }, 'compute worker listening'); + }, + close, + }; +} + +export async function startComputeWorkerFromEnv(): Promise { + const runtime = await createComputeWorkerApp(); + await runtime.start({ registerSignalHandlers: true }); +} diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts index 8c62d03..1bf176c 100644 --- a/compute/worker/src/server.ts +++ b/compute/worker/src/server.ts @@ -1,1338 +1,6 @@ -import Fastify, { type FastifyRequest } from 'fastify'; -import { z } from 'zod'; -import { - connect, - nanos, - credsAuthenticator, - type NatsConnection, -} from '@nats-io/transport-node'; -import { - AckPolicy, - DeliverPolicy, - ReplayPolicy, - RetentionPolicy, - StorageType, - jetstream, - jetstreamManager, - type Consumer, - type JetStreamClient, - type JetStreamManager, - type JsMsg, -} from '@nats-io/jetstream'; -import { Kvm } from '@nats-io/kv'; -import { - ensureComputeModels, - runPdfLayoutFromPdfBuffer, - runWhisperAlignmentFromAudioBuffer, -} from '@openreader/compute-core/local-runtime'; -import { - getComputeTimeoutConfig, - getComputeOpStaleMs, - getAvailableCpuCores, - getOnnxThreadsPerJob, - withIdleTimeoutAndHardCap, - withTimeout, -} from '@openreader/compute-core'; -import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerJobState, - WorkerJobTiming, - WorkerOperationKind, - WorkerOperationRequest, - WorkerOperationState, - PdfLayoutProgress, -} from '@openreader/compute-core/api-contracts'; -import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { - JetStreamOperationEventStream, - OP_EVENTS_SUBJECT_WILDCARD, - JetStreamOperationQueue, - JetStreamOperationStateStore, - hashOpKey, -} from './control-plane/jetstream'; -import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; -import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; +import { startComputeWorkerFromEnv } from './runtime'; -const JOBS_STREAM_NAME = 'compute_jobs'; -const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; -const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; -const WHISPER_CONSUMER_NAME = 'compute_whisper'; -const LAYOUT_CONSUMER_NAME = 'compute_layout'; -const EVENTS_STREAM_NAME = 'compute_events'; -const COMPUTE_STATE_BUCKET = 'compute_state'; -const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; -const LOOP_ERROR_BACKOFF_MS = 500; -const RUNNING_HEARTBEAT_MS = 5000; -const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; -const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; -const WHISPER_MAX_DELIVER = 1; -const NATS_API_TIMEOUT_MS = 60_000; -// Disconnect from NATS after this much continuous idle so the worker stops -// generating outbound traffic (pull polling + keepalive PINGs) and Railway can -// put it to sleep. Reconnect happens lazily on the next inbound request. -const IDLE_DISCONNECT_MS = 120_000; -const IDLE_CHECK_INTERVAL_MS = 5_000; -// Bounded pull window so consumer loops yield periodically and can be stopped -// cleanly when going idle, instead of blocking on a long-lived pull. -const PULL_EXPIRES_MS = 5_000; -const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); -const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); -const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { - whisper_align: 15_000, - pdf_layout: 120_000, -}; - -interface QueuedJob { - jobId: string; - opId: string; - opKey: string; - kind: WorkerOperationKind; - queuedAt: number; - payload: TPayload; -} - -interface NatsSession { - nc: NatsConnection; - js: JetStreamClient; - jsm: JetStreamManager; - kv: Awaited>; - whisperConsumer: Consumer; - layoutConsumer: Consumer; -} - -type StreamedOperationState = WorkerOperationState; - -function requireEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} - -function readIntEnv(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -function normalizeNatsReplicas(value: number): number { - if (value === 3 || value === 5) return value; - return 1; -} - -function parseBoolEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const normalized = raw.toLowerCase(); - return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; -} - -function buildLoggerConfig(): boolean | Record { - const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); - const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; - if (format === 'json') { - return { - level, - base: null, - }; - } - return { - level, - base: null, - transport: { - target: 'pino-pretty', - options: { - colorize: true, - translateTime: 'SYS:standard', - ignore: 'pid,hostname', - }, - }, - }; -} - -function normalizeS3Prefix(prefix: string | undefined): string { - const value = (prefix || 'openreader').trim(); - return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; -} - -function sanitizeNamespace(namespace: string | null): string | null { - if (!namespace) return null; - return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; -} - -function documentParsedKey(id: string, namespace: string | null, prefix: string): string { - if (!DOCUMENT_ID_REGEX.test(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; -} - -function buildS3Client(): S3Client { - const bucket = requireEnv('S3_BUCKET'); - const region = requireEnv('S3_REGION'); - const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); - const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); - const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; - const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); - - void bucket; - - return new S3Client({ - region, - endpoint, - forcePathStyle, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - credentials: { - accessKeyId, - secretAccessKey, - }, - }); -} - -async function bodyToBuffer(body: unknown): Promise { - if (!body) return Buffer.alloc(0); - if (body instanceof Uint8Array) return Buffer.from(body); - if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); - if (body instanceof ArrayBuffer) return Buffer.from(body); - if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { - const maybe = body as { transformToByteArray?: () => Promise }; - if (typeof maybe.transformToByteArray === 'function') { - return Buffer.from(await maybe.transformToByteArray()); - } - } - if (typeof body === 'object' && body !== null && 'on' in body) { - const stream = body as NodeJS.ReadableStream; - const chunks: Buffer[] = []; - for await (const chunk of stream) { - if (Buffer.isBuffer(chunk)) chunks.push(chunk); - else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); - else chunks.push(Buffer.from(chunk as Uint8Array)); - } - return Buffer.concat(chunks); - } - throw new Error('Unsupported S3 response body type'); -} - -function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { - const copy = new Uint8Array(bytes.byteLength); - copy.set(bytes); - return copy.buffer; -} - -function isAuthed(request: FastifyRequest, expectedToken: string): boolean { - const auth = request.headers.authorization; - if (!auth?.startsWith('Bearer ')) return false; - const token = auth.slice('Bearer '.length).trim(); - return token === expectedToken; -} - -function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { - if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; - return Math.max(0, Math.floor((end as number) - (start as number))); -} - -function toErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - return String(error); -} - -function requestPath(request: FastifyRequest): string { - return request.url.split('?')[0] ?? request.url; -} - -function isHealthPath(path: string): boolean { - return path === '/health/live' || path === '/health/ready'; -} - -function extractTraceId(request: FastifyRequest): string | null { - const header = request.headers['x-openreader-trace-id']; - if (Array.isArray(header)) return header[0] ?? null; - return typeof header === 'string' ? header : null; -} - -function extractOpId(request: FastifyRequest, path: string): string | null { - const params = request.params as { opId?: unknown } | undefined; - if (params && typeof params.opId === 'string' && params.opId.trim()) { - return params.opId.trim(); - } - const match = path.match(/^\/ops\/([^/]+)/); - if (!match?.[1]) return null; - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; - } -} - -function isAlreadyExistsError(error: unknown): boolean { - const message = toErrorMessage(error).toLowerCase(); - return message.includes('already in use') || message.includes('already exists'); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -class ConcurrencyGate { - private readonly maxInFlight: number; - private inFlight = 0; - private readonly queue: Array<() => void> = []; - - constructor(limit: number) { - this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; - } - - async acquire(): Promise { - if (this.inFlight < this.maxInFlight) { - this.inFlight += 1; - return; - } - - await new Promise((resolve) => { - this.queue.push(() => { - this.inFlight += 1; - resolve(); - }); - }); - } - - release(): void { - this.inFlight = Math.max(0, this.inFlight - 1); - const next = this.queue.shift(); - if (next) next(); - } -} - -function isTerminalStatus(status: WorkerJobState): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { - if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; - const maybe = result as { parsedObjectKey?: unknown }; - return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; -} - -const alignSchema = z.object({ - text: z.string().trim().min(1), - lang: z.string().trim().min(1).max(16).optional(), - cacheKey: z.string().trim().min(1).max(256).optional(), - audioObjectKey: z.string().trim().min(1).max(2048), -}); - -const layoutSchema = z.object({ - documentId: z.string().trim().min(1), - namespace: z.string().trim().min(1).max(128).nullable(), - documentObjectKey: z.string().trim().min(1).max(2048), -}); - -const operationCreateSchema = z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('whisper_align'), - opKey: z.string().trim().min(1).max(1024), - payload: alignSchema, - }), - z.object({ - kind: z.literal('pdf_layout'), - opKey: z.string().trim().min(1).max(1024), - payload: layoutSchema, - }), -]); - -async function ensureJetStreamResources( - jsm: JetStreamManager, - whisperTimeoutMs: number, - pdfTimeoutMs: number, - pdfAttempts: number, - jobsMaxBytes: number, - eventsMaxBytes: number, - natsReplicas: number, -): Promise { - const streamConfig = { - name: JOBS_STREAM_NAME, - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - retention: RetentionPolicy.Workqueue, - storage: StorageType.File, - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(streamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(JOBS_STREAM_NAME, { - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }); - } - - const eventsStreamConfig = { - name: EVENTS_STREAM_NAME, - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - retention: RetentionPolicy.Limits, - storage: StorageType.File, - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(eventsStreamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(EVENTS_STREAM_NAME, { - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }); - } - - const ensureConsumer = async ( - name: string, - subject: string, - ackWaitMs: number, - maxDeliver: number, - ): Promise => { - const config = { - durable_name: name, - ack_policy: AckPolicy.Explicit, - deliver_policy: DeliverPolicy.All, - replay_policy: ReplayPolicy.Instant, - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }; - - try { - await jsm.consumers.add(JOBS_STREAM_NAME, config); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.consumers.update(JOBS_STREAM_NAME, name, { - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }); - } - }; - - await Promise.all([ - ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), - ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), - ]); -} - -async function main(): Promise { - const port = readIntEnv('PORT', 8081); - const host = process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'; - const workerToken = requireEnv('COMPUTE_WORKER_TOKEN'); - const natsUrl = requireEnv('NATS_URL'); - const timeoutConfig = getComputeTimeoutConfig(); - - const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); - const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; - const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; - const pdfHardCapMs = timeoutConfig.pdfHardCapMs; - const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); - const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); - const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); - const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); - const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); - const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); - const opStaleMs = getComputeOpStaleMs(); - - const connectOpts: Parameters[0] = { servers: natsUrl }; - const natsCreds = process.env.NATS_CREDS?.trim(); - const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); - - if (natsCreds) { - console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); - connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); - } else if (natsCredsFile) { - console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); - const { readFileSync } = await import('node:fs'); - const credsData = readFileSync(natsCredsFile); - connectOpts.authenticator = credsAuthenticator(credsData); - } - - // Lazy NATS connection lifecycle. The worker connects on demand (first request - // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle - // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound /ops request both wakes the container and re-establishes - // the session via ensureConnected(). - let session: NatsSession | null = null; - let connecting: Promise | null = null; - let workerLoops: Promise[] = []; - let idleTimer: NodeJS.Timeout | null = null; - let stopping = false; - let loopStopRequested = false; - - // Activity accounting feeding the idle detector. The worker is considered idle - // only when no HTTP request is in flight, no SSE stream is open, no job is - // processing, and nothing has happened for IDLE_DISCONNECT_MS. - let inFlightHttp = 0; - let activeSse = 0; - let inFlightJobs = 0; - let lastActivityAt = Date.now(); - const jobGate = new ConcurrencyGate(jobConcurrency); - - const markActivity = (): void => { - lastActivityAt = Date.now(); - }; - - function startIdleTimer(): void { - if (idleTimer) return; - idleTimer = setInterval(() => { - if (!session || stopping) return; - if (inFlightHttp > 0 || activeSse > 0 || inFlightJobs > 0) return; - if (Date.now() - lastActivityAt < IDLE_DISCONNECT_MS) return; - void disconnect('idle'); - }, IDLE_CHECK_INTERVAL_MS); - // Don't let the idle checker keep the process alive on its own. - idleTimer.unref?.(); - } - - async function disconnect(reason: string): Promise { - const current = session; - if (!current) return; - // Clear synchronously (before any await) so concurrent requests reconnect a - // fresh session instead of using the connection we're about to close. - session = null; - loopStopRequested = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - try { - await current.nc.close(); - } catch { - // ignore close errors - } - await Promise.allSettled(workerLoops); - workerLoops = []; - loopStopRequested = false; - app.log.info({ reason }, 'nats disconnected'); - } - - async function ensureConnected(): Promise { - if (session) return session; - if (connecting) return connecting; - connecting = (async () => { - const nc: NatsConnection = await connect(connectOpts); - const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); - const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); - await ensureJetStreamResources( - jsm, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - jobsStreamMaxBytes, - eventsStreamMaxBytes, - natsReplicas, - ); - const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { - replicas: natsReplicas, - history: 1, - ttl: COMPUTE_STATE_TTL_MS, - max_bytes: jobStatesMaxBytes, - }); - const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); - const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); - const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; - session = next; - markActivity(); - startWorkerLoops(next); - startIdleTimer(); - // Safety net: if the connection closes for any reason (network drop after - // exhausting reconnects, or our own disconnect), drop the stale session so - // the next request reconnects cleanly. - void nc.closed().then(() => { - if (session?.nc === nc) session = null; - }); - app.log.info('nats connected'); - return next; - })(); - try { - return await connecting; - } finally { - connecting = null; - } - } - - const s3 = buildS3Client(); - const s3Bucket = requireEnv('S3_BUCKET'); - const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); - - const ensureSafeKey = (key: string): string => { - const trimmed = key.trim(); - if (!trimmed.startsWith(`${s3Prefix}/`)) { - throw new Error('Object key prefix mismatch'); - } - return trimmed; - }; - - const readObjectByKey = async (key: string): Promise => { - const safeKey = ensureSafeKey(key); - const response = await s3.send(new GetObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - const bytes = await bodyToBuffer(response.Body); - return toArrayBuffer(new Uint8Array(bytes)); - }; - - const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { - const key = documentParsedKey(documentId, namespace, s3Prefix); - const body = Buffer.from(JSON.stringify(parsed)); - await s3.send(new PutObjectCommand({ - Bucket: s3Bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - })); - return key; - }; - - if (prewarmModels) { - await ensureComputeModels(); - } - - const app = Fastify({ - logger: buildLoggerConfig(), - disableRequestLogging: true, - }); - app.log.info({ - jobConcurrency, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - opStaleMs, - availableCpuCores: getAvailableCpuCores(), - onnxThreadsPerJob: getOnnxThreadsPerJob(), - natsApiTimeoutMs: NATS_API_TIMEOUT_MS, - natsReplicas, - eventsStreamMaxBytes, - pdfLayoutHardCapMs: pdfHardCapMs, - }, 'compute runtime config'); - - const whisperJobCodec = createJsonCodec>(); - const layoutJobCodec = createJsonCodec>(); - - const operationStateStore = new JetStreamOperationStateStore({ - getKv: async () => (await ensureConnected()).kv, - }); - - const operationEventStream = new JetStreamOperationEventStream({ - getJs: async () => (await ensureConnected()).js, - getJsm: async () => (await ensureConnected()).jsm, - eventsStreamName: EVENTS_STREAM_NAME, - }); - - const operationQueue = new JetStreamOperationQueue({ - getJs: async () => (await ensureConnected()).js, - whisperSubject: WHISPER_JOBS_SUBJECT, - layoutSubject: LAYOUT_JOBS_SUBJECT, - }); - - const orchestrator = new OperationOrchestrator({ - queue: operationQueue, - stateStore: operationStateStore, - eventStream: operationEventStream, - config: { - opStaleMs, - maxCasRetries: 10, - }, - }); - - const getOpState = async (opId: string): Promise => { - return await operationStateStore.getOpState(opId); - }; - - const releaseHttp = (request: FastifyRequest): void => { - const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; - if (!counted[REQUEST_COUNTED_KEY]) return; - counted[REQUEST_COUNTED_KEY] = false; - inFlightHttp = Math.max(0, inFlightHttp - 1); - markActivity(); - }; - - app.addHook('onRequest', async (request, reply) => { - const path = requestPath(request); - (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); - // Count every request as in-flight activity so the idle detector never - // disconnects mid-request. Released in onResponse, or manually after hijack - // for SSE streams (where onResponse does not fire). - (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; - inFlightHttp += 1; - markActivity(); - if (isHealthPath(path)) return; - if (!isAuthed(request, workerToken)) { - return reply.code(401).send({ error: 'Unauthorized' }); - } - return; - }); - - app.addHook('onResponse', async (request, reply) => { - releaseHttp(request); - const path = requestPath(request); - if (isHealthPath(path)) return; - if (reply.statusCode >= 500) { - const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; - const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; - app.log.error({ - reqId: request.id, - method: request.method, - path, - statusCode: reply.statusCode, - durationMs, - traceId: extractTraceId(request) ?? null, - opId: extractOpId(request, path), - }, 'http.error'); - } - }); - - app.get('/health/live', async () => ({ ok: true })); - - // Reports readiness without forcing a NATS round-trip. Probing NATS here would - // reconnect (and keep) the connection open, defeating idle sleep, so we only - // report the current connection state. The worker reconnects lazily on the next - // /ops request regardless of what this returns. - app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); - - app.post('/ops', async (request, reply) => { - const parsed = operationCreateSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - const requestOp = parsed.data as WorkerOperationRequest; - const op = await orchestrator.enqueueOrReuse(requestOp); - app.log.info({ - kind: requestOp.kind, - opId: op.opId, - jobId: op.jobId, - status: op.status, - opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), - }, 'op.accepted'); - reply.code(202); - return op; - }); - - app.get('/ops/:opId', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const state = await getOpState(params.data.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; - }); - - app.get('/ops/:opId/events', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const initial = await getOpState(params.data.opId); - if (!initial) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; - const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); - const lastEventIdHeader = request.headers['last-event-id']; - const cursorFromHeader = Number( - Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), - ); - const sinceEventId = Math.max( - 0, - Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, - Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, - ); - - reply.hijack(); - // onResponse will not fire for a hijacked reply, so release the HTTP in-flight - // count here and track the long-lived stream via activeSse instead. - releaseHttp(request); - activeSse += 1; - markActivity(); - reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); - reply.raw.setHeader('Connection', 'keep-alive'); - reply.raw.setHeader('X-Accel-Buffering', 'no'); - - let closed = false; - let unsubscribe: (() => void) | null = null; - - const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { - if (closed || reply.raw.writableEnded) return; - const frameEvent: WorkerOperationEvent = { - eventId, - snapshot, - }; - reply.raw.write(encodeSseFrame({ - id: eventId, - event: 'snapshot', - data: frameEvent, - })); - }; - - const closeStream = (): void => { - if (closed) return; - closed = true; - if (unsubscribe) { - unsubscribe(); - unsubscribe = null; - } - activeSse = Math.max(0, activeSse - 1); - markActivity(); - if (!reply.raw.writableEnded) { - reply.raw.end(); - } - }; - - request.raw.on('close', () => { - closeStream(); - }); - - try { - let current = initial; - let signature = JSON.stringify(current); - writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); - if (isTerminalStatus(current.status)) { - return reply; - } - - unsubscribe = await operationEventStream.subscribe({ - opId: params.data.opId, - sinceEventId, - onEvent: (event) => { - if (closed) return; - if (event.snapshot.opId !== params.data.opId) return; - const nextSignature = JSON.stringify(event.snapshot); - if (nextSignature !== signature) { - current = event.snapshot; - signature = nextSignature; - writeSnapshot(current, event.eventId); - } - if (isTerminalStatus(event.snapshot.status)) { - closeStream(); - } - }, - onError: (error) => { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - closeStream(); - }, - }); - - await new Promise((resolve) => { - request.raw.once('close', () => resolve()); - }); - } catch (error) { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - } finally { - closeStream(); - } - - return reply; - }); - - const runWhisper = async ( - payload: WhisperAlignJobRequest, - queueWaitMs: number, - ): Promise => { - const parsed = alignSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const audioBuffer = await withTimeout( - readObjectByKey(parsed.audioObjectKey), - whisperTimeoutMs, - 'whisper s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - const computeStartedAt = Date.now(); - const result = await withTimeout( - runWhisperAlignmentFromAudioBuffer({ - audioBuffer, - text: parsed.text, - cacheKey: parsed.cacheKey, - lang: parsed.lang, - }), - whisperTimeoutMs, - 'whisper alignment job', - ); - - const computeMs = Date.now() - computeStartedAt; - return { - ...result, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - const runLayout = async ( - payload: PdfLayoutJobRequest, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ): Promise => { - const parsed = layoutSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const pdfBytes = await withTimeout( - readObjectByKey(parsed.documentObjectKey), - Math.max(pdfTimeoutMs, 1_000), - 'pdf s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - let lastTotalPages = 0; - let lastPagesParsed = 0; - const computeStartedAt = Date.now(); - const result = await withIdleTimeoutAndHardCap({ - idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), - hardCapMs: pdfHardCapMs, - label: 'pdf layout job', - run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ - documentId: parsed.documentId, - pdfBytes, - onPageStarted: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageStart({ - pageNumber, - totalPages, - })); - }, - onPageParsed: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - lastPagesParsed = pageNumber; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageParsed({ - pageNumber, - totalPages, - })); - }, - }), - }); - - const computeMs = Date.now() - computeStartedAt; - if (hooks?.onProgress && lastTotalPages > 0) { - await hooks.onProgress({ - totalPages: lastTotalPages, - pagesParsed: lastPagesParsed, - currentPage: lastPagesParsed || undefined, - phase: 'merge', - }); - } - const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); - return { - parsedObjectKey, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - type ProcessMessageInput = { - msg: JsMsg; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }; - - type ProcessMessageContext = { - decoded: QueuedJob; - workerLabel: string; - startedAt: number; - queueWaitTiming?: { queueWaitMs: number }; - latestProgress?: PdfLayoutProgress; - }; - - function buildQueueWaitTiming(queuedAt: number, now: number): { queueWaitMs: number } | undefined { - const queueWaitMs = safeDurationMs(queuedAt, now); - return typeof queueWaitMs === 'number' ? { queueWaitMs } : undefined; - } - - function extractTiming(result: unknown): WorkerJobTiming | undefined { - if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; - return (result as { timing?: WorkerJobTiming }).timing; - } - - async function markRunning( - context: ProcessMessageContext, - updatedAt: number, - options?: { includeStartedAt?: boolean }, - ): Promise { - if (context.latestProgress) { - await orchestrator.markProgress({ - opId: context.decoded.opId, - progress: context.latestProgress, - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - return; - } - - await orchestrator.markRunning({ - opId: context.decoded.opId, - ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - } - - async function markProgress( - context: ProcessMessageContext, - progress: PdfLayoutProgress, - updatedAt: number, - ): Promise { - context.latestProgress = progress; - await markRunning(context, updatedAt); - } - - async function markTerminal(input: { - context: ProcessMessageContext; - status: 'succeeded' | 'failed'; - result?: TResult; - errorMessage?: string; - timing?: WorkerJobTiming; - updatedAt: number; - }): Promise { - if (input.status === 'succeeded') { - await orchestrator.markSucceeded({ - opId: input.context.decoded.opId, - result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - return; - } - - await orchestrator.markFailed({ - opId: input.context.decoded.opId, - error: { message: input.errorMessage ?? 'unknown worker failure' }, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - } - - async function handleRetry(input: { - context: ProcessMessageContext | null; - msg: JsMsg; - errorMessage: string; - }): Promise { - const deliveryCount = input.msg.info.deliveryCount; - const isWhisperAlign = input.context?.decoded.kind === 'whisper_align'; - const maxAttempts = isWhisperAlign ? WHISPER_MAX_DELIVER : pdfAttempts; - const hasRetriesLeft = !isWhisperAlign && deliveryCount < maxAttempts; - - if (input.context) { - const now = Date.now(); - const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); - const persistOpUpdate = hasRetriesLeft - ? (input.context.latestProgress - ? orchestrator.markProgress({ - opId: input.context.decoded.opId, - progress: input.context.latestProgress, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }) - : orchestrator.markRunning({ - opId: input.context.decoded.opId, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - })) - : markTerminal({ - context: input.context, - status: 'failed', - errorMessage: input.errorMessage, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }); - - await persistOpUpdate.catch((stateError) => { - app.log.error({ - worker: input.context?.workerLabel, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation state'); - }); - } - - if (hasRetriesLeft) { - input.msg.nak(); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'running', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retryAction: 'nack_retry', - }, 'job.terminal'); - return; - } - - input.msg.term(input.errorMessage); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'failed', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retrySuppressed: isWhisperAlign ? 'whisper_align' : undefined, - retryAction: 'term', - }, 'job.terminal'); - } - - async function processMessage(input: ProcessMessageInput): Promise { - let context: ProcessMessageContext | null = null; - let heartbeat: NodeJS.Timeout | null = null; - try { - const decoded = input.codec.decode(input.msg.data); - const startedAt = Date.now(); - context = { - decoded, - workerLabel: input.workerLabel, - startedAt, - queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), - }; - - await markRunning(context, startedAt); - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, - deliveryCount: input.msg.info.deliveryCount, - }, 'job.started'); - - heartbeat = setInterval(() => { - const now = Date.now(); - void markRunning(context!, now).catch((stateError) => { - app.log.error({ - worker: input.workerLabel, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation heartbeat state'); - }); - }, RUNNING_HEARTBEAT_MS); - - const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { - onProgress: async (progress) => { - await markProgress(context!, progress, Date.now()); - }, - }); - const resultTiming = extractTiming(result); - const now = Date.now(); - - await markTerminal({ - context, - status: 'succeeded', - result, - timing: resultTiming, - updatedAt: now, - }); - - input.msg.ack(); - const terminalDurationMs = safeDurationMs(context.startedAt, now); - const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; - if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - durationMs: terminalDurationMs ?? null, - timing: resultTiming ?? null, - }, 'job.stage'); - } - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - status: 'succeeded', - durationMs: terminalDurationMs ?? null, - resultRef: extractResultRef(decoded.kind, result), - timing: resultTiming ?? null, - }, 'job.terminal'); - } catch (error) { - await handleRetry({ - context, - msg: input.msg, - errorMessage: toErrorMessage(error), - }); - } finally { - if (heartbeat) clearInterval(heartbeat); - } - } - - async function createWorkerLoop(input: { - owner: NatsSession; - consumer: Consumer; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }): Promise { - // Exit when the loop's connection is no longer the active session (idle - // disconnect, unexpected close, or replaced by a reconnect). - const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; - while (!detached()) { - let msg: JsMsg | null = null; - try { - try { - // Bounded pull so the loop yields periodically and exits promptly when - // the session is torn down for idle (nc.close() rejects the pending pull). - msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); - } catch (error) { - if (detached()) return; - app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); - await sleep(LOOP_ERROR_BACKOFF_MS); - continue; - } - - // An empty pull is not activity; let the idle window advance. - if (!msg) continue; - markActivity(); - inFlightJobs += 1; - await jobGate.acquire(); - if (detached()) { - return; - } - await processMessage({ - msg, - codec: input.codec, - run: input.run, - workerLabel: input.workerLabel, - }); - } finally { - if (msg) { - jobGate.release(); - inFlightJobs = Math.max(0, inFlightJobs - 1); - markActivity(); - } - } - } - } - - function startWorkerLoops(active: NatsSession): void { - // Always starts a fresh set bound to the new session. Any loops from a prior - // session self-terminate once they observe session !== their owner. - workerLoops = []; - loopStopRequested = false; - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.whisperConsumer, - codec: whisperJobCodec, - run: runWhisper, - workerLabel: `whisper-${i + 1}`, - })); - } - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.layoutConsumer, - codec: layoutJobCodec, - run: runLayout, - workerLabel: `layout-${i + 1}`, - })); - } - } - - const close = async (): Promise => { - if (stopping) return; - stopping = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - await app.close(); - await Promise.allSettled(workerLoops); - const current = session; - session = null; - if (current) { - try { - await current.nc.drain(); - } catch { - try { - await current.nc.close(); - } catch { - // ignore close errors - } - } - } - }; - - process.once('SIGINT', () => { - void close().finally(() => process.exit(0)); - }); - - process.once('SIGTERM', () => { - void close().finally(() => process.exit(0)); - }); - - await app.listen({ host, port }); - app.log.info({ host, port }, 'compute worker listening'); -} - -void main().catch((error) => { +void startComputeWorkerFromEnv().catch((error) => { console.error('[compute-worker] fatal startup error', error); process.exit(1); }); diff --git a/compute/worker/src/worker-loop-policy.ts b/compute/worker/src/worker-loop-policy.ts new file mode 100644 index 0000000..7462b4c --- /dev/null +++ b/compute/worker/src/worker-loop-policy.ts @@ -0,0 +1,22 @@ +import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts'; + +export type RetryAction = 'nak_retry' | 'term_fail'; + +export function buildQueueWaitTiming(queuedAt: number, now: number): { queueWaitMs: number } | undefined { + if (!Number.isFinite(queuedAt) || !Number.isFinite(now)) return undefined; + return { queueWaitMs: Math.max(0, Math.floor(now - queuedAt)) }; +} + +export function decideRetryAction(input: { + kind: WorkerOperationKind; + deliveryCount: number; + pdfAttempts: number; + whisperMaxDeliver?: number; +}): RetryAction { + const whisperMaxDeliver = input.whisperMaxDeliver ?? 1; + if (input.kind === 'whisper_align') { + return input.deliveryCount < whisperMaxDeliver ? 'nak_retry' : 'term_fail'; + } + + return input.deliveryCount < input.pdfAttempts ? 'nak_retry' : 'term_fail'; +} diff --git a/compute/worker/tests/api/routes.test.ts b/compute/worker/tests/api/routes.test.ts new file mode 100644 index 0000000..77c4d05 --- /dev/null +++ b/compute/worker/tests/api/routes.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { createComputeWorkerApp } from '../../src/runtime'; +import { FakeControlPlane } from '../fixtures/fake-control-plane'; + +const AUTH = { authorization: 'Bearer test-token' }; + +describe('compute worker API routes', () => { + let fake: FakeControlPlane; + let runtime: Awaited>; + + beforeEach(async () => { + fake = new FakeControlPlane(); + runtime = await createComputeWorkerApp({ + workerToken: 'test-token', + disableWorkers: true, + routeDeps: fake.deps, + }); + }); + + afterEach(async () => { + await runtime.close(); + }); + + test('allows unauthenticated health checks but protects operation routes', async () => { + const live = await runtime.app.inject({ method: 'GET', url: '/health/live' }); + expect(live.statusCode).toBe(200); + + const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' }); + expect(protectedRoute.statusCode).toBe(401); + }); + + test('validates operation creation body and returns 400 for invalid payload', async () => { + const response = await runtime.app.inject({ + method: 'POST', + url: '/ops', + headers: AUTH, + payload: { + kind: 'pdf_layout', + opKey: '', + payload: { + documentId: 'd1', + namespace: null, + documentObjectKey: 's3://bucket/doc.pdf', + }, + }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json()).toMatchObject({ error: 'Invalid request body' }); + }); + + test('creates operation and fetches state by op id', async () => { + const create = await runtime.app.inject({ + method: 'POST', + url: '/ops', + headers: AUTH, + payload: { + kind: 'pdf_layout', + opKey: 'doc-1:layout', + payload: { + documentId: 'doc-1', + namespace: null, + documentObjectKey: 'openreader/doc-1.pdf', + }, + }, + }); + + expect(create.statusCode).toBe(202); + const created = create.json(); + expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' }); + + const fetch = await runtime.app.inject({ + method: 'GET', + url: `/ops/${created.opId}`, + headers: AUTH, + }); + + expect(fetch.statusCode).toBe(200); + expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' }); + }); + + test('returns not found for unknown operation and event stream lookups', async () => { + const opResponse = await runtime.app.inject({ + method: 'GET', + url: '/ops/missing', + headers: AUTH, + }); + expect(opResponse.statusCode).toBe(404); + + const eventsResponse = await runtime.app.inject({ + method: 'GET', + url: '/ops/missing/events', + headers: AUTH, + }); + expect(eventsResponse.statusCode).toBe(404); + }); + + test('streams initial SSE snapshot for terminal operation and honors cursor id', async () => { + fake.seedState({ + opId: 'op-terminal', + opKey: 'k-terminal', + kind: 'pdf_layout', + jobId: 'job-op-terminal', + status: 'succeeded', + queuedAt: 1000, + updatedAt: 2000, + result: { parsedObjectKey: 'openreader/parsed.json' }, + }); + + const stream = await runtime.app.inject({ + method: 'GET', + url: '/ops/op-terminal/events?sinceEventId=7', + headers: AUTH, + }); + + expect(stream.statusCode).toBe(200); + expect(stream.headers['content-type']).toContain('text/event-stream'); + expect(stream.body).toContain('event: snapshot'); + expect(stream.body).toContain('id: 7'); + expect(stream.body).toContain('"status":"succeeded"'); + }); +}); diff --git a/compute/worker/tests/fixtures/fake-control-plane.ts b/compute/worker/tests/fixtures/fake-control-plane.ts new file mode 100644 index 0000000..2d18f2b --- /dev/null +++ b/compute/worker/tests/fixtures/fake-control-plane.ts @@ -0,0 +1,119 @@ +import type { + PdfLayoutJobResult, + WhisperAlignJobResult, + WorkerOperationEvent, + WorkerOperationRequest, + WorkerOperationState, +} from '@openreader/compute-core/api-contracts'; +import type { ComputeWorkerRouteDeps } from '../../src/runtime'; + +type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; +type ComputeState = WorkerOperationState; +type ComputeEvent = WorkerOperationEvent; + +export class FakeControlPlane { + private readonly stateByOpId = new Map(); + private readonly opIdByOpKey = new Map(); + private readonly eventsByOpId = new Map(); + private nextOpId = 1; + + readonly deps: ComputeWorkerRouteDeps = { + orchestrator: { + enqueueOrReuse: async (request) => this.enqueueOrReuse(request), + markRunning: async (input) => this.updateState(input.opId, { + status: 'running', + startedAt: input.startedAt, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markProgress: async (input) => this.updateState(input.opId, { + status: 'running', + progress: input.progress, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markSucceeded: async (input) => this.updateState(input.opId, { + status: 'succeeded', + result: input.result as ComputeResult, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markFailed: async (input) => this.updateState(input.opId, { + status: 'failed', + error: typeof input.error === 'string' ? { message: input.error } : input.error, + updatedAt: input.updatedAt, + timing: input.timing, + }), + }, + operationStateStore: { + getOpState: async (opId) => this.stateByOpId.get(opId) ?? null, + }, + operationEventStream: { + subscribe: async ({ opId, sinceEventId, onEvent }) => { + const since = Math.max(0, Math.floor(sinceEventId ?? 0)); + const replay = (this.eventsByOpId.get(opId) ?? []).filter((event) => event.eventId > since); + for (const event of replay) { + await onEvent(event); + } + return () => undefined; + }, + }, + }; + + seedState(state: ComputeState): void { + this.stateByOpId.set(state.opId, state); + this.opIdByOpKey.set(state.opKey, state.opId); + } + + seedEvent(opId: string, event: ComputeEvent): void { + const list = this.eventsByOpId.get(opId) ?? []; + list.push(event); + this.eventsByOpId.set(opId, list); + } + + private async enqueueOrReuse(request: WorkerOperationRequest): Promise { + const existingId = this.opIdByOpKey.get(request.opKey); + if (existingId) { + const existing = this.stateByOpId.get(existingId); + if (existing) return existing; + } + + const now = Date.now(); + const opId = `op-${this.nextOpId++}`; + const state: ComputeState = { + opId, + opKey: request.opKey, + kind: request.kind, + jobId: `job-${opId}`, + status: 'queued', + queuedAt: now, + updatedAt: now, + }; + + this.stateByOpId.set(opId, state); + this.opIdByOpKey.set(request.opKey, opId); + this.seedEvent(opId, { eventId: 1, snapshot: state }); + return state; + } + + private async updateState( + opId: string, + patch: Partial, + ): Promise { + const current = this.stateByOpId.get(opId); + if (!current) { + throw new Error(`Unknown opId: ${opId}`); + } + const next: ComputeState = { + ...current, + ...patch, + updatedAt: patch.updatedAt ?? Date.now(), + }; + this.stateByOpId.set(opId, next); + const currentEvents = this.eventsByOpId.get(opId) ?? []; + const nextEventId = (currentEvents.at(-1)?.eventId ?? 0) + 1; + currentEvents.push({ eventId: nextEventId, snapshot: next }); + this.eventsByOpId.set(opId, currentEvents); + return next; + } +} diff --git a/compute/worker/tests/setup-env.ts b/compute/worker/tests/setup-env.ts new file mode 100644 index 0000000..d013982 --- /dev/null +++ b/compute/worker/tests/setup-env.ts @@ -0,0 +1,8 @@ +process.env.COMPUTE_WORKER_TOKEN = process.env.COMPUTE_WORKER_TOKEN || 'test-token'; +process.env.NATS_URL = process.env.NATS_URL || 'nats://127.0.0.1:4222'; +process.env.COMPUTE_PREWARM_MODELS = 'false'; +process.env.S3_BUCKET = process.env.S3_BUCKET || 'test-bucket'; +process.env.S3_REGION = process.env.S3_REGION || 'us-east-1'; +process.env.S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID || 'test'; +process.env.S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY || 'test'; +process.env.S3_PREFIX = process.env.S3_PREFIX || 'openreader'; diff --git a/compute/worker/tests/tsconfig.json b/compute/worker/tests/tsconfig.json new file mode 100644 index 0000000..f969c4c --- /dev/null +++ b/compute/worker/tests/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["./**/*.ts"], + "exclude": [] +} diff --git a/tests/unit/compute-worker-control-plane-jetstream.spec.ts b/compute/worker/tests/unit/jetstream-adapters.test.ts similarity index 55% rename from tests/unit/compute-worker-control-plane-jetstream.spec.ts rename to compute/worker/tests/unit/jetstream-adapters.test.ts index fb3f3e3..9f30ccc 100644 --- a/tests/unit/compute-worker-control-plane-jetstream.spec.ts +++ b/compute/worker/tests/unit/jetstream-adapters.test.ts @@ -1,16 +1,17 @@ -import { expect, test } from '@playwright/test'; -import { OperationOrchestrator } from '../../compute/core/src/control-plane'; -import type { WorkerOperationRequest } from '../../compute/core/src/api-contracts'; +import { describe, expect, test } from 'vitest'; +import { OperationOrchestrator } from '@openreader/compute-core/control-plane'; +import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, JetStreamOperationStateStore, + hashOpKey, opEventsSubject, opIndexKvKey, opStateKvKey, type KvEntryLike, type KvStoreLike, -} from '../../compute/worker/src/control-plane/jetstream'; +} from '../../src/control-plane/jetstream'; class FakeKvStore implements KvStoreLike { private readonly data = new Map(); @@ -18,46 +19,31 @@ class FakeKvStore implements KvStoreLike { async get(key: string): Promise { const value = this.data.get(key); - if (!value) return null; - return { - operation: value.operation, - value: value.value.slice(), - revision: value.revision, - }; + return value + ? { + operation: value.operation, + value: value.value.slice(), + revision: value.revision, + } + : null; } async put(key: string, data: Uint8Array): Promise { this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } async create(key: string, data: Uint8Array): Promise { - if (this.data.has(key)) { - throw new Error('key exists'); - } + if (this.data.has(key)) throw new Error('key exists'); this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } async update(key: string, data: Uint8Array, version: number): Promise { const current = this.data.get(key); - if (!current || current.revision !== version) { - throw new Error('wrong last sequence'); - } + if (!current || current.revision !== version) throw new Error('wrong last sequence'); this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } } @@ -72,7 +58,7 @@ class FakeJetStream { async publish(subject: string, data: Uint8Array): Promise<{ seq: number; stream: string; duplicate: boolean }> { this.seq += 1; - const payload = JSON.parse(new TextDecoder().decode(data)) as unknown; + const payload = JSON.parse(new TextDecoder().decode(data)); this.published.push({ subject, payload, seq: this.seq }); return { seq: this.seq, stream: 'fake', duplicate: false }; } @@ -90,97 +76,81 @@ function buildPdfRequest(opKey: string): WorkerOperationRequest { }; } -test.describe('worker jetstream control-plane adapters', () => { - test('state store compareAndSet handles create/update semantics', async () => { +describe('jetstream adapters', () => { + test('state store compareAndSet enforces create/update semantics', async () => { const kv = new FakeKvStore(); const store = new JetStreamOperationStateStore({ getKv: async () => kv }); - const created = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-1', - expectedOpId: null, - }); - const failedCreate = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: null, - }); - const wrongExpected = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: 'op-x', - }); - const updated = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: 'op-1', - }); + const created = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-1', expectedOpId: null }); + const duplicateCreate = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: null }); + const wrongExpected = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: 'op-x' }); + const updated = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: 'op-1' }); - expect(created).toBeTruthy(); - expect(failedCreate).toBeFalsy(); - expect(wrongExpected).toBeFalsy(); - expect(updated).toBeTruthy(); + expect(created).toBe(true); + expect(duplicateCreate).toBe(false); + expect(wrongExpected).toBe(false); + expect(updated).toBe(true); expect(await store.getOpIndex('k1')).toEqual({ opId: 'op-2' }); }); - test('queue and event adapters publish expected JetStream subjects', async () => { + test('queue routes layout jobs to the expected subject and publishes events', async () => { const js = new FakeJetStream(); const queue = new JetStreamOperationQueue({ - getJs: async () => js as any, + getJs: async () => js as never, whisperSubject: 'jobs.whisper', layoutSubject: 'jobs.layout', }); const events = new JetStreamOperationEventStream({ - getJs: async () => js as any, + getJs: async () => js as never, getJsm: async () => ({ consumers: { add: async () => ({ name: 'noop' }), delete: async () => true, }, - }) as any, + }) as never, eventsStreamName: 'compute_events', }); await queue.enqueue({ - jobId: 'j1', - opId: 'o1', - opKey: 'k1', + jobId: 'job-1', + opId: 'op-1', + opKey: 'k-1', kind: 'pdf_layout', queuedAt: 1000, payload: { documentId: 'd1', namespace: null, documentObjectKey: 'obj' }, }); - const appended = await events.append('o1', { - opId: 'o1', - opKey: 'k1', + const appended = await events.append('op-1', { + opId: 'op-1', + opKey: 'k-1', kind: 'pdf_layout', - jobId: 'j1', + jobId: 'job-1', status: 'queued', queuedAt: 1000, updatedAt: 1000, }); - expect(js.published.map((entry) => entry.subject)).toEqual(['jobs.layout', opEventsSubject('o1')]); + expect(js.published.map((entry) => entry.subject)).toEqual(['jobs.layout', opEventsSubject('op-1')]); expect(appended.eventId).toBe(2); }); - test('orchestrator integration writes index/state and reuses active op', async () => { + test('orchestrator writes expected index/state keys', async () => { const kv = new FakeKvStore(); const js = new FakeJetStream(); - const store = new JetStreamOperationStateStore({ getKv: async () => kv }); - const events = new JetStreamOperationEventStream({ - getJs: async () => js as any, + const stateStore = new JetStreamOperationStateStore({ getKv: async () => kv }); + const eventStream = new JetStreamOperationEventStream({ + getJs: async () => js as never, getJsm: async () => ({ consumers: { add: async () => ({ name: 'noop' }), delete: async () => true, }, - }) as any, + }) as never, eventsStreamName: 'compute_events', }); const queue = new JetStreamOperationQueue({ - getJs: async () => js as any, + getJs: async () => js as never, whisperSubject: 'jobs.whisper', layoutSubject: 'jobs.layout', }); @@ -189,8 +159,8 @@ test.describe('worker jetstream control-plane adapters', () => { let nextId = 1; const orchestrator = new OperationOrchestrator({ queue, - stateStore: store, - eventStream: events, + stateStore, + eventStream, config: { opStaleMs: 10_000, maxCasRetries: 3 }, clock: { now: () => now }, idFactory: { @@ -199,18 +169,18 @@ test.describe('worker jetstream control-plane adapters', () => { }, }); - const req = buildPdfRequest('k-integration'); - const first = await orchestrator.enqueueOrReuse(req); + const first = await orchestrator.enqueueOrReuse(buildPdfRequest('k-integration')); now = 2_000; - const reused = await orchestrator.enqueueOrReuse(req); + const reused = await orchestrator.enqueueOrReuse(buildPdfRequest('k-integration')); expect(first.opId).toBe('op-1'); expect(reused.opId).toBe('op-1'); const indexEntry = await kv.get(opIndexKvKey('k-integration')); const stateEntry = await kv.get(opStateKvKey('op-1')); + expect(indexEntry?.operation).toBe('PUT'); expect(stateEntry?.operation).toBe('PUT'); - expect(js.published.map((entry) => entry.subject)).toEqual(['ops.events.op-1', 'jobs.layout']); + expect(hashOpKey('k-integration')).toHaveLength(64); }); }); diff --git a/compute/worker/tests/unit/key-and-progress.test.ts b/compute/worker/tests/unit/key-and-progress.test.ts new file mode 100644 index 0000000..6887cad --- /dev/null +++ b/compute/worker/tests/unit/key-and-progress.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from 'vitest'; +import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/control-plane/jetstream'; +import { + buildInferProgressForPageParsed, + buildInferProgressForPageStart, +} from '../../src/pdf-progress'; + +describe('compute worker helpers', () => { + test('hash and kv key helpers are stable and deterministic', () => { + const hashA = hashOpKey('doc-123|layout|v1'); + const hashB = hashOpKey('doc-123|layout|v1'); + const hashC = hashOpKey('doc-123|layout|v2'); + + expect(hashA).toBe(hashB); + expect(hashA).toHaveLength(64); + expect(hashC).not.toBe(hashA); + + expect(opIndexKvKey('doc-123|layout|v1')).toBe(`op_index.${hashA}`); + expect(opStateKvKey('op-1')).toBe('op_state.op-1'); + }); + + test('progress helpers infer page counters correctly', () => { + expect(buildInferProgressForPageStart({ pageNumber: 1, totalPages: 12 })).toEqual({ + totalPages: 12, + pagesParsed: 0, + currentPage: 1, + phase: 'infer', + }); + + expect(buildInferProgressForPageParsed({ pageNumber: 5, totalPages: 12 })).toEqual({ + totalPages: 12, + pagesParsed: 5, + currentPage: 5, + phase: 'infer', + }); + }); +}); diff --git a/compute/worker/tests/unit/worker-loop-policy.test.ts b/compute/worker/tests/unit/worker-loop-policy.test.ts new file mode 100644 index 0000000..818e459 --- /dev/null +++ b/compute/worker/tests/unit/worker-loop-policy.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from 'vitest'; +import { buildQueueWaitTiming, decideRetryAction } from '../../src/worker-loop-policy'; + +describe('worker loop policy', () => { + test('returns queue wait timing with non-negative clamped duration', () => { + expect(buildQueueWaitTiming(1000, 1300)).toEqual({ queueWaitMs: 300 }); + expect(buildQueueWaitTiming(1500, 1300)).toEqual({ queueWaitMs: 0 }); + }); + + test('retry policy: layout jobs can retry until max attempts', () => { + expect(decideRetryAction({ kind: 'pdf_layout', deliveryCount: 1, pdfAttempts: 3 })).toBe('nak_retry'); + expect(decideRetryAction({ kind: 'pdf_layout', deliveryCount: 3, pdfAttempts: 3 })).toBe('term_fail'); + }); + + test('retry policy: whisper jobs terminate immediately by default', () => { + expect(decideRetryAction({ kind: 'whisper_align', deliveryCount: 1, pdfAttempts: 10 })).toBe('term_fail'); + }); +}); diff --git a/compute/worker/tsconfig.json b/compute/worker/tsconfig.json index 5912d1a..428e252 100644 --- a/compute/worker/tsconfig.json +++ b/compute/worker/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "noEmit": true + "noEmit": true, }, "include": ["src/**/*.ts"], "exclude": [] diff --git a/docs-site/docs/configure/admin-panel.md b/docs-site/docs/configure/admin-panel.md index 3d36571..f8d32df 100644 --- a/docs-site/docs/configure/admin-panel.md +++ b/docs-site/docs/configure/admin-panel.md @@ -90,6 +90,21 @@ Each row shows a source badge: Turning `restrictUserApiKeys` off allows user-supplied API keys to flow through this server. Use this only for trusted/self-hosted deployments where that tradeoff is acceptable. ::: +## Rate limiting + +A dedicated **Rate limiting** group (within the same admin panel) collects the daily quotas, the PDF parsing throttle, and the upload size cap: + +| Key | What it controls | +| --- | --- | +| `disableTtsRateLimit` | Disable the per-user/IP daily TTS character limits. When `false`, the daily-limit fields below it apply. | +| `disableComputeRateLimit` | Disable per-user PDF parsing rate limiting. When `false`, the burst/sustained limit fields below it apply. | +| `maxUploadMb` | Maximum size (MB) accepted for a single document upload. Enforced server-side and signed into the presigned S3 PUT. | + +The **Disable TTS daily rate limiting** and **Disable PDF parsing rate limiting** toggles each reveal a collapsible group of numeric inputs when set to `false`: + +- TTS: anonymous/authenticated per-user daily limits and anonymous/authenticated IP daily backstops. +- PDF parsing: burst limit + window (seconds) and sustained limit + window (seconds). The sustained window doubles as a concurrency cap. + ## Migrating off env vars The future-direction goal is to remove `RUNTIME_SEED_*` / `API_KEY` / `API_BASE` from your `.env` entirely. To do that safely: diff --git a/docs-site/docs/configure/tts-rate-limiting.md b/docs-site/docs/configure/tts-rate-limiting.md index ed507d1..fedf7f0 100644 --- a/docs-site/docs/configure/tts-rate-limiting.md +++ b/docs-site/docs/configure/tts-rate-limiting.md @@ -7,7 +7,8 @@ This page explains OpenReader's TTS character rate limiting controls. ## Overview - TTS rate limiting is disabled by default. -- To enable it, set `TTS_ENABLE_RATE_LIMIT=true`. +- Primary control is **Settings → Admin → Site features → Disable TTS daily rate limiting**. +- Optional first-boot seed: `RUNTIME_SEED_DISABLE_TTS_LIMIT=true`. - Limits are enforced per day in UTC. - Enforcement applies only when auth is enabled. @@ -28,24 +29,14 @@ If a request exceeds the active limit, the TTS API returns `429` with reset meta - `DISABLE_AUTH_RATE_LIMIT` only affects Better Auth's own request throttling. - `DISABLE_AUTH_RATE_LIMIT` does not disable TTS character limits. -## Environment variables +## Runtime config + seed var -Enable/disable: - -- `TTS_ENABLE_RATE_LIMIT` (default: `false`) - -Per-user daily limits: - -- `TTS_DAILY_LIMIT_ANONYMOUS` (default: `50000`) -- `TTS_DAILY_LIMIT_AUTHENTICATED` (default: `500000`) - -IP backstop daily limits: - -- `TTS_IP_DAILY_LIMIT_ANONYMOUS` (default: `100000`) -- `TTS_IP_DAILY_LIMIT_AUTHENTICATED` (default: `1000000`) +- First-boot seed toggle: `RUNTIME_SEED_DISABLE_TTS_LIMIT` (default: `true`) +- Per-user and IP backstop limit values are configured in **Settings → Admin → Site features** and stored in DB runtime settings. ## Related docs - TTS/rate-limit environment variables: [Environment Variables](../reference/environment-variables#tts-provider-and-request-behavior) +- PDF parsing rate limiting (separate, compute-side throttle): [Admin Panel → Site features](./admin-panel#site-features) and [Environment Variables](../reference/environment-variables#compute-pdf-parsing-rate-limiting-runtime-settings) - Auth configuration: [Auth](./auth) - Provider setup: [TTS Providers](./tts-providers) diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index b4a4078..2fc71f9 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -24,11 +24,6 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `TTS_RETRY_MAX_MS` | TTS retry | `2000` | Tune max retry delay | | `TTS_RETRY_BACKOFF` | TTS retry | `2` | Tune exponential backoff factor | | `TTS_UPSTREAM_TIMEOUT_MS` | TTS request timeout | `285000` | Set max upstream TTS request duration before fail-fast | -| `TTS_ENABLE_RATE_LIMIT` | Rate limiting | `false` | Set `true` to enable TTS per-user/IP daily character limits | -| `TTS_DAILY_LIMIT_ANONYMOUS` | Rate limiting | `50000` | Override anonymous per-user daily character limit | -| `TTS_DAILY_LIMIT_AUTHENTICATED` | Rate limiting | `500000` | Override authenticated per-user daily character limit | -| `TTS_IP_DAILY_LIMIT_ANONYMOUS` | Rate limiting | `100000` | Override anonymous IP backstop daily limit | -| `TTS_IP_DAILY_LIMIT_AUTHENTICATED` | Rate limiting | `1000000` | Override authenticated IP backstop daily limit | | `BASE_URL` | Auth | unset | Required (with `AUTH_SECRET`) to enable auth | | `AUTH_SECRET` | Auth | unset | Required (with `BASE_URL`) to enable auth | | `AUTH_TRUSTED_ORIGINS` | Auth | empty | Add extra allowed origins | @@ -36,6 +31,7 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `GITHUB_CLIENT_ID` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_SECRET` to enable GitHub sign-in | | `GITHUB_CLIENT_SECRET` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_ID` to enable GitHub sign-in | | `DISABLE_AUTH_RATE_LIMIT` | Rate limiting | `false` | Set `true` to disable auth-layer rate limiting | +| `ENABLE_TEST_NAMESPACE` | Testing/CI | unset | Honor the `x-openreader-test-namespace` header on a production build; leave unset on real deployments | | `ADMIN_EMAILS` | Auth/Admin | empty | Comma-separated emails auto-promoted to admin (requires auth enabled) | | `POSTGRES_URL` | Database | unset (SQLite mode) | Set to switch metadata/auth DB to Postgres | | `USE_EMBEDDED_WEED_MINI` | Storage | `true` when unset | Set `false` to use external S3-compatible storage only | @@ -48,8 +44,6 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `S3_ENDPOINT` | Storage | derived in embedded mode | Set for S3-compatible providers (MinIO/SeaweedFS/R2/etc.) | | `S3_FORCE_PATH_STYLE` | Storage | `true` in embedded mode | Set per provider requirement | | `S3_PREFIX` | Storage | `openreader` | Customize object key prefix | -| `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations | -| `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass | | `IMPORT_LIBRARY_DIR` | Library import | `docstore/library` fallback | Set a single server library root | | `IMPORT_LIBRARY_DIRS` | Library import | unset | Set multiple roots (comma/colon/semicolon separated) | | `COMPUTE_WORKER_URL` | Heavy compute backend | unset | Set only for standalone external compute worker; leave unset for embedded worker startup | @@ -76,6 +70,10 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `RUNTIME_SEED_RESTRICT_USER_API_KEYS` | Legacy bootstrap seed | runtime-dependent | Optional first-boot seed to restrict per-user BYOK | | `RUNTIME_SEED_DEFAULT_TTS_PROVIDER` | Legacy bootstrap seed | `custom-openai` | Optional first-boot seed for default TTS provider slug | | `RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT` | Legacy bootstrap seed | `true` | Optional first-boot seed to enable audiobook export UI | +| `RUNTIME_SEED_DISABLE_TTS_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps TTS daily rate limiting disabled | +| `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps PDF parsing rate limiting disabled (other compute-limit values + max upload size are admin-only) | +| `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations | +| `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass | @@ -158,37 +156,22 @@ Maximum upstream TTS request timeout in milliseconds. - Applies to outbound provider calls from server routes using shared TTS generation - Increase for slower providers/models; decrease to fail fast and surface retryable errors sooner -### TTS_ENABLE_RATE_LIMIT +### TTS Daily Rate Limiting (Runtime Settings) -Controls TTS character rate limiting in the TTS API. +TTS character rate limiting is now managed from **Settings → Admin → Site features**. -- Default: `false` (TTS char limits disabled) -- Set to `true` to enforce `TTS_DAILY_LIMIT_*` and `TTS_IP_DAILY_LIMIT_*` -- For behavior details and examples, see [TTS Rate Limiting](../configure/tts-rate-limiting) +- `disableTtsRateLimit` default: `true` (rate limiting disabled) +- Daily limit defaults: + - Anonymous per-user: `50000` + - Authenticated per-user: `500000` + - Anonymous IP backstop: `100000` + - Authenticated IP backstop: `1000000` -### TTS_DAILY_LIMIT_ANONYMOUS +Optional first-boot seeds: -Anonymous per-user daily character limit. +- `RUNTIME_SEED_DISABLE_TTS_LIMIT` -- Default: `50000` - -### TTS_DAILY_LIMIT_AUTHENTICATED - -Authenticated per-user daily character limit. - -- Default: `500000` - -### TTS_IP_DAILY_LIMIT_ANONYMOUS - -Anonymous IP backstop daily character limit. - -- Default: `100000` - -### TTS_IP_DAILY_LIMIT_AUTHENTICATED - -Authenticated IP backstop daily character limit. - -- Default: `1000000` +After first boot, these values are DB-backed admin runtime settings. ## Auth and Identity @@ -247,6 +230,15 @@ Controls Better Auth rate limiting. - This does not affect TTS character rate limiting - Related docs: [Auth](../configure/auth) +### ENABLE_TEST_NAMESPACE + +Honors the `x-openreader-test-namespace` request header, which scopes documents/storage into an isolated namespace for end-to-end tests. + +- Default: unset (header ignored on production builds) +- Non-production builds (`NODE_ENV !== 'production'`) honor the header without this flag. +- Production builds (`pnpm build && pnpm start`) honor it only when `ENABLE_TEST_NAMESPACE=true`. The Playwright web server sets this automatically. +- **Leave unset on real deployments.** It is test/CI scaffolding only. + ### ADMIN_EMAILS Comma-separated list of email addresses that are auto-promoted to admin. @@ -349,24 +341,13 @@ Prefix prepended to stored object keys. - Default: `openreader` - Related docs: [Object / Blob Storage](../configure/object-blob-storage) -## Migration Controls +### Maximum Upload Size (Runtime Settings) -### RUN_DRIZZLE_MIGRATIONS +The maximum size accepted for a single document upload is an admin runtime setting (no env var). -Controls startup migration execution in shared entrypoint. - -- Default: `true` -- Set `false` to skip automatic startup Drizzle schema migrations -- Related docs: [Migrations](../configure/migrations), [Database](../configure/database) - -### RUN_FS_MIGRATIONS - -Controls startup filesystem-to-object-store migration execution in shared entrypoint. - -- Default: `true` -- Runs `scripts/migrate-fs-v2.mjs` at startup after DB migrations -- Set `false` to skip automatic storage migration pass -- Related docs: [Migrations](../configure/migrations), [Database](../configure/database), [Object / Blob Storage](../configure/object-blob-storage) +- Runtime key: `maxUploadMb` (default: `200`) +- Configure in **Settings → Admin → Site features → Max upload size (MB)**. +- Enforced up front (`413`) and signed into the presigned S3 PUT so a client cannot stream more than it declared. ## Library Import @@ -511,6 +492,19 @@ Absolute path or executable name for the ffmpeg binary used by audiobook/process - Resolution order: `FFMPEG_BIN` -> `ffmpeg-static` - Example: `/var/task/node_modules/ffmpeg-static/ffmpeg` +### Compute (PDF Parsing) Rate Limiting (Runtime Settings) + +Per-user throttling of expensive PDF layout parsing is managed from **Settings → Admin → Site features**, not env vars. Enforcement applies only when auth is enabled. + +- `disableComputeRateLimit` default: `true` (rate limiting disabled, like the TTS limit; enable it in the admin panel) +- When enabled, the following admin-tunable sub-limits apply: + - `computeParseBurstMax` (default `8`) over `computeParseBurstWindowSec` (default `60`) + - `computeParseSustainedMax` (default `24`) over `computeParseSustainedWindowSec` (default `600`) +- The sustained window also acts as a concurrency cap: because the worker bounds each job's duration, the count of parses started in that window is an upper bound on those still running. +- Exceeding a limit returns `429` (`application/problem+json`) with `Retry-After`. + +Optional first-boot seed: `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT`. All other values are DB-backed admin runtime settings. + ## Legacy First-Boot Runtime Seeds (optional) These variables exist only as **first-boot seeds** for the admin-managed runtime config. Prefer changing site features from **Settings → Admin → Site features**. Keep these only when you need bootstrap defaults before the first admin login. See [Admin Panel](../configure/admin-panel) for migration behavior. @@ -583,3 +577,38 @@ Controls whether audiobook export UI/actions are shown in the client. - Default: `true` (enabled) - Affects export entry points in PDF/EPUB pages and document settings UI - Runtime key: `enableAudiobookExport` + +### RUNTIME_SEED_DISABLE_TTS_LIMIT + +Seeds the TTS daily character rate-limit on/off state on first boot. + +- Default: `true` (TTS rate limiting disabled) +- Runtime key: `disableTtsRateLimit` +- Per-user/IP daily limit values are admin-only runtime settings (see [TTS Daily Rate Limiting](#tts-daily-rate-limiting-runtime-settings)). + +### RUNTIME_SEED_DISABLE_COMPUTE_LIMIT + +Seeds the PDF parsing rate-limit on/off state on first boot. + +- Default: `true` (PDF parsing rate limiting disabled, matching `RUNTIME_SEED_DISABLE_TTS_LIMIT`) +- Runtime key: `disableComputeRateLimit` +- The burst/sustained limits, their windows, and the max upload size are admin-only runtime settings (see [Compute (PDF Parsing) Rate Limiting](#compute-pdf-parsing-rate-limiting-runtime-settings)). + +## Migration Controls + +### RUN_DRIZZLE_MIGRATIONS + +Controls startup migration execution in shared entrypoint. + +- Default: `true` +- Set `false` to skip automatic startup Drizzle schema migrations +- Related docs: [Migrations](../configure/migrations), [Database](../configure/database) + +### RUN_FS_MIGRATIONS + +Controls startup filesystem-to-object-store migration execution in shared entrypoint. + +- Default: `true` +- Runs `scripts/migrate-fs-v2.mjs` at startup after DB migrations +- Set `false` to skip automatic storage migration pass +- Related docs: [Migrations](../configure/migrations), [Database](../configure/database), [Object / Blob Storage](../configure/object-blob-storage) diff --git a/drizzle/postgres/0008_user_job_events.sql b/drizzle/postgres/0008_user_job_events.sql new file mode 100644 index 0000000..be33cc7 --- /dev/null +++ b/drizzle/postgres/0008_user_job_events.sql @@ -0,0 +1,9 @@ +CREATE TABLE "user_job_events" ( + "user_id" text NOT NULL, + "action" text NOT NULL, + "op_id" text NOT NULL, + "created_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint NOT NULL, + CONSTRAINT "user_job_events_user_id_action_op_id_pk" PRIMARY KEY("user_id","action","op_id") +); +--> statement-breakpoint +CREATE INDEX "idx_user_job_events_user_action_created" ON "user_job_events" USING btree ("user_id","action","created_at"); \ No newline at end of file diff --git a/drizzle/postgres/meta/0008_snapshot.json b/drizzle/postgres/meta/0008_snapshot.json new file mode 100644 index 0000000..03a4af9 --- /dev/null +++ b/drizzle/postgres/meta/0008_snapshot.json @@ -0,0 +1,1906 @@ +{ + "id": "0381962f-77d4-451e-b4ec-08b9a0dde70d", + "prevId": "68baf959-cd1d-418b-a04e-11397aa05c39", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admin_providers": { + "name": "admin_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_settings": { + "name": "admin_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value_json": { + "name": "value_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobook_chapters": { + "name": "audiobook_chapters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "name": "audiobook_chapters_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobooks": { + "name": "audiobooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "name": "audiobooks_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_previews": { + "name": "document_previews", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_until_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "name": "document_previews_document_id_namespace_variant_pk", + "columns": [ + "document_id", + "namespace", + "variant" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_settings": { + "name": "document_settings", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "name": "document_settings_document_id_user_id_pk", + "columns": [ + "document_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_modified": { + "name": "last_modified", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parse_state": { + "name": "parse_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parsed_json_key": { + "name": "parsed_json_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_modified", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "name": "documents_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_entries": { + "name": "tts_segment_entries", + "schema": "", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_version": { + "name": "document_version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_reader_rank", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_char_offset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_href", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_location", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "name": "tts_segment_entries_segment_entry_id_user_id_pk", + "columns": [ + "segment_entry_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_variants": { + "name": "tts_segment_variants", + "schema": "", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_json": { + "name": "settings_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "settings_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "name": "tts_segment_variants_segment_id_user_id_pk", + "columns": [ + "segment_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_document_progress": { + "name": "user_document_progress", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "name": "user_document_progress_user_id_document_id_pk", + "columns": [ + "user_id", + "document_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_job_events": { + "name": "user_job_events", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "name": "user_job_events_user_id_action_op_id_pk", + "columns": [ + "user_id", + "action", + "op_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preferences": { + "name": "user_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_tts_chars": { + "name": "user_tts_chars", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "name": "user_tts_chars_user_id_date_pk", + "columns": [ + "user_id", + "date" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index eccd0d6..6760c50 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1779378127846, "tag": "0007_pdf_parse_progress", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1780162695652, + "tag": "0008_user_job_events", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/sqlite/0008_user_job_events.sql b/drizzle/sqlite/0008_user_job_events.sql new file mode 100644 index 0000000..f3d8b8b --- /dev/null +++ b/drizzle/sqlite/0008_user_job_events.sql @@ -0,0 +1,9 @@ +CREATE TABLE `user_job_events` ( + `user_id` text NOT NULL, + `action` text NOT NULL, + `op_id` text NOT NULL, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + PRIMARY KEY(`user_id`, `action`, `op_id`) +); +--> statement-breakpoint +CREATE INDEX `idx_user_job_events_user_action_created` ON `user_job_events` (`user_id`,`action`,`created_at`); \ No newline at end of file diff --git a/drizzle/sqlite/meta/0008_snapshot.json b/drizzle/sqlite/meta/0008_snapshot.json new file mode 100644 index 0000000..b920031 --- /dev/null +++ b/drizzle/sqlite/meta/0008_snapshot.json @@ -0,0 +1,1751 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5e21c9cb-0d9b-48fe-92ec-9e98ac43c198", + "prevId": "031a62ae-47a7-4476-a577-7b14ff787eba", + "tables": { + "admin_providers": { + "name": "admin_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "admin_settings": { + "name": "admin_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobook_chapters": { + "name": "audiobook_chapters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobook_chapters_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobooks": { + "name": "audiobooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobooks_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_previews": { + "name": "document_previews", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + "status", + "lease_until_ms" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "columns": [ + "document_id", + "namespace", + "variant" + ], + "name": "document_previews_document_id_namespace_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_settings": { + "name": "document_settings", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "columns": [ + "document_id", + "user_id" + ], + "name": "document_settings_document_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parse_state": { + "name": "parse_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parsed_json_key": { + "name": "parsed_json_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + "user_id", + "last_modified" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "documents_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_entries": { + "name": "tts_segment_entries", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_version": { + "name": "document_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + "user_id", + "document_id", + "document_version", + "locator_reader_rank", + "locator_spine_index", + "locator_char_offset", + "locator_spine_href", + "locator_page", + "locator_location", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + "user_id", + "document_id", + "document_version", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + "user_id", + "document_id", + "document_version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "columns": [ + "segment_entry_id", + "user_id" + ], + "name": "tts_segment_entries_segment_entry_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_variants": { + "name": "tts_segment_variants", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + "user_id", + "segment_entry_id", + "updated_at" + ], + "isUnique": false + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + "user_id", + "status" + ], + "isUnique": false + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + "user_id", + "segment_entry_id", + "settings_hash" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "columns": [ + "segment_id", + "user_id" + ], + "name": "tts_segment_variants_segment_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_document_progress": { + "name": "user_document_progress", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + "user_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "columns": [ + "user_id", + "document_id" + ], + "name": "user_document_progress_user_id_document_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_job_events": { + "name": "user_job_events", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + "user_id", + "action", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "columns": [ + "user_id", + "action", + "op_id" + ], + "name": "user_job_events_user_id_action_op_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_preferences": { + "name": "user_preferences", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_tts_chars": { + "name": "user_tts_chars", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + "date" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "columns": [ + "user_id", + "date" + ], + "name": "user_tts_chars_user_id_date_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index 8db8d88..5db4a53 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1779378125905, "tag": "0007_pdf_parse_progress", "breakpoints": true + }, + { + "idx": 8, + "version": "6", + "when": 1780162695101, + "tag": "0008_user_job_events", + "breakpoints": true } ] } \ No newline at end of file diff --git a/package.json b/package.json index eb62569..a7aa4d5 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,9 @@ "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", + "test:e2e": "playwright test", + "test:unit": "vitest run", + "test:compute": "vitest run --project compute-core --project compute-worker", "test:ci-env": "CI=true playwright test", "migrate": "node drizzle/scripts/migrate.mjs", "migrate-fs": "node scripts/migrate-fs-v2.mjs", @@ -90,6 +93,7 @@ "eslint-config-next": "^15.5.18", "postcss": "^8.5.14", "tailwindcss": "^3.4.19", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^4.1.7" } } diff --git a/playwright.config.ts b/playwright.config.ts index a1f9f25..07b27bb 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -16,6 +16,7 @@ if (process.env.CI === 'true') { */ export default defineConfig({ testDir: './tests', + testIgnore: '**/unit/**', tsconfig: './tsconfig.json', timeout: 30 * 1000, outputDir: './tests/results', @@ -39,8 +40,10 @@ export default defineConfig({ /* Run your local dev server before starting the tests */ webServer: { - // Disable auth rate limiting for tests to support parallel workers creating sessions - command: `pnpm build && DISABLE_AUTH_RATE_LIMIT=true pnpm start`, + // Disable auth rate limiting for tests to support parallel workers creating sessions. + // ENABLE_TEST_NAMESPACE opts the production build into honoring the + // x-openreader-test-namespace header (ignored on real prod deployments). + command: `pnpm build && DISABLE_AUTH_RATE_LIMIT=true ENABLE_TEST_NAMESPACE=true pnpm start`, url: 'http://localhost:3003', reuseExistingServer: !process.env.CI, timeout: 120 * 1000, @@ -58,7 +61,6 @@ export default defineConfig({ { name: 'firefox', - testIgnore: '**/unit/**', use: { ...devices['Desktop Firefox'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'firefox' }, @@ -67,7 +69,6 @@ export default defineConfig({ { name: 'webkit', - testIgnore: '**/unit/**', use: { ...devices['Desktop Safari'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'webkit' }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f15c27b..9d3aedd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,7 +42,7 @@ importers: version: 7.0.1 better-auth: specifier: ^1.6.11 - version: 1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + version: 1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))) better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -194,6 +194,9 @@ importers: typescript: specifier: ^5.9.3 version: 5.9.3 + vitest: + specifier: ^4.1.7 + version: 4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) compute/core: dependencies: @@ -1569,6 +1572,12 @@ packages: '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@napi-rs/wasm-runtime@1.1.4': + resolution: {integrity: sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + '@nats-io/jetstream@3.4.0': resolution: {integrity: sha512-GzHQodNJ942+R5LRb8PuZ5ugVWVWMRiufxUYLLVWkXKfwDXYN+Owo0d7L/b9O7BPyrbYD7jQWAC6+ZVuXa9Gyw==} @@ -1679,6 +1688,9 @@ packages: resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} + '@oxc-project/types@0.132.0': + resolution: {integrity: sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -1717,6 +1729,104 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1 + '@rolldown/binding-android-arm64@1.0.2': + resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.0.2': + resolution: {integrity: sha512-vdFA9+C/rekyGce7WqHs/xoT0ioZEWaOFyZLIV1mEeNFaFDUQrPIo8Vs2GvJ6eetb3rzDUtUBgzto3ExpXJB3w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.0.2': + resolution: {integrity: sha512-BewSOwTHazv77DTYiAZXSqqKZ4KP/KonFisDMVU7PImxoWfB2aepnPhd2E4SWz3zDzYgDNbs6jBmTdgNnF02GA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.0.2': + resolution: {integrity: sha512-m41o7M0YWtUdqk61Tb+jnKb2rN++iRdIASlExkUoKfIAH30DOHCB8fVLzSUpbWHHU8esmEioY62PxzexE8MBuA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-jcojB9H7W/jS29pMKWAK1N+fU99vXodHDTatS3b3y/XSOCiHo0kkA74pL3jJmkoQtYpOCxDvaKs1fo2Ij/1X5w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-1jn6qDU5iiOgFgygDzKUuKP0maTi0/f1+sBLgvij/76C77Nm3ts6ufz9Bjg5q5dduxiUIxtq86JIoBvo1xQ4Ig==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-QVLO/czFMdoMFSqlX3bcswcJNm/23r+qoa/jgtmFc/qEp6/jXmIkDjF/XIo8dPfGaiwy1xfQn8o77L79GeXFgw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + resolution: {integrity: sha512-hgO5Abm0w5UL6FEa2iFnZqo2KlK7TQ5QhV5x09hujBf7t5KzHQ1VmfPuTpqRy/rNlSxua3eWH374xxiVrP+lcA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.0.2': + resolution: {integrity: sha512-fy8rXxuYEu602abC8MUNaPjYLIFzReOaEIEMKMUa0rFEUxNpVXhs15KSSQ4qlqSaM7B6rcj9rDZgADh/IGDzLQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-0+bOkiQ779+r1WpoHOWHqncvyySci0vKph+myNDYb+im6meJAzHQXay6oEgnkHuUGouM1LKTZwqKpBow6Kj7CQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-mjSkrzZK5Qsl0a9d1JgILOiuZOSDTVdKENcSXBoqbzSrspLR/4/IRVDo5wd2GgZjNss/viBFJdeq+j7qH2nypw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.0.2': + resolution: {integrity: sha512-1v5vHasdfQAZoEHakBV72LIFAC9JjnymsiKxp+GEr/ma3+NJCPSaYK+qavInOovJkgwFrs7GccX2d6IgDA3Z5w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.0.2': + resolution: {integrity: sha512-mb1VobWn6NheziTk5/WEaR6AKVbrwT5sOi6C7zk3gy/pD1qtJfU1j4PgTo2NJnOtbL9Dl3Aeei8w9jJ7qC2jZQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@rolldown/binding-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-SqKonF56vA/L2yHwHYcEp2P34URpOZ7d1fS635cTkpDnUtEGdUbhI6NzsPdqeSWvAAeGDrxjWjNmibDIdFf9/A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-v7qRI7gXLRINcOGXt+7YmAZ6iFuyZVMIoXAxhd8oP+DR9dLfL9GfNIx7PLMxmhZdvq8waUJBQiWN9EKNy+TRBQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} @@ -1960,9 +2070,15 @@ packages: '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/debug@4.1.13': resolution: {integrity: sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==} + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -2207,6 +2323,35 @@ packages: vue-router: optional: true + '@vitest/expect@4.1.7': + resolution: {integrity: sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==} + + '@vitest/mocker@4.1.7': + resolution: {integrity: sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.1.7': + resolution: {integrity: sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==} + + '@vitest/runner@4.1.7': + resolution: {integrity: sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==} + + '@vitest/snapshot@4.1.7': + resolution: {integrity: sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==} + + '@vitest/spy@4.1.7': + resolution: {integrity: sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==} + + '@vitest/utils@4.1.7': + resolution: {integrity: sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==} + '@xmldom/xmldom@0.9.10': resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} engines: {node: '>=14.6'} @@ -2327,6 +2472,10 @@ packages: resolution: {integrity: sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==} engines: {node: '>= 0.4'} + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} @@ -2569,6 +2718,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -2635,6 +2788,9 @@ packages: resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} engines: {'0': node >= 6.0} + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + cookie@1.1.1: resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} engines: {node: '>=18'} @@ -2909,6 +3065,9 @@ packages: resolution: {integrity: sha512-HVLACW1TppGYjJ8H6/jqH/pqOtKRw6wMlrB23xfExmFWxFquAIWCmwoLsOyN96K4a5KbmOf5At9ZUO3GZbetAw==} engines: {node: '>= 0.4'} + es-module-lexer@2.1.0: + resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} @@ -3087,6 +3246,9 @@ packages: estree-util-is-identifier-name@3.0.0: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} @@ -3109,6 +3271,10 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + ext@1.7.0: resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} @@ -3624,6 +3790,80 @@ packages: light-my-request@6.6.0: resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -3658,6 +3898,9 @@ packages: resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} engines: {node: 20 || >=22} + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + make-cancellable-promise@1.3.2: resolution: {integrity: sha512-GCXh3bq/WuMbS+Ky4JBPW1hYTOU+znU+Q5m9Pu+pI8EoUqIHk9+tviOKC6/qhHh8C4/As3tzJ69IF32kdz85ww==} @@ -3953,6 +4196,9 @@ packages: resolution: {integrity: sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==} engines: {node: '>= 0.4'} + obug@2.1.1: + resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + on-exit-leak-free@2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} @@ -4037,6 +4283,9 @@ packages: resolution: {integrity: sha512-+vnG6S4dYcYxZd+CZxzXCNKdELYZSKfohrk98yajCo1PtRoDgCTrrwOvK1GT0UoAdVszagDVllQc0U1vaX4NUQ==} engines: {node: '>=6'} + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdfjs-dist@4.8.69: resolution: {integrity: sha512-IHZsA4T7YElCKNNXtiLgqScw4zPd3pG9do8UrznC757gMd7UPeHSL2qwNNMJo4r79fl8oj1Xx+1nh2YkzdMpLQ==} engines: {node: '>=18'} @@ -4177,6 +4426,10 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} + postcss@8.5.15: + resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} + engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} @@ -4411,6 +4664,11 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + rolldown@1.0.2: + resolution: {integrity: sha512-oZx5zVDtVB44AW3eaifgDml1gWRDZGvjcfdxonE4swNPG98PrrXjaO/KrnUjzlMnztCCRVlUueA1kCXhARGk6g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + rou3@0.7.12: resolution: {integrity: sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg==} @@ -4511,6 +4769,9 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -4545,6 +4806,12 @@ packages: stable-hash@0.0.5: resolution: {integrity: sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA==} + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@4.1.0: + resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + stop-iteration-iterator@1.1.0: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} @@ -4692,10 +4959,21 @@ packages: tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@1.2.3: + resolution: {integrity: sha512-g62dB+w1/OEFnPvmX0yd/HnetYITOL+1nJW7kitOycOeAvmbWC/nu0fwmmQ/kupNojqExzyC/T++pST/jRJ2mQ==} + engines: {node: '>=18'} + tinyglobby@0.2.16: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} + engines: {node: '>=14.0.0'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -4825,6 +5103,90 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite@8.0.14: + resolution: {integrity: sha512-s4BJJ+5y1pYL6Otw51FHhVJQhPnuRinKig64g/1+EUNaJsd3gCKdD31IPFvswUgW9/60QT9oFHbZHbQK5imcxw==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.1.18 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.1.7: + resolution: {integrity: sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.7 + '@vitest/browser-preview': 4.1.7 + '@vitest/browser-webdriverio': 4.1.7 + '@vitest/coverage-istanbul': 4.1.7 + '@vitest/coverage-v8': 4.1.7 + '@vitest/ui': 4.1.7 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + warning@4.0.3: resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} @@ -4849,6 +5211,11 @@ packages: engines: {node: '>= 8'} hasBin: true + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} @@ -6248,6 +6615,13 @@ snapshots: '@tybys/wasm-util': 0.10.2 optional: true + '@napi-rs/wasm-runtime@1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@tybys/wasm-util': 0.10.2 + optional: true + '@nats-io/jetstream@3.4.0': dependencies: '@nats-io/nats-core': 3.4.0 @@ -6326,6 +6700,8 @@ snapshots: '@opentelemetry/semantic-conventions@1.41.1': {} + '@oxc-project/types@0.132.0': {} + '@pinojs/redact@0.4.0': {} '@pkgjs/parseargs@0.11.0': @@ -6360,6 +6736,57 @@ snapshots: dependencies: react: 19.2.6 + '@rolldown/binding-android-arm64@1.0.2': + optional: true + + '@rolldown/binding-darwin-arm64@1.0.2': + optional: true + + '@rolldown/binding-darwin-x64@1.0.2': + optional: true + + '@rolldown/binding-freebsd-x64@1.0.2': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.0.2': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-x64-musl@1.0.2': + optional: true + + '@rolldown/binding-openharmony-arm64@1.0.2': + optional: true + + '@rolldown/binding-wasm32-wasi@1.0.2': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.0.2': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.0.2': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + '@rtsao/scc@1.1.0': {} '@rushstack/eslint-patch@1.16.1': {} @@ -6660,10 +7087,17 @@ snapshots: dependencies: '@types/node': 20.19.41 + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + '@types/debug@4.1.13': dependencies: '@types/ms': 2.1.0 + '@types/deep-eql@4.0.2': {} + '@types/estree-jsx@1.0.5': dependencies: '@types/estree': 1.0.9 @@ -6873,6 +7307,47 @@ snapshots: next: 15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 + '@vitest/expect@4.1.7': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 + chai: 6.2.2 + tinyrainbow: 3.1.0 + + '@vitest/mocker@4.1.7(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))': + dependencies: + '@vitest/spy': 4.1.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3) + + '@vitest/pretty-format@4.1.7': + dependencies: + tinyrainbow: 3.1.0 + + '@vitest/runner@4.1.7': + dependencies: + '@vitest/utils': 4.1.7 + pathe: 2.0.3 + + '@vitest/snapshot@4.1.7': + dependencies: + '@vitest/pretty-format': 4.1.7 + '@vitest/utils': 4.1.7 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.1.7': {} + + '@vitest/utils@4.1.7': + dependencies: + '@vitest/pretty-format': 4.1.7 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + '@xmldom/xmldom@0.9.10': {} abort-controller@3.0.0: @@ -7031,6 +7506,8 @@ snapshots: get-intrinsic: 1.3.0 is-array-buffer: 3.0.5 + assertion-error@2.0.1: {} + ast-types-flow@0.0.8: {} async-function@1.0.0: {} @@ -7096,7 +7573,7 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6): + better-auth@1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))): dependencies: '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0)) @@ -7123,6 +7600,7 @@ snapshots: pg: 8.20.0 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) + vitest: 4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) transitivePeerDependencies: - '@cloudflare/workers-types' - '@opentelemetry/api' @@ -7219,6 +7697,8 @@ snapshots: ccount@2.0.1: {} + chai@6.2.2: {} + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -7287,6 +7767,8 @@ snapshots: readable-stream: 3.6.2 typedarray: 0.0.6 + convert-source-map@2.0.0: {} + cookie@1.1.1: {} core-js@3.49.0: {} @@ -7531,6 +8013,8 @@ snapshots: iterator.prototype: 1.1.5 math-intrinsics: 1.1.0 + es-module-lexer@2.1.0: {} + es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 @@ -7891,6 +8375,10 @@ snapshots: estree-util-is-identifier-name@3.0.0: {} + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + esutils@2.0.3: {} event-emitter@0.3.5: @@ -7910,6 +8398,8 @@ snapshots: expand-template@2.0.3: {} + expect-type@1.3.0: {} + ext@1.7.0: dependencies: type: 2.7.3 @@ -8485,6 +8975,55 @@ snapshots: process-warning: 4.0.1 set-cookie-parser: 2.7.2 + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} @@ -8511,6 +9050,10 @@ snapshots: lru-cache@11.3.6: {} + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + make-cancellable-promise@1.3.2: {} make-event-props@1.6.2: {} @@ -9008,6 +9551,8 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 + obug@2.1.1: {} + on-exit-leak-free@2.1.2: {} once@1.4.0: @@ -9087,6 +9632,8 @@ snapshots: path2d@0.2.2: optional: true + pathe@2.0.3: {} + pdfjs-dist@4.8.69: optionalDependencies: canvas: 3.2.3 @@ -9232,6 +9779,12 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postcss@8.5.15: + dependencies: + nanoid: 3.3.12 + picocolors: 1.1.1 + source-map-js: 1.2.1 + postgres-array@2.0.0: {} postgres-bytea@1.0.1: {} @@ -9537,6 +10090,27 @@ snapshots: rfdc@1.4.1: {} + rolldown@1.0.2: + dependencies: + '@oxc-project/types': 0.132.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.0.2 + '@rolldown/binding-darwin-arm64': 1.0.2 + '@rolldown/binding-darwin-x64': 1.0.2 + '@rolldown/binding-freebsd-x64': 1.0.2 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.2 + '@rolldown/binding-linux-arm64-gnu': 1.0.2 + '@rolldown/binding-linux-arm64-musl': 1.0.2 + '@rolldown/binding-linux-ppc64-gnu': 1.0.2 + '@rolldown/binding-linux-s390x-gnu': 1.0.2 + '@rolldown/binding-linux-x64-gnu': 1.0.2 + '@rolldown/binding-linux-x64-musl': 1.0.2 + '@rolldown/binding-openharmony-arm64': 1.0.2 + '@rolldown/binding-wasm32-wasi': 1.0.2 + '@rolldown/binding-win32-arm64-msvc': 1.0.2 + '@rolldown/binding-win32-x64-msvc': 1.0.2 + rou3@0.7.12: {} run-parallel@1.2.0: @@ -9678,6 +10252,8 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} + signal-exit@4.1.0: {} simple-concat@1.0.1: {} @@ -9707,6 +10283,10 @@ snapshots: stable-hash@0.0.5: {} + stackback@0.0.2: {} + + std-env@4.1.0: {} + stop-iteration-iterator@1.1.0: dependencies: es-errors: 1.3.0 @@ -9928,11 +10508,17 @@ snapshots: tiny-invariant@1.3.3: {} + tinybench@2.9.0: {} + + tinyexec@1.2.3: {} + tinyglobby@0.2.16: dependencies: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 + tinyrainbow@3.1.0: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -10110,6 +10696,47 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 + vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.4 + postcss: 8.5.15 + rolldown: 1.0.2 + tinyglobby: 0.2.16 + optionalDependencies: + '@types/node': 20.19.41 + esbuild: 0.28.0 + fsevents: 2.3.3 + jiti: 1.21.7 + tsx: 4.22.3 + + vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)): + dependencies: + '@vitest/expect': 4.1.7 + '@vitest/mocker': 4.1.7(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) + '@vitest/pretty-format': 4.1.7 + '@vitest/runner': 4.1.7 + '@vitest/snapshot': 4.1.7 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.2.3 + tinyglobby: 0.2.16 + tinyrainbow: 3.1.0 + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.19.41 + transitivePeerDependencies: + - msw + warning@4.0.3: dependencies: loose-envify: 1.4.0 @@ -10159,6 +10786,11 @@ snapshots: dependencies: isexe: 2.0.0 + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + word-wrap@1.2.5: {} wrap-ansi@7.0.0: diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index 57f4c0b..19e1806 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -243,7 +243,7 @@ export default function EPUBPage() { )} {isAtLimit ? (
-
+
diff --git a/src/app/(app)/html/[id]/page.tsx b/src/app/(app)/html/[id]/page.tsx index 21ad5c6..90fa49a 100644 --- a/src/app/(app)/html/[id]/page.tsx +++ b/src/app/(app)/html/[id]/page.tsx @@ -234,7 +234,7 @@ export default function HTMLPage() { )} {isAtLimit ? (
-
+
diff --git a/src/app/(app)/pdf/[id]/page.tsx b/src/app/(app)/pdf/[id]/page.tsx index 7695b4b..7635833 100644 --- a/src/app/(app)/pdf/[id]/page.tsx +++ b/src/app/(app)/pdf/[id]/page.tsx @@ -441,7 +441,7 @@ export default function PDFViewerPage() { )} {isAtLimit ? (
-
+
diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index c2a2f27..f0875de 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -8,7 +8,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { audiobooks, audiobookChapters } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -532,7 +532,15 @@ export async function POST(request: NextRequest) { sharedDefaultInstructions: credResolved.adminRecord?.defaultInstructions, }); - if (authEnabled && userId && isTtsRateLimitEnabled()) { + const ttsRateLimitEnabled = !runtimeConfig.disableTtsRateLimit; + const limits = resolveRateLimitThresholds({ + anonymous: runtimeConfig.ttsDailyLimitAnonymous, + authenticated: runtimeConfig.ttsDailyLimitAuthenticated, + ipAnonymous: runtimeConfig.ttsIpDailyLimitAnonymous, + ipAuthenticated: runtimeConfig.ttsIpDailyLimitAuthenticated, + }); + + if (authEnabled && userId && ttsRateLimitEnabled) { const isAnonymous = Boolean(user?.isAnonymous); const charCount = data.text.length; const ip = getClientIp(request); @@ -549,6 +557,10 @@ export async function POST(request: NextRequest) { deviceId: device?.deviceId ?? null, ip, }, + { + enabled: ttsRateLimitEnabled, + limits, + }, ); if (!rateLimitResult.allowed) { @@ -570,7 +582,7 @@ export async function POST(request: NextRequest) { resetTimeMs, userType: isAnonymous ? 'anonymous' : 'authenticated', upgradeHint: isAnonymous - ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + ? `Sign up to increase your limit from ${formatLimitForHint(limits.anonymous)} to ${formatLimitForHint(limits.authenticated)} characters per day` : undefined, instance: request.nextUrl.pathname, }; diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 51ef630..3ca677f 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -22,6 +22,9 @@ import { } from '@/lib/server/documents/parse-state'; import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; +import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; @@ -263,11 +266,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } if (effectiveStatus === 'failed' && retryFailed) { + const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig); + if (!rateDecision.allowed) { + return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); + } const created = await createOrReusePdfWorkerOperation({ documentId: id, namespace: testNamespace, documentObjectKey: documentKey(id, testNamespace), }); + await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); const snapshot = snapshotFromWorkerState(created); return NextResponse.json({ parseStatus: snapshot.parseStatus, @@ -385,12 +394,19 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string } } + const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig); + if (!rateDecision.allowed) { + return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); + } + const created = await createOrReusePdfWorkerOperation({ documentId: id, namespace: testNamespace, documentObjectKey: documentKey(id, testNamespace), forceToken: randomUUID(), }); + await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); const snapshot = snapshotFromWorkerState(created); diff --git a/src/app/api/documents/blob/upload/fallback/route.ts b/src/app/api/documents/blob/upload/fallback/route.ts index bc188df..7fc8208 100644 --- a/src/app/api/documents/blob/upload/fallback/route.ts +++ b/src/app/api/documents/blob/upload/fallback/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -33,7 +34,55 @@ export async function PUT(req: NextRequest) { } const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream'; - const body = Buffer.from(await req.arrayBuffer()); + + const { maxUploadMb } = await getResolvedRuntimeConfig(); + const maxUploadBytes = maxUploadMb * 1024 * 1024; + // Reject before buffering when the declared length is already over the cap. + const declaredLength = Number(req.headers.get('content-length') || ''); + if (Number.isFinite(declaredLength) && declaredLength > maxUploadBytes) { + return NextResponse.json( + { error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, maxBytes: maxUploadBytes }, + { status: 413 }, + ); + } + + // Backstop for chunked/omitted Content-Length: stream the body so we can + // bail out as soon as the running total crosses the cap instead of + // buffering the entire payload first. + const stream = req.body; + if (!stream) { + return NextResponse.json({ error: 'Missing request body' }, { status: 400 }); + } + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + let overLimit = false; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + totalBytes += value.byteLength; + if (totalBytes > maxUploadBytes) { + overLimit = true; + break; + } + chunks.push(value); + } + } finally { + if (overLimit) { + await reader.cancel().catch(() => {}); + } + reader.releaseLock(); + } + if (overLimit) { + return NextResponse.json( + { error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, maxBytes: maxUploadBytes }, + { status: 413 }, + ); + } + const body = Buffer.concat(chunks, totalBytes); + const namespace = getOpenReaderTestNamespace(req.headers); try { diff --git a/src/app/api/documents/blob/upload/presign/route.ts b/src/app/api/documents/blob/upload/presign/route.ts index 07756aa..6d91ee4 100644 --- a/src/app/api/documents/blob/upload/presign/route.ts +++ b/src/app/api/documents/blob/upload/presign/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -53,10 +54,25 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 }); } + const { maxUploadMb } = await getResolvedRuntimeConfig(); + const maxUploadBytes = maxUploadMb * 1024 * 1024; + const oversized = uploads.find((upload) => upload.size > maxUploadBytes); + if (oversized) { + return NextResponse.json( + { + error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, + maxBytes: maxUploadBytes, + }, + { status: 413 }, + ); + } + const namespace = getOpenReaderTestNamespace(req.headers); const signed = await Promise.all( uploads.map(async (upload) => { - const res = await presignPut(upload.id, upload.contentType, namespace); + const res = await presignPut(upload.id, upload.contentType, namespace, { + contentLength: upload.size, + }); return { id: upload.id, url: res.url, diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 91866af..c849adf 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -11,6 +11,8 @@ import { documents } from '@/db/schema'; import { safeDocumentName } from '@/lib/server/documents/utils'; import { enqueueDocumentPreview } from '@/lib/server/documents/previews'; import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; +import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; @@ -173,6 +175,9 @@ export async function POST(req: NextRequest) { }, 'Failed to enqueue preview for converted DOCX'); }); + // Record upload-driven parse load (see register route for rationale). + const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig); enqueueParsePdfJob({ documentId: id, userId: storageUserId, diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index c56891c..880b5c2 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { and, count, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; @@ -12,6 +13,8 @@ import { enqueueDocumentPreview, } from '@/lib/server/documents/previews'; import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; +import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; import { normalizeParseStatus, @@ -90,6 +93,11 @@ export async function POST(req: NextRequest) { const stored: BaseDocument[] = []; + // Resolve the parse rate-limit config once (only when a PDF is present). + const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf') + ? getPdfLayoutRateConfig(await getResolvedRuntimeConfig()) + : null; + for (const doc of documentsData) { let headSize = doc.size; @@ -184,6 +192,11 @@ export async function POST(req: NextRequest) { }); if (doc.type === 'pdf') { + // Account for upload-driven parse load in the same ledger the explicit + // re-parse limiter reads. We record (not reject) here so a legitimate + // bulk upload always parses; the recorded load still throttles + // subsequent loopable re-parse spam via /parsed. + await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] }); enqueueParsePdfJob({ documentId: doc.id, userId: storageUserId, diff --git a/src/app/api/rate-limit/status/route.ts b/src/app/api/rate-limit/status/route.ts index e541f9f..f21d8dc 100644 --- a/src/app/api/rate-limit/status/route.ts +++ b/src/app/api/rate-limit/status/route.ts @@ -1,11 +1,12 @@ import { NextResponse, type NextRequest } from 'next/server'; import { auth } from '@/lib/server/auth/auth'; -import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { headers } from 'next/headers'; import { isAuthEnabled } from '@/lib/server/auth/config'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; import { nextUtcMidnightTimestampMs } from '@/lib/shared/timestamps'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; @@ -17,7 +18,14 @@ function getUtcResetTimeMs(): number { export async function GET(req: NextRequest) { try { - const ttsRateLimitEnabled = isTtsRateLimitEnabled(); + const runtimeConfig = await getResolvedRuntimeConfig(); + const limits = resolveRateLimitThresholds({ + anonymous: runtimeConfig.ttsDailyLimitAnonymous, + authenticated: runtimeConfig.ttsDailyLimitAuthenticated, + ipAnonymous: runtimeConfig.ttsIpDailyLimitAnonymous, + ipAuthenticated: runtimeConfig.ttsIpDailyLimitAuthenticated, + }); + const ttsRateLimitEnabled = !runtimeConfig.disableTtsRateLimit; // If auth is not enabled, return unlimited status if (!isAuthEnabled() || !auth) { @@ -46,8 +54,8 @@ export async function GET(req: NextRequest) { return NextResponse.json({ allowed: true, currentCount: 0, - limit: ttsRateLimitEnabled ? RATE_LIMITS.ANONYMOUS : Number.MAX_SAFE_INTEGER, - remainingChars: ttsRateLimitEnabled ? RATE_LIMITS.ANONYMOUS : Number.MAX_SAFE_INTEGER, + limit: ttsRateLimitEnabled ? limits.anonymous : Number.MAX_SAFE_INTEGER, + remainingChars: ttsRateLimitEnabled ? limits.anonymous : Number.MAX_SAFE_INTEGER, resetTimeMs, userType: 'unauthenticated', authEnabled: true @@ -57,7 +65,7 @@ export async function GET(req: NextRequest) { const isAnonymous = Boolean((session.user as { isAnonymous?: boolean }).isAnonymous); const ip = getClientIp(req); - const device = isTtsRateLimitEnabled() ? (isAnonymous ? getOrCreateDeviceId(req) : null) : null; + const device = ttsRateLimitEnabled ? (isAnonymous ? getOrCreateDeviceId(req) : null) : null; const result = await rateLimiter.getCurrentUsage( { @@ -67,7 +75,11 @@ export async function GET(req: NextRequest) { { deviceId: device?.deviceId ?? null, ip, - } + }, + { + enabled: ttsRateLimitEnabled, + limits, + }, ); const response = NextResponse.json({ diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index a3dc204..80178fb 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -23,7 +23,7 @@ import { } from '@/lib/server/tts/segments'; import { isBuiltInTtsProviderId, isTtsProviderType } from '@/lib/shared/tts-provider-catalog'; import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; -import { rateLimiter, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { resolveTtsCredentials } from '@/lib/server/admin/resolve-credentials'; import { resolveEffectiveTtsInstructions } from '@/lib/server/admin/tts-instructions'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; @@ -169,6 +169,13 @@ export async function POST(request: NextRequest) { const scope = await resolveSegmentDocumentScope(request, parsed.documentId); if (scope instanceof Response) return scope; const runtimeConfig = await getResolvedRuntimeConfig(); + const ttsRateLimitEnabled = !runtimeConfig.disableTtsRateLimit; + const limits = resolveRateLimitThresholds({ + anonymous: runtimeConfig.ttsDailyLimitAnonymous, + authenticated: runtimeConfig.ttsDailyLimitAuthenticated, + ipAnonymous: runtimeConfig.ttsIpDailyLimitAnonymous, + ipAuthenticated: runtimeConfig.ttsIpDailyLimitAuthenticated, + }); const requestCreds = await resolveTtsCredentials({ providerHeader: parsed.settings.providerRef, apiKeyHeader: request.headers.get('x-openai-key'), @@ -461,7 +468,7 @@ export async function POST(request: NextRequest) { segmentId: segment.segmentId, }); - if (scope.authEnabled && scope.userId && isTtsRateLimitEnabled()) { + if (scope.authEnabled && scope.userId && ttsRateLimitEnabled) { const charCount = segment.text.length; const ip = getClientIp(request); const device = scope.isAnonymousUser ? getOrCreateDeviceId(request) : null; @@ -477,6 +484,10 @@ export async function POST(request: NextRequest) { deviceId: device?.deviceId ?? null, ip, }, + { + enabled: ttsRateLimitEnabled, + limits, + }, ); if (!rateLimitResult.allowed) { @@ -484,6 +495,8 @@ export async function POST(request: NextRequest) { rateLimitResult, isAnonymousUser: scope.isAnonymousUser, pathname: request.nextUrl.pathname, + anonymousLimit: limits.anonymous, + authenticatedLimit: limits.authenticated, }); attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); return response; diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 1315dbe..e2dcdad 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -1,6 +1,6 @@ import "./globals.css"; import { ReactNode } from "react"; -import { Metadata } from "next"; +import { Metadata, Viewport } from "next"; import { Figtree } from "next/font/google"; import { ConsentAwareAnalytics } from "@/components/ConsentAwareAnalytics"; import { CookieConsentBanner } from "@/components/CookieConsentBanner"; @@ -30,6 +30,10 @@ const themeInitScript = ` })(); `; +export const viewport: Viewport = { + viewportFit: "cover", +}; + export const metadata: Metadata = { title: { default: "OpenReader", diff --git a/src/components/CookieConsentBanner.tsx b/src/components/CookieConsentBanner.tsx index c2c4608..97e5126 100644 --- a/src/components/CookieConsentBanner.tsx +++ b/src/components/CookieConsentBanner.tsx @@ -41,7 +41,7 @@ export function CookieConsentBanner() { leave="transition ease-in duration-200 transform" leaveFrom="translate-y-0 opacity-100" leaveTo="translate-y-full opacity-0" - className="fixed bottom-0 left-0 right-0 z-[60] p-4 md:p-6" + className="fixed bottom-0 left-0 right-0 z-[60] px-4 pt-4 pb-[max(1rem,env(safe-area-inset-bottom))] md:px-6 md:pt-6 md:pb-[max(1.5rem,env(safe-area-inset-bottom))]" >
diff --git a/src/components/HomeContent.tsx b/src/components/HomeContent.tsx index 8e9240f..e548230 100644 --- a/src/components/HomeContent.tsx +++ b/src/components/HomeContent.tsx @@ -1,7 +1,8 @@ 'use client'; +import { useState } from 'react'; import { DocumentList } from '@/components/doclist/DocumentList'; -import { SettingsModal } from '@/components/SettingsModal'; +import { SettingsModal, SettingsTrigger } from '@/components/SettingsModal'; import { UserMenu } from '@/components/auth/UserMenu'; const Brand = () => ( @@ -14,20 +15,24 @@ const Brand = () => (
); -const AppActions = () => ( -
- - -
-); - export function HomeContent() { + const [settingsOpen, setSettingsOpen] = useState(false); + + const appActions = ( +
+ setSettingsOpen(true)} + /> + +
+ ); + return (
- } appActions={} /> + } appActions={appActions} /> +
); } diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index 50fee42..330f452 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -130,19 +130,42 @@ const SIDEBAR_SECTIONS: SidebarSection[] = [ type AdminSubTab = 'providers' | 'features'; -export function SettingsModal({ +export function SettingsTrigger({ className = '', triggerLabel, + onOpen, }: { className?: string; triggerLabel?: string; + onOpen: () => void; +}) { + return ( + + ); +} + +export function SettingsModal({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; }) { const runtimeConfig = useRuntimeConfig(); const enableDestructiveDelete = runtimeConfig.enableDestructiveDeleteActions; const showAllProviderModels = runtimeConfig.showAllProviderModels; const enableTTSProvidersTab = runtimeConfig.enableTtsProvidersTab; const restrictUserApiKeys = runtimeConfig.restrictUserApiKeys; - const [isOpen, setIsOpen] = useState(false); + const isOpen = open; + const setIsOpen = onOpenChange; const [isChangelogOpen, setIsChangelogOpen] = useState(false); const [activeSection, setActiveSection] = useState(enableTTSProvidersTab ? 'api' : 'theme'); @@ -215,7 +238,7 @@ export function SettingsModal({ if (changelogOpenSignal <= 0) return; setIsOpen(true); setIsChangelogOpen(true); - }, [changelogOpenSignal]); + }, [changelogOpenSignal, setIsOpen]); useEffect(() => { setLocalApiKey(apiKey); @@ -409,7 +432,7 @@ export function SettingsModal({ } else { setCustomModelInput(''); } - }, [apiKey, baseUrl, providerRef, providerType, ttsModel, ttsInstructions, ttsModels]); + }, [apiKey, baseUrl, providerRef, providerType, ttsModel, ttsInstructions, ttsModels, setIsOpen]); const [systemIsDark, setSystemIsDark] = useState( typeof window !== 'undefined' && window.matchMedia('(prefers-color-scheme: dark)').matches @@ -489,19 +512,6 @@ export function SettingsModal({ return ( <> - - ({ ...d, [key]: value })); setDirty((s) => { const next = new Set(s); - next.add(key); + const baselineValue = data?.values?.[key]; + if (Object.is(value, baselineValue)) next.delete(key); + else next.add(key); return next; }); }; + const updatePositiveIntDraft = (key: string, raw: string) => { + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return; + updateDraft(key, Math.max(1, Math.floor(parsed))); + }; + const resetField = (key: string) => { if (saving) return; resetMutation.mutate(key); @@ -151,6 +159,8 @@ export function AdminFeaturesPanel() { } as ProviderOption : fallbackShared; const selectedProviderOption = effectiveSelectedProvider; + const shouldRenderRateLimitInputs = draft.disableTtsRateLimit === false; + const shouldRenderComputeRateLimitInputs = draft.disableComputeRateLimit === false; const handleProviderChange = (opt: ProviderOption) => { updateDraft('defaultTtsProvider', opt.id); @@ -272,6 +282,173 @@ export function AdminFeaturesPanel() { /> +
Limits} + > + updateDraft('disableTtsRateLimit', checked)} + right={renderSource('disableTtsRateLimit')} + variant="flat" + /> + {shouldRenderRateLimitInputs ? ( +
+
+
+ + {renderSource('ttsDailyLimitAnonymous')} +
+ updatePositiveIntDraft('ttsDailyLimitAnonymous', event.target.value)} + /> +
+
+
+ + {renderSource('ttsDailyLimitAuthenticated')} +
+ updatePositiveIntDraft('ttsDailyLimitAuthenticated', event.target.value)} + /> +
+
+
+ + {renderSource('ttsIpDailyLimitAnonymous')} +
+ updatePositiveIntDraft('ttsIpDailyLimitAnonymous', event.target.value)} + /> +
+
+
+ + {renderSource('ttsIpDailyLimitAuthenticated')} +
+ updatePositiveIntDraft('ttsIpDailyLimitAuthenticated', event.target.value)} + /> +
+
+ ) : null} + + updateDraft('disableComputeRateLimit', checked)} + right={renderSource('disableComputeRateLimit')} + variant="flat" + /> + {shouldRenderComputeRateLimitInputs ? ( +
+
+
+ + {renderSource('computeParseBurstMax')} +
+ updatePositiveIntDraft('computeParseBurstMax', event.target.value)} + /> +
+
+
+ + {renderSource('computeParseBurstWindowSec')} +
+ updatePositiveIntDraft('computeParseBurstWindowSec', event.target.value)} + /> +
+
+
+ + {renderSource('computeParseSustainedMax')} +
+ updatePositiveIntDraft('computeParseSustainedMax', event.target.value)} + /> +
+
+
+ + {renderSource('computeParseSustainedWindowSec')} +
+ updatePositiveIntDraft('computeParseSustainedWindowSec', event.target.value)} + /> +
+
+ ) : null} + +
+
+
+ Max upload size + Largest single document upload accepted. +
+
{renderSource('maxUploadMb')}
+
+ updatePositiveIntDraft('maxUploadMb', event.target.value)} + /> + MB +
+
+
+
+
{ + const res = await fetch('/api/admin/settings'); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = (await res.json()) as { values?: { defaultTtsProvider?: unknown } }; + return typeof data.values?.defaultTtsProvider === 'string' ? data.values.defaultTtsProvider : ''; +} + +async function patchDefaultProviderSlug(slug: string): Promise { + const res = await fetch('/api/admin/settings', { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ updates: { defaultTtsProvider: slug } }), + }); + if (res.status === 200 || res.status === 204) return; + if (res.status === 207) { + // Multi-status: the request succeeded for some keys but failed for others. + // Only treat it as success when the defaultTtsProvider field itself was + // accepted (no matching entry in the errors array). + const payload = (await res.json().catch(() => ({}))) as { + errors?: Array<{ key?: string; message?: string }>; + }; + const failure = payload.errors?.find((entry) => entry?.key === 'defaultTtsProvider'); + if (failure) { + throw new Error(failure.message || 'Failed to update default provider'); + } + return; + } + throw new Error(`HTTP ${res.status}`); +} + +async function patchProviderEnabled(input: { id: string; enabled: boolean }): Promise { + const res = await fetch(`/api/admin/providers/${input.id}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ enabled: input.enabled }), + }); + if (!res.ok) { + const data = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(data.error || `HTTP ${res.status}`); + } +} function createEmptyForm(): FormState { return { @@ -70,6 +114,11 @@ function createEmptyForm(): FormState { }; } +function truncateModelLabel(value: string, maxLength = 56): string { + if (value.length <= maxLength) return value; + return `${value.slice(0, maxLength - 3)}...`; +} + async function fetchAdminProviders(): Promise { const res = await fetch('/api/admin/providers'); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -122,19 +171,35 @@ export function AdminProvidersPanel() { queryKey: ADMIN_PROVIDERS_QUERY_KEY, queryFn: fetchAdminProviders, }); + const { + data: defaultProviderSlug = '', + error: defaultProviderError, + } = useQuery({ + queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY, + queryFn: fetchDefaultProviderSlug, + }); useEffect(() => { if (!error) return; console.error('[AdminProvidersPanel] load failed:', error); toast.error('Failed to load admin providers'); }, [error]); + useEffect(() => { + if (!defaultProviderError) return; + console.error('[AdminProvidersPanel] default provider load failed:', defaultProviderError); + toast.error('Failed to load default provider'); + }, [defaultProviderError]); const saveMutation = useMutation({ mutationFn: upsertAdminProvider, onSuccess: async (_data, variables) => { toast.success(variables.editingId === '__new' ? 'Provider created' : 'Provider updated'); cancelEdit(); - await queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), + ]); }, onError: (mutationError) => { console.error('[AdminProvidersPanel] save failed:', mutationError); @@ -146,13 +211,46 @@ export function AdminProvidersPanel() { mutationFn: deleteAdminProvider, onSuccess: async () => { toast.success('Provider deleted'); - await queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), + ]); }, onError: (mutationError) => { console.error('[AdminProvidersPanel] delete failed:', mutationError); toast.error((mutationError as Error).message || 'Delete failed'); }, }); + const toggleEnabledMutation = useMutation({ + mutationFn: patchProviderEnabled, + onSuccess: async (_data, vars) => { + toast.success(vars.enabled ? 'Provider enabled' : 'Provider disabled'); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), + ]); + }, + onError: (mutationError) => { + console.error('[AdminProvidersPanel] toggle enabled failed:', mutationError); + toast.error((mutationError as Error).message || 'Update failed'); + }, + }); + const setDefaultMutation = useMutation({ + mutationFn: patchDefaultProviderSlug, + onSuccess: async () => { + toast.success('Default provider updated'); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), + ]); + }, + onError: (mutationError) => { + console.error('[AdminProvidersPanel] set default failed:', mutationError); + toast.error((mutationError as Error).message || 'Failed to set default'); + }, + }); const startCreate = () => { setForm(createEmptyForm()); @@ -191,6 +289,14 @@ export function AdminProvidersPanel() { if (deleteMutation.isPending) return; deleteMutation.mutate(id); }; + const toggleEnabled = (provider: AdminProviderMasked) => { + if (toggleEnabledMutation.isPending) return; + toggleEnabledMutation.mutate({ id: provider.id, enabled: !provider.enabled }); + }; + const setDefault = (slug: string) => { + if (setDefaultMutation.isPending) return; + setDefaultMutation.mutate(slug); + }; const isEditingExisting = editingId !== null && editingId !== '__new'; const editingProvider = isEditingExisting @@ -504,38 +610,101 @@ export function AdminProvidersPanel() {

No shared providers configured yet.

) : ( providers.map((p) => ( -
-
+
+
-
+
{p.displayName} {p.slug} + {defaultProviderSlug === p.slug && Default} {!p.enabled && Disabled}
-
+
{p.providerType} - {p.defaultModel ? ` · ${p.defaultModel}` : ''} + {p.defaultModel ? ( + <> + {' · '} + {truncateModelLabel(p.defaultModel)} + + ) : ( + ' · no default model' + )} {p.defaultInstructions ? ' · instructions' : ''} - {p.baseUrl ? ` · ${p.baseUrl}` : ''} - {' · '}key {p.apiKeyMask} +
+
+ {p.baseUrl ? p.baseUrl : 'provider base URL default'} · key {p.apiKeyMask}
-
- - -
+ + + {({ active }) => ( + + )} + + + {({ active, disabled }) => ( + + )} + + + {({ active }) => ( + + )} + + + {({ active }) => ( + + )} + + + +
)) diff --git a/src/components/doclist/window/FinderSidebar.tsx b/src/components/doclist/window/FinderSidebar.tsx index d6216cb..0ecf12c 100644 --- a/src/components/doclist/window/FinderSidebar.tsx +++ b/src/components/doclist/window/FinderSidebar.tsx @@ -354,7 +354,10 @@ export function FinderSidebar({
{bottomSlot && ( -
e.stopPropagation()}> +
e.stopPropagation()} + > {bottomSlot}
)} diff --git a/src/components/doclist/window/FinderStatusBar.tsx b/src/components/doclist/window/FinderStatusBar.tsx index e8c74de..acbd573 100644 --- a/src/components/doclist/window/FinderStatusBar.tsx +++ b/src/components/doclist/window/FinderStatusBar.tsx @@ -19,7 +19,7 @@ export function FinderStatusBar({
{summary} diff --git a/src/components/player/TTSPlayer.tsx b/src/components/player/TTSPlayer.tsx index c38dc23..2689747 100644 --- a/src/components/player/TTSPlayer.tsx +++ b/src/components/player/TTSPlayer.tsx @@ -32,7 +32,7 @@ export default function TTSPlayer({ currentPage, numPages }: { return (
-
+
{/* Speed control */} [ + primaryKey({ columns: [table.userId, table.action, table.opId] }), + index('idx_user_job_events_user_action_created').on(table.userId, table.action, table.createdAt), +]); + export const userPreferences = pgTable('user_preferences', { userId: text('user_id').primaryKey().references(() => user.id, { onDelete: 'cascade' }), dataJson: jsonb('data_json').notNull().default({}), diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index 2781e7d..83d3420 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -66,6 +66,22 @@ export const userTtsChars = sqliteTable("user_tts_chars", { index('idx_user_tts_chars_date').on(table.date), ]); +// Generic per-user job-creation ledger for rate/concurrency limiting of +// expensive compute operations (e.g. PDF layout parsing). One row per created +// worker op. A trailing-window COUNT over (user_id, action) enforces both a +// short-window burst cap and a wider sustained/concurrency cap; because the +// worker bounds each op by a hard cap, "ops created in the last hard-cap +// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically. +export const userJobEvents = sqliteTable('user_job_events', { + userId: text('user_id').notNull(), + action: text('action').notNull(), + opId: text('op_id').notNull(), + createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS), +}, (table) => [ + primaryKey({ columns: [table.userId, table.action, table.opId] }), + index('idx_user_job_events_user_action_created').on(table.userId, table.action, table.createdAt), +]); + export const userPreferences = sqliteTable('user_preferences', { userId: text('user_id').primaryKey().references(() => user.id, { onDelete: 'cascade' }), dataJson: text('data_json').notNull().default('{}'), diff --git a/src/lib/server/admin/providers.ts b/src/lib/server/admin/providers.ts index 8faa99a..0b99533 100644 --- a/src/lib/server/admin/providers.ts +++ b/src/lib/server/admin/providers.ts @@ -5,6 +5,8 @@ import { adminProviders } from '@/db/schema'; import { apiKeyLast4, decryptSecret, encryptSecret } from '@/lib/server/crypto/secrets'; import { type TtsProviderId } from '@/lib/shared/tts-provider-catalog'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; +import { resolvePreferredSharedProviderSlug } from '@/lib/shared/shared-provider-selection'; +import { getRuntimeConfig, setRuntimeConfigKey } from '@/lib/server/admin/settings'; export const BUILT_IN_PROVIDER_IDS: readonly TtsProviderId[] = [ 'custom-openai', @@ -195,6 +197,19 @@ export async function listAdminProviders(): Promise { return (rows as Array>).map(rowToRecord); } +export async function listEnabledAdminProviders(): Promise { + const rows = await db + .select() + .from(adminProviders) + .where(eq(adminProviders.enabled, 1)) + .orderBy( + desc(adminProviders.updatedAt), + desc(adminProviders.createdAt), + asc(adminProviders.slug), + ); + return (rows as Array>).map(rowToRecord); +} + export async function getAdminProviderBySlug(slug: string): Promise { const rows = await db.select().from(adminProviders).where(eq(adminProviders.slug, slug)).limit(1); const arr = rows as Array>; @@ -258,6 +273,8 @@ export async function updateAdminProvider( ): Promise { const current = await getAdminProviderById(id); if (!current) throw new AdminProviderError('provider not found', 404); + const runtimeConfigBefore = await getRuntimeConfig(); + const wasDefaultProvider = runtimeConfigBefore.defaultTtsProvider === current.slug; const update: Record = { updatedAt: Date.now() }; const nextModel = @@ -309,13 +326,27 @@ export async function updateAdminProvider( await db.update(adminProviders).set(update).where(eq(adminProviders.id, id)); const updated = await getAdminProviderById(id); if (!updated) throw new AdminProviderError('failed to load updated provider', 500); + if (wasDefaultProvider) { + if (updated.enabled && updated.slug !== current.slug) { + await setRuntimeConfigKey('defaultTtsProvider', updated.slug); + } else if (!updated.enabled) { + await swapDefaultSharedProvider(updated.slug); + } + } + await ensureDefaultSharedProviderValidity(); return updated; } export async function deleteAdminProvider(id: string): Promise { const existing = await getAdminProviderById(id); if (!existing) throw new AdminProviderError('provider not found', 404); + const runtimeConfigBefore = await getRuntimeConfig(); + const wasDefaultProvider = runtimeConfigBefore.defaultTtsProvider === existing.slug; await db.delete(adminProviders).where(eq(adminProviders.id, id)); + if (wasDefaultProvider) { + await swapDefaultSharedProvider(existing.slug); + } + await ensureDefaultSharedProviderValidity(); } /** Lookup helper used by TTS routes: returns null if not found or disabled. */ @@ -333,11 +364,49 @@ export async function getEnabledAdminProviderBySlug( } export async function getFirstEnabledAdminProvider(): Promise { - const rows = await db - .select() - .from(adminProviders) - .where(eq(adminProviders.enabled, 1)) - .limit(1); - const arr = rows as Array>; - return arr[0] ? rowToRecord(arr[0]) : null; + const rows = await listEnabledAdminProviders(); + return rows[0] ?? null; +} + +export async function resolvePreferredEnabledAdminProvider(input: { + requestedSlug?: string | null; + runtimeDefaultSlug?: string | null; +}): Promise { + const providers = await listEnabledAdminProviders(); + const selectedSlug = resolvePreferredSharedProviderSlug({ + providers, + requestedSlug: input.requestedSlug, + runtimeDefaultSlug: input.runtimeDefaultSlug, + }); + if (!selectedSlug) return null; + return providers.find((provider) => provider.slug === selectedSlug) ?? null; +} + +async function ensureDefaultSharedProviderValidity(): Promise { + const runtimeConfig = await getRuntimeConfig(); + const currentDefaultSlug = runtimeConfig.defaultTtsProvider; + if (!currentDefaultSlug || BUILT_IN_PROVIDER_IDS.includes(currentDefaultSlug as TtsProviderId)) { + return; + } + + const currentDefaultEnabled = await getEnabledAdminProviderBySlug(currentDefaultSlug); + if (currentDefaultEnabled) return; + + const nextProvider = await resolvePreferredEnabledAdminProvider({ + runtimeDefaultSlug: currentDefaultSlug, + }); + if (!nextProvider) return; + + await setRuntimeConfigKey('defaultTtsProvider', nextProvider.slug); +} + +async function swapDefaultSharedProvider(excludedSlug: string): Promise { + const providers = await listEnabledAdminProviders(); + const filtered = providers.filter((provider) => provider.slug !== excludedSlug); + const selectedSlug = resolvePreferredSharedProviderSlug({ + providers: filtered, + runtimeDefaultSlug: null, + }); + if (!selectedSlug) return; + await setRuntimeConfigKey('defaultTtsProvider', selectedSlug); } diff --git a/src/lib/server/admin/resolve-credentials.ts b/src/lib/server/admin/resolve-credentials.ts index 95dd7a3..2193b39 100644 --- a/src/lib/server/admin/resolve-credentials.ts +++ b/src/lib/server/admin/resolve-credentials.ts @@ -1,7 +1,7 @@ import { getEnabledAdminProviderBySlug, - getFirstEnabledAdminProvider, decryptedKeyFor, + resolvePreferredEnabledAdminProvider, type AdminProviderRecord, } from '@/lib/server/admin/providers'; @@ -46,37 +46,20 @@ export async function resolveTtsCredentials(opts: { if (opts.restrictUserApiKeys) { const requestedIsBuiltIn = isBuiltInProviderId(requestedProvider); const fallback = opts.fallbackProvider || ''; - const fallbackIsBuiltIn = isBuiltInProviderId(fallback); - const requestedSharedSlug = requestedIsBuiltIn ? '' : requestedProvider; - const fallbackSharedSlug = fallbackIsBuiltIn ? '' : fallback; - const preferredSharedSlug = requestedSharedSlug || fallbackSharedSlug; - - if (preferredSharedSlug) { - const admin = await getEnabledAdminProviderBySlug(preferredSharedSlug); - if (!admin) { - return { error: 'provider_unknown', slug: preferredSharedSlug }; - } - const apiKey = await decryptedKeyFor(admin); - return { - provider: admin.providerType, - apiKey, - baseUrl: admin.baseUrl || undefined, - fromAdmin: true, - adminRecord: admin, - }; - } - - const firstShared = await getFirstEnabledAdminProvider(); - if (!firstShared) { + const selected = await resolvePreferredEnabledAdminProvider({ + requestedSlug: requestedIsBuiltIn ? null : requestedProvider, + runtimeDefaultSlug: fallback, + }); + if (!selected) { return { error: 'no_shared_provider_configured', slug: requestedProvider }; } - const apiKey = await decryptedKeyFor(firstShared); + const apiKey = await decryptedKeyFor(selected); return { - provider: firstShared.providerType, + provider: selected.providerType, apiKey, - baseUrl: firstShared.baseUrl || undefined, + baseUrl: selected.baseUrl || undefined, fromAdmin: true, - adminRecord: firstShared, + adminRecord: selected, }; } diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index f10dffe..1abbd34 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -76,6 +76,24 @@ function stringValue(defaultValue: string, envVar: string): RuntimeConfigKeyDef< }; } +function positiveIntValue(defaultValue: number, envVar?: string): RuntimeConfigKeyDef { + return { + default: defaultValue, + envVar, + parseEnv(raw) { + const trimmed = raw.trim(); + if (!trimmed) return undefined; + const parsed = Number(trimmed); + if (!Number.isFinite(parsed) || parsed < 1) return undefined; + return Math.floor(parsed); + }, + validate(value) { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 1) return undefined; + return Math.floor(value); + }, + }; +} + export const RUNTIME_CONFIG_SCHEMA = { defaultTtsProvider: stringValue('custom-openai', 'RUNTIME_SEED_DEFAULT_TTS_PROVIDER'), changelogFeedUrl: stringValue('https://docs.openreader.richardr.dev/changelog/manifest.json', 'RUNTIME_SEED_CHANGELOG_FEED_URL'), @@ -88,6 +106,23 @@ export const RUNTIME_CONFIG_SCHEMA = { enableDocxConversion: booleanFlag(true, 'RUNTIME_SEED_ENABLE_DOCX_CONVERSION'), enableDestructiveDeleteActions: booleanFlag(true, 'RUNTIME_SEED_ENABLE_DESTRUCTIVE_DELETE_ACTIONS'), showAllProviderModels: runtimeBoolean(true), + disableTtsRateLimit: booleanFlag(true, 'RUNTIME_SEED_DISABLE_TTS_LIMIT'), + ttsDailyLimitAnonymous: positiveIntValue(50_000), + ttsDailyLimitAuthenticated: positiveIntValue(500_000), + ttsIpDailyLimitAnonymous: positiveIntValue(100_000), + ttsIpDailyLimitAuthenticated: positiveIntValue(1_000_000), + // Per-user throttle for expensive PDF-layout parsing. Disabled by default + // (admins enable it in Settings → Admin), mirroring disableTtsRateLimit. + // When enabled, the sub-limits below apply (admin-tunable, no env seed): + // a short "burst" window plus a wider "sustained" window that also bounds + // concurrency (the worker caps each job's duration). + disableComputeRateLimit: booleanFlag(true, 'RUNTIME_SEED_DISABLE_COMPUTE_LIMIT'), + computeParseBurstMax: positiveIntValue(8), + computeParseBurstWindowSec: positiveIntValue(60), + computeParseSustainedMax: positiveIntValue(24), + computeParseSustainedWindowSec: positiveIntValue(600), + // Maximum size (MB) accepted for a single document upload. + maxUploadMb: positiveIntValue(200), } as const satisfies Record>; export type RuntimeConfigKey = keyof typeof RUNTIME_CONFIG_SCHEMA; diff --git a/src/lib/server/audiobooks/settings.ts b/src/lib/server/audiobooks/settings.ts index 1e61ef7..cf74673 100644 --- a/src/lib/server/audiobooks/settings.ts +++ b/src/lib/server/audiobooks/settings.ts @@ -8,6 +8,7 @@ import { isBuiltInTtsProviderId, isTtsProviderType, type TtsProviderId } from '@ import type { AudiobookGenerationSettings } from '@/types/client'; import type { TTSAudiobookFormat } from '@/types/tts'; import { resolveEffectiveTtsInstructions } from '@/lib/server/admin/tts-instructions'; +import { resolvePreferredSharedProviderSlug } from '@/lib/shared/shared-provider-selection'; function isAudiobookFormat(value: unknown): value is TTSAudiobookFormat { return value === 'mp3' || value === 'm4b'; @@ -101,13 +102,12 @@ function resolveRestrictedProviderRef( fallbackProviderRef: string, sharedProviders: SharedProviderPolicyEntry[], ): string { - const requestedIsBuiltIn = isBuiltInTtsProviderId(providerRef); - const fallbackIsBuiltIn = isBuiltInTtsProviderId(fallbackProviderRef); - const requestedSharedSlug = requestedIsBuiltIn ? '' : providerRef; - const fallbackSharedSlug = fallbackIsBuiltIn ? '' : fallbackProviderRef; - const preferredSharedSlug = requestedSharedSlug || fallbackSharedSlug; - if (preferredSharedSlug) return preferredSharedSlug; - return sharedProviders[0]?.slug || providerRef; + const preferred = resolvePreferredSharedProviderSlug({ + providers: sharedProviders, + requestedSlug: isBuiltInTtsProviderId(providerRef) ? null : providerRef, + runtimeDefaultSlug: isBuiltInTtsProviderId(fallbackProviderRef) ? null : fallbackProviderRef, + }); + return preferred || providerRef; } export function canonicalizeAudiobookSettingsForRuntime(input: { diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index 5ecae17..fb84f00 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -104,17 +104,28 @@ export async function presignPut( id: string, contentType: string, namespace: string | null, + options?: { contentLength?: number }, ): Promise<{ url: string; headers: Record }> { const cfg = getS3Config(); const client = getS3Client(); const key = documentKey(id, namespace); const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; + // When the client declares an exact size, sign Content-Length so S3 rejects a + // PUT whose body does not match (the browser always sends an accurate + // Content-Length for a known body). Skipped when size is unknown/zero so the + // upload still works against stores that enforce the signed header. + const contentLength = + typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0 + ? Math.floor(options.contentLength) + : undefined; + const command = new PutObjectCommand({ Bucket: cfg.bucket, Key: key, ContentType: normalizedType, ServerSideEncryption: 'AES256', + ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), }); const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); diff --git a/src/lib/server/rate-limit/job-rate-limiter.ts b/src/lib/server/rate-limit/job-rate-limiter.ts new file mode 100644 index 0000000..f5a41e5 --- /dev/null +++ b/src/lib/server/rate-limit/job-rate-limiter.ts @@ -0,0 +1,167 @@ +import { and, eq, gte, lt, sql } from 'drizzle-orm'; +import { db } from '@/db'; +import { userJobEvents } from '@/db/schema'; +import { isAuthEnabled } from '@/lib/server/auth/config'; +import { nowTimestampMs } from '@/lib/shared/timestamps'; +import type { RuntimeConfig } from '@/lib/server/admin/settings'; + +/** + * Per-user rate / concurrency limiting for expensive compute operations. + * + * Backed by the `user_job_events` ledger: one row is recorded per created + * worker op, and limits are enforced by COUNTing rows for a (userId, action) + * within trailing time windows. Two windows are configured per action: + * - a short "burst" window (stops tight retry/replace loops), and + * - a wider "sustained" window. Because the worker bounds each op by a hard + * cap, "ops created in the sustained window" is an upper bound on the + * number that can still be running — i.e. an effective concurrency cap. + * + * Limits are configured by site admins via runtime config (not env), mirroring + * the TTS rate limiter. The design is generic: add a new `JobRateAction` plus a + * config mapping to throttle a different job type. Limits are a backstop — a + * small overshoot under highly concurrent requests is acceptable (the check is + * read-then-record, not a single atomic transaction). + */ +export type JobRateAction = 'pdf_layout'; + +export interface JobRateWindow { + windowMs: number; + max: number; +} + +export interface JobRateConfig { + enabled: boolean; + windows: JobRateWindow[]; +} + +export interface JobRateDecision { + allowed: boolean; + /** Milliseconds until the binding window frees a slot (0 when allowed). */ + retryAfterMs: number; + counts: Array<{ windowMs: number; max: number; count: number }>; +} + +/** Builds the PDF-layout parse limiter config from resolved runtime config. */ +export function getPdfLayoutRateConfig(runtime: Pick< + RuntimeConfig, + | 'disableComputeRateLimit' + | 'computeParseBurstMax' + | 'computeParseBurstWindowSec' + | 'computeParseSustainedMax' + | 'computeParseSustainedWindowSec' +>): JobRateConfig { + return { + enabled: !runtime.disableComputeRateLimit, + windows: [ + { windowMs: runtime.computeParseBurstWindowSec * 1000, max: runtime.computeParseBurstMax }, + { windowMs: runtime.computeParseSustainedWindowSec * 1000, max: runtime.computeParseSustainedMax }, + ], + }; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const safeDb = () => db as any; + +function isActive(config: Pick, userId: string | null | undefined): userId is string { + return isAuthEnabled() && config.enabled && Boolean(userId); +} + +async function countSince(userId: string, action: JobRateAction, since: number): Promise { + const rows = await safeDb() + .select({ c: sql`count(*)` }) + .from(userJobEvents) + .where(and( + eq(userJobEvents.userId, userId), + eq(userJobEvents.action, action), + gte(userJobEvents.createdAt, since), + )); + return Number(rows[0]?.c ?? 0); +} + +async function oldestSince(userId: string, action: JobRateAction, since: number): Promise { + const rows = await safeDb() + .select({ oldest: sql`min(${userJobEvents.createdAt})` }) + .from(userJobEvents) + .where(and( + eq(userJobEvents.userId, userId), + eq(userJobEvents.action, action), + gte(userJobEvents.createdAt, since), + )); + const value = rows[0]?.oldest; + return value == null ? null : Number(value); +} + +/** + * Returns whether a new op may be created for this user/action. Read-only — call + * `recordJobEvent` after the op is actually created. + */ +export async function checkJobRate( + userId: string | null | undefined, + action: JobRateAction, + config: JobRateConfig, +): Promise { + if (!isActive(config, userId)) { + return { allowed: true, retryAfterMs: 0, counts: [] }; + } + + const now = nowTimestampMs(); + const counts: JobRateDecision['counts'] = []; + let retryAfterMs = 0; + let allowed = true; + + for (const window of config.windows) { + if (!Number.isFinite(window.windowMs) || window.windowMs <= 0 || window.max <= 0) continue; + const since = now - window.windowMs; + const count = await countSince(userId, action, since); + counts.push({ windowMs: window.windowMs, max: window.max, count }); + if (count >= window.max) { + allowed = false; + const oldest = await oldestSince(userId, action, since); + // When the oldest in-window event ages out, a slot frees up. + const freesAt = (oldest ?? now) + window.windowMs; + retryAfterMs = Math.max(retryAfterMs, Math.max(0, freesAt - now)); + } + } + + return { allowed, retryAfterMs, counts }; +} + +/** Records a created op so it counts toward future window checks. */ +export async function recordJobEvent( + userId: string | null | undefined, + action: JobRateAction, + opId: string, + config: JobRateConfig, +): Promise { + if (!isActive(config, userId) || !opId) return; + + const now = nowTimestampMs(); + try { + await safeDb() + .insert(userJobEvents) + .values({ userId, action, opId, createdAt: now }) + .onConflictDoNothing({ target: [userJobEvents.userId, userJobEvents.action, userJobEvents.opId] }); + } catch { + // Recording is best-effort; never block op creation on ledger writes. + } + + // Opportunistic prune of rows older than the largest configured window so + // the ledger stays small without a separate cron, but never deletes events + // that could still affect an in-window count. + if (Math.random() < 0.05) { + const largestWindowMs = config.windows.reduce( + (max, w) => (Number.isFinite(w.windowMs) && w.windowMs > max ? w.windowMs : max), + 24 * 60 * 60 * 1000, + ); + try { + await safeDb() + .delete(userJobEvents) + .where(and( + eq(userJobEvents.action, action), + lt(userJobEvents.createdAt, now - largestWindowMs), + )); + } catch { + // ignore prune failures + } + } +} diff --git a/src/lib/server/rate-limit/problem-response.ts b/src/lib/server/rate-limit/problem-response.ts index 9f7e2ad..5fa9d3a 100644 --- a/src/lib/server/rate-limit/problem-response.ts +++ b/src/lib/server/rate-limit/problem-response.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server'; -import { RATE_LIMITS, type RateLimitResult } from '@/lib/server/rate-limit/rate-limiter'; +import { type RateLimitResult } from '@/lib/server/rate-limit/rate-limiter'; +import { type JobRateDecision } from '@/lib/server/rate-limit/job-rate-limiter'; function formatLimitForHint(limit: number): string { if (!Number.isFinite(limit) || limit <= 0) return String(limit); @@ -15,8 +16,10 @@ export function buildDailyQuotaExceededResponse(input: { rateLimitResult: RateLimitResult; isAnonymousUser: boolean; pathname: string; + anonymousLimit: number; + authenticatedLimit: number; }): NextResponse { - const { rateLimitResult, isAnonymousUser, pathname } = input; + const { rateLimitResult, isAnonymousUser, pathname, anonymousLimit, authenticatedLimit } = input; const resetTimeMs = rateLimitResult.resetTimeMs; const retryAfterSeconds = Math.max(0, Math.ceil((resetTimeMs - Date.now()) / 1000)); @@ -32,7 +35,7 @@ export function buildDailyQuotaExceededResponse(input: { resetTimeMs, userType: isAnonymousUser ? 'anonymous' : 'authenticated', upgradeHint: isAnonymousUser - ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + ? `Sign up to increase your limit from ${formatLimitForHint(anonymousLimit)} to ${formatLimitForHint(authenticatedLimit)} characters per day` : undefined, instance: pathname, }), { @@ -43,3 +46,28 @@ export function buildDailyQuotaExceededResponse(input: { }, }); } + +/** + * 429 response for the compute job rate / concurrency limiter (e.g. PDF parse). + */ +export function buildComputeRateLimitedResponse(input: { + decision: JobRateDecision; + pathname: string; +}): NextResponse { + const retryAfterSeconds = Math.max(1, Math.ceil(input.decision.retryAfterMs / 1000)); + return new NextResponse(JSON.stringify({ + type: 'https://openreader.app/problems/compute-rate-limited', + title: 'Too many compute requests', + status: 429, + detail: 'You have started too many processing operations recently. Please wait and try again.', + code: 'COMPUTE_RATE_LIMITED', + retryAfterMs: input.decision.retryAfterMs, + instance: input.pathname, + }), { + status: 429, + headers: { + 'Content-Type': 'application/problem+json', + 'Retry-After': String(retryAfterSeconds), + }, + }); +} diff --git a/src/lib/server/rate-limit/rate-limiter.ts b/src/lib/server/rate-limit/rate-limiter.ts index 053cb8d..0a9dd68 100644 --- a/src/lib/server/rate-limit/rate-limiter.ts +++ b/src/lib/server/rate-limit/rate-limiter.ts @@ -3,53 +3,40 @@ import { userTtsChars } from '@/db/schema'; import { isAuthEnabled } from '@/lib/server/auth/config'; import { eq, and, lt, sql } from 'drizzle-orm'; import { nextUtcMidnightTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; -import { serverLogger } from '@/lib/server/logger'; -import { logDegraded } from '@/lib/server/errors/logging'; -function readPositiveIntEnv(name: string, fallback: number): number { - const raw = process.env[name]; - if (!raw || raw.trim() === '') return fallback; - - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - logDegraded(serverLogger, { - event: 'rate_limit.config.invalid_env', - msg: 'Invalid rate limiter env value; using default', - step: 'read_limit_env', - context: { - envVar: name, - envValue: raw, - fallbackValue: fallback, - }, - }); - return fallback; - } - - return Math.floor(parsed); +export interface RateLimitThresholds { + anonymous: number; + authenticated: number; + ipAnonymous: number; + ipAuthenticated: number; } -function readBooleanEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]; - if (!raw || raw.trim() === '') return fallback; - const normalized = raw.trim().toLowerCase(); - if (normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on') return true; - if (normalized === '0' || normalized === 'false' || normalized === 'no' || normalized === 'off') return false; - return fallback; +export interface RateLimitRuntimeOptions { + enabled: boolean; + limits: RateLimitThresholds; } -export function isTtsRateLimitEnabled(): boolean { - return readBooleanEnv('TTS_ENABLE_RATE_LIMIT', false); -} +export const DEFAULT_RATE_LIMITS: RateLimitThresholds = { + anonymous: 50_000, + authenticated: 500_000, + ipAnonymous: 100_000, + ipAuthenticated: 1_000_000, +}; -// Rate limits configuration - character counts per day -export const RATE_LIMITS = { - ANONYMOUS: readPositiveIntEnv('TTS_DAILY_LIMIT_ANONYMOUS', 50_000), - AUTHENTICATED: readPositiveIntEnv('TTS_DAILY_LIMIT_AUTHENTICATED', 500_000), - // IP-based backstop limits to make it harder to reset limits by creating new accounts - // or clearing storage/cookies - IP_ANONYMOUS: readPositiveIntEnv('TTS_IP_DAILY_LIMIT_ANONYMOUS', 100_000), - IP_AUTHENTICATED: readPositiveIntEnv('TTS_IP_DAILY_LIMIT_AUTHENTICATED', 1_000_000), -} as const; +export function resolveRateLimitThresholds(input?: Partial): RateLimitThresholds { + const source = input ?? {}; + const normalize = (value: unknown, fallback: number): number => { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return fallback; + return Math.floor(value); + }; + + return { + anonymous: normalize(source.anonymous, DEFAULT_RATE_LIMITS.anonymous), + authenticated: normalize(source.authenticated, DEFAULT_RATE_LIMITS.authenticated), + ipAnonymous: normalize(source.ipAnonymous, DEFAULT_RATE_LIMITS.ipAnonymous), + ipAuthenticated: normalize(source.ipAuthenticated, DEFAULT_RATE_LIMITS.ipAuthenticated), + }; +} // Helper to ensure DB is strictly typed when we know it exists // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -163,8 +150,15 @@ export class RateLimiter { /** * Check if a user can use TTS and increment their char count if allowed */ - async checkAndIncrementLimit(user: UserInfo, charCount: number, backstops?: RateLimitBackstops): Promise { - if (!isAuthEnabled() || !isTtsRateLimitEnabled()) { + async checkAndIncrementLimit( + user: UserInfo, + charCount: number, + backstops?: RateLimitBackstops, + options?: RateLimitRuntimeOptions, + ): Promise { + const limits = resolveRateLimitThresholds(options?.limits); + const enabled = options?.enabled ?? true; + if (!isAuthEnabled() || !enabled) { return { allowed: true, currentCount: 0, @@ -176,7 +170,7 @@ export class RateLimiter { const today = new Date().toISOString().split('T')[0]; const dateValue = today as unknown as UserTtsCharsDateValue; - const userLimit = user.isAnonymous ? RATE_LIMITS.ANONYMOUS : RATE_LIMITS.AUTHENTICATED; + const userLimit = user.isAnonymous ? limits.anonymous : limits.authenticated; const buckets: Bucket[] = [{ key: user.id, limit: userLimit }]; @@ -184,13 +178,13 @@ export class RateLimiter { const ip = backstops?.ip?.toString() || null; if (user.isAnonymous && deviceId) { - buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: RATE_LIMITS.ANONYMOUS }); + buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: limits.anonymous }); } if (ip) { buckets.push({ key: normalizeBackstopKey('ip', ip), - limit: user.isAnonymous ? RATE_LIMITS.IP_ANONYMOUS : RATE_LIMITS.IP_AUTHENTICATED, + limit: user.isAnonymous ? limits.ipAnonymous : limits.ipAuthenticated, }); } @@ -254,7 +248,7 @@ export class RateLimiter { }); } catch (error) { if (error instanceof RateLimitExceeded) { - const current = await this.getCurrentUsage(user, backstops); + const current = await this.getCurrentUsage(user, backstops, options); return { ...current, allowed: false }; } throw error; @@ -264,8 +258,14 @@ export class RateLimiter { /** * Get current usage for a user without incrementing */ - async getCurrentUsage(user: UserInfo, backstops?: RateLimitBackstops): Promise { - if (!isAuthEnabled() || !isTtsRateLimitEnabled()) { + async getCurrentUsage( + user: UserInfo, + backstops?: RateLimitBackstops, + options?: RateLimitRuntimeOptions, + ): Promise { + const limits = resolveRateLimitThresholds(options?.limits); + const enabled = options?.enabled ?? true; + if (!isAuthEnabled() || !enabled) { return { allowed: true, currentCount: 0, @@ -276,7 +276,7 @@ export class RateLimiter { } const today = new Date().toISOString().split('T')[0]; - const userLimit = user.isAnonymous ? RATE_LIMITS.ANONYMOUS : RATE_LIMITS.AUTHENTICATED; + const userLimit = user.isAnonymous ? limits.anonymous : limits.authenticated; const buckets: Bucket[] = [{ key: user.id, limit: userLimit }]; @@ -284,13 +284,13 @@ export class RateLimiter { const ip = backstops?.ip?.toString() || null; if (user.isAnonymous && deviceId) { - buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: RATE_LIMITS.ANONYMOUS }); + buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: limits.anonymous }); } if (ip) { buckets.push({ key: normalizeBackstopKey('ip', ip), - limit: user.isAnonymous ? RATE_LIMITS.IP_ANONYMOUS : RATE_LIMITS.IP_AUTHENTICATED, + limit: user.isAnonymous ? limits.ipAnonymous : limits.ipAuthenticated, }); } @@ -321,7 +321,7 @@ export class RateLimiter { * Transfer char counts when anonymous user creates an account */ async transferAnonymousUsage(anonymousUserId: string, authenticatedUserId: string): Promise { - if (!isAuthEnabled() || !isTtsRateLimitEnabled()) return; + if (!isAuthEnabled()) return; const today = new Date().toISOString().split('T')[0]; const dateValue = today as unknown as UserTtsCharsDateValue; diff --git a/src/lib/server/rate-limit/request-ip.ts b/src/lib/server/rate-limit/request-ip.ts index 16d852e..2037790 100644 --- a/src/lib/server/rate-limit/request-ip.ts +++ b/src/lib/server/rate-limit/request-ip.ts @@ -1,26 +1,55 @@ import type { NextRequest } from 'next/server'; /** - * Best-effort client IP extraction that works on Vercel and typical reverse proxies. + * Best-effort client IP extraction for rate-limit backstops. * - * Note: IP-based limits are a backstop only; they are not perfectly reliable. + * Security note: `X-Forwarded-For` is a list the client can *prepend* to, so the + * left-most entry is attacker-controlled and must never be trusted as the client + * IP — doing so lets an attacker land every request in a fresh bucket and defeat + * the IP backstop entirely. + * + * Precedence (most trustworthy first): + * 1. x-vercel-forwarded-for — set by Vercel; inbound client copies are + * stripped, so it cannot be spoofed. Only consulted when running on Vercel. + * 2. x-real-ip / cf-connecting-ip — single values set by the reverse proxy + * (Vercel / Cloudflare overwrite any client-supplied copy). + * 3. x-forwarded-for — right-most hop only (the address seen by the closest + * trusted proxy), as a best-effort fallback for single-proxy self-hosts. + * 4. NextRequest.ip — runtime-provided connecting address. + * + * IP-based limits remain a backstop only; the per-user bucket is the primary + * control for authenticated abuse. */ +function firstIp(value: string | null): string | null { + if (!value) return null; + const first = value.split(',')[0]?.trim(); + return first || null; +} + export function getClientIp(req: NextRequest): string | null { - // Standard proxy header. Vercel also sets this. - const forwardedFor = req.headers.get('x-forwarded-for'); - if (forwardedFor) { - const first = forwardedFor.split(',')[0]?.trim(); - if (first) return first; + // 1. Vercel-internal header (clients cannot set x-vercel-* — Vercel strips them). + if (process.env.VERCEL) { + const vercelIp = firstIp(req.headers.get('x-vercel-forwarded-for')); + if (vercelIp) return vercelIp; } - const realIp = req.headers.get('x-real-ip'); - if (realIp) return realIp.trim(); + // 2. Single-value proxy headers set (and overwritten) by the edge. + const realIp = req.headers.get('x-real-ip')?.trim(); + if (realIp) return realIp; - // Some proxies use this. - const cfConnectingIp = req.headers.get('cf-connecting-ip'); - if (cfConnectingIp) return cfConnectingIp.trim(); + const cfConnectingIp = req.headers.get('cf-connecting-ip')?.trim(); + if (cfConnectingIp) return cfConnectingIp; - // NextRequest may expose ip depending on runtime. + // 3. Fall back to the RIGHT-most X-Forwarded-For hop (closest trusted proxy's + // view). Never the left-most, which the client controls. + const forwardedFor = req.headers.get('x-forwarded-for'); + if (forwardedFor) { + const parts = forwardedFor.split(',').map((part) => part.trim()).filter(Boolean); + const rightmost = parts[parts.length - 1]; + if (rightmost) return rightmost; + } + + // 4. Runtime-provided connecting address, when available. // eslint-disable-next-line @typescript-eslint/no-explicit-any const reqAny = req as any; const ip = typeof reqAny.ip === 'string' ? (reqAny.ip as string) : null; diff --git a/src/lib/server/testing/test-namespace.ts b/src/lib/server/testing/test-namespace.ts index eb9278a..044e46f 100644 --- a/src/lib/server/testing/test-namespace.ts +++ b/src/lib/server/testing/test-namespace.ts @@ -5,7 +5,24 @@ import { ensureSystemUserExists } from '@/db'; const TEST_NAMESPACE_HEADER = 'x-openreader-test-namespace'; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +/** + * The test-namespace header is test/CI scaffolding and must never be honored on + * a real production deployment. It is enabled when `ENABLE_TEST_NAMESPACE=true` + * (set explicitly by the Playwright web server / CI env) or, for local dev + * convenience, whenever the build is not a production build. + * + * Note: the E2E suite runs against a production build (`pnpm build && pnpm + * start`), so a `NODE_ENV` check alone is insufficient — CI relies on the + * explicit flag. + */ +function isTestNamespaceEnabled(): boolean { + if (process.env.ENABLE_TEST_NAMESPACE?.trim().toLowerCase() === 'true') return true; + return process.env.NODE_ENV !== 'production'; +} + export function getOpenReaderTestNamespace(headers: Headers): string | null { + if (!isTestNamespaceEnabled()) return null; + const raw = headers.get(TEST_NAMESPACE_HEADER)?.trim(); if (!raw) return null; diff --git a/src/lib/shared/shared-provider-selection.ts b/src/lib/shared/shared-provider-selection.ts new file mode 100644 index 0000000..8dbe87f --- /dev/null +++ b/src/lib/shared/shared-provider-selection.ts @@ -0,0 +1,34 @@ +import { isBuiltInTtsProviderId } from '@/lib/shared/tts-provider-catalog'; + +export interface SharedProviderSlugEntry { + slug: string; +} + +function normalizeSharedSlug(value: string | null | undefined): string { + const trimmed = typeof value === 'string' ? value.trim() : ''; + if (!trimmed) return ''; + return isBuiltInTtsProviderId(trimmed) ? '' : trimmed; +} + +export function resolvePreferredSharedProviderSlug(input: { + providers: readonly T[]; + requestedSlug?: string | null; + runtimeDefaultSlug?: string | null; +}): string | null { + const providers = input.providers; + if (providers.length === 0) return null; + + const bySlug = new Map(); + for (const provider of providers) { + bySlug.set(provider.slug, provider); + } + + const requested = normalizeSharedSlug(input.requestedSlug); + if (requested && bySlug.has(requested)) return requested; + + const runtimeDefault = normalizeSharedSlug(input.runtimeDefaultSlug); + if (runtimeDefault && bySlug.has(runtimeDefault)) return runtimeDefault; + + if (bySlug.has('default-openai')) return 'default-openai'; + return providers[0]?.slug ?? null; +} diff --git a/tests/unit/admin-providers-validation.spec.ts b/tests/unit/admin-providers-validation.vitest.spec.ts similarity index 97% rename from tests/unit/admin-providers-validation.spec.ts rename to tests/unit/admin-providers-validation.vitest.spec.ts index ca3b68b..e977cc7 100644 --- a/tests/unit/admin-providers-validation.spec.ts +++ b/tests/unit/admin-providers-validation.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { inArray } from 'drizzle-orm'; import { db } from '../../src/db'; import { adminProviders } from '../../src/db/schema'; @@ -12,7 +12,7 @@ import { type AdminProviderRecord, } from '../../src/lib/server/admin/providers'; -test.describe('admin provider validation', () => { +describe('admin provider validation', () => { test('normalizes valid slugs to lowercase', () => { expect(validateSlug('My-Provider-1')).toBe('my-provider-1'); }); diff --git a/tests/unit/admin-tts-instructions.spec.ts b/tests/unit/admin-tts-instructions.vitest.spec.ts similarity index 92% rename from tests/unit/admin-tts-instructions.spec.ts rename to tests/unit/admin-tts-instructions.vitest.spec.ts index 0a7198e..11e06e8 100644 --- a/tests/unit/admin-tts-instructions.spec.ts +++ b/tests/unit/admin-tts-instructions.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { resolveEffectiveTtsInstructions } from '../../src/lib/server/admin/tts-instructions'; -test.describe('resolveEffectiveTtsInstructions', () => { +describe('resolveEffectiveTtsInstructions', () => { test('uses explicit request instructions when model supports them', () => { const out = resolveEffectiveTtsInstructions({ model: 'gpt-4o-mini-tts', diff --git a/tests/unit/audiobook-scope.spec.ts b/tests/unit/audiobook-scope.vitest.spec.ts similarity index 93% rename from tests/unit/audiobook-scope.spec.ts rename to tests/unit/audiobook-scope.vitest.spec.ts index b73ba40..ca9705b 100644 --- a/tests/unit/audiobook-scope.spec.ts +++ b/tests/unit/audiobook-scope.vitest.spec.ts @@ -1,7 +1,7 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '../../src/lib/server/audiobooks/user-scope'; -test.describe('audiobook scope selection', () => { +describe('audiobook scope selection', () => { test('uses only unclaimed scope when auth is disabled', () => { const result = buildAllowedAudiobookUserIds(false, null, 'unclaimed::ns'); expect(result.preferredUserId).toBe('unclaimed::ns'); diff --git a/tests/unit/audiobook-settings.spec.ts b/tests/unit/audiobook-settings.vitest.spec.ts similarity index 96% rename from tests/unit/audiobook-settings.spec.ts rename to tests/unit/audiobook-settings.vitest.spec.ts index af3c788..1cc371c 100644 --- a/tests/unit/audiobook-settings.spec.ts +++ b/tests/unit/audiobook-settings.vitest.spec.ts @@ -1,10 +1,10 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { canonicalizeAudiobookSettingsForRuntime, coerceAudiobookGenerationSettings, } from '../../src/lib/server/audiobooks/settings'; -test.describe('coerceAudiobookGenerationSettings', () => { +describe('coerceAudiobookGenerationSettings', () => { test('accepts current metadata shape without migration', () => { const result = coerceAudiobookGenerationSettings({ providerRef: 'openai', diff --git a/tests/unit/audiobook.spec.ts b/tests/unit/audiobook.vitest.spec.ts similarity index 95% rename from tests/unit/audiobook.spec.ts rename to tests/unit/audiobook.vitest.spec.ts index 53417a2..0533385 100644 --- a/tests/unit/audiobook.spec.ts +++ b/tests/unit/audiobook.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { escapeFFMetadata, encodeChapterTitleTag, @@ -7,7 +7,7 @@ import { decodeChapterFileName } from '../../src/lib/server/audiobooks/chapters'; -test.describe('escapeFFMetadata', () => { +describe('escapeFFMetadata', () => { test('escapes special characters correctly', () => { const input = 'Title with = ; # and backslash \\'; // Expected: Equal -> \=, Semicolon -> \;, Hash -> \#, Backslash -> \\ @@ -37,7 +37,7 @@ test.describe('escapeFFMetadata', () => { }); }); -test.describe('Title Tags', () => { +describe('Title Tags', () => { test('encodeChapterTitleTag formats correctly', () => { expect(encodeChapterTitleTag(0, 'Intro')).toBe('0001 - Intro'); expect(encodeChapterTitleTag(9, 'Chapter Ten')).toBe('0010 - Chapter Ten'); @@ -63,7 +63,7 @@ test.describe('Title Tags', () => { }); }); -test.describe('Chapter File Names', () => { +describe('Chapter File Names', () => { test('encodeChapterFileName formats correctly', () => { expect(encodeChapterFileName(0, 'Intro', 'mp3')).toBe('0001__Intro.mp3'); expect(encodeChapterFileName(1, 'Part 2', 'm4b')).toBe('0002__Part%202.m4b'); diff --git a/tests/unit/audiobooks-blobstore.spec.ts b/tests/unit/audiobooks-blobstore.vitest.spec.ts similarity index 94% rename from tests/unit/audiobooks-blobstore.spec.ts rename to tests/unit/audiobooks-blobstore.vitest.spec.ts index 198b2bf..cb1ee96 100644 --- a/tests/unit/audiobooks-blobstore.spec.ts +++ b/tests/unit/audiobooks-blobstore.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { beforeAll, describe, expect, test } from 'vitest'; import { audiobookKey, audiobookPrefix, @@ -14,8 +14,8 @@ function configureS3Env() { process.env.S3_PREFIX = 'openreader-test'; } -test.describe('audiobooks-blobstore', () => { - test.beforeAll(() => { +describe('audiobooks-blobstore', () => { + beforeAll(() => { configureS3Env(); }); diff --git a/tests/unit/auth-config.spec.ts b/tests/unit/auth-config.spec.ts deleted file mode 100644 index 24fb496..0000000 --- a/tests/unit/auth-config.spec.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { test, expect } from '@playwright/test'; -import { isAnonymousAuthSessionsEnabled, isGithubAuthEnabled } from '../../src/lib/server/auth/config'; - -const ORIGINAL_BASE_URL = process.env.BASE_URL; -const ORIGINAL_AUTH_SECRET = process.env.AUTH_SECRET; -const ORIGINAL_USE_ANON = process.env.USE_ANONYMOUS_AUTH_SESSIONS; -const ORIGINAL_GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID; -const ORIGINAL_GITHUB_CLIENT_SECRET = process.env.GITHUB_CLIENT_SECRET; - -function restoreEnv() { - if (ORIGINAL_BASE_URL === undefined) delete process.env.BASE_URL; - else process.env.BASE_URL = ORIGINAL_BASE_URL; - - if (ORIGINAL_AUTH_SECRET === undefined) delete process.env.AUTH_SECRET; - else process.env.AUTH_SECRET = ORIGINAL_AUTH_SECRET; - - if (ORIGINAL_USE_ANON === undefined) delete process.env.USE_ANONYMOUS_AUTH_SESSIONS; - else process.env.USE_ANONYMOUS_AUTH_SESSIONS = ORIGINAL_USE_ANON; - - if (ORIGINAL_GITHUB_CLIENT_ID === undefined) delete process.env.GITHUB_CLIENT_ID; - else process.env.GITHUB_CLIENT_ID = ORIGINAL_GITHUB_CLIENT_ID; - - if (ORIGINAL_GITHUB_CLIENT_SECRET === undefined) delete process.env.GITHUB_CLIENT_SECRET; - else process.env.GITHUB_CLIENT_SECRET = ORIGINAL_GITHUB_CLIENT_SECRET; -} - -function setAuthEnabledEnv() { - process.env.BASE_URL = 'http://localhost:3003'; - process.env.AUTH_SECRET = 'test-secret'; -} - -test.describe('auth config anonymous-session flag', () => { - test.afterEach(() => { - restoreEnv(); - }); - - test('returns false when auth is disabled', () => { - delete process.env.BASE_URL; - delete process.env.AUTH_SECRET; - process.env.USE_ANONYMOUS_AUTH_SESSIONS = 'true'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); - - test('defaults to false when env var is unset', () => { - setAuthEnabledEnv(); - delete process.env.USE_ANONYMOUS_AUTH_SESSIONS; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); - - test('returns true only when env var is true', () => { - setAuthEnabledEnv(); - process.env.USE_ANONYMOUS_AUTH_SESSIONS = 'true'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(true); - }); - - test('returns false when env var is false', () => { - setAuthEnabledEnv(); - process.env.USE_ANONYMOUS_AUTH_SESSIONS = 'false'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); - - test('falls back to false for invalid values', () => { - setAuthEnabledEnv(); - process.env.USE_ANONYMOUS_AUTH_SESSIONS = '1'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); -}); - -test.describe('auth config github-auth flag', () => { - test.afterEach(() => { - restoreEnv(); - }); - - test('returns false when auth is disabled', () => { - delete process.env.BASE_URL; - delete process.env.AUTH_SECRET; - process.env.GITHUB_CLIENT_ID = 'some-id'; - process.env.GITHUB_CLIENT_SECRET = 'some-secret'; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns false when GITHUB_CLIENT_ID is missing', () => { - setAuthEnabledEnv(); - delete process.env.GITHUB_CLIENT_ID; - process.env.GITHUB_CLIENT_SECRET = 'some-secret'; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns false when GITHUB_CLIENT_SECRET is missing', () => { - setAuthEnabledEnv(); - process.env.GITHUB_CLIENT_ID = 'some-id'; - delete process.env.GITHUB_CLIENT_SECRET; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns false when both GitHub env vars are missing', () => { - setAuthEnabledEnv(); - delete process.env.GITHUB_CLIENT_ID; - delete process.env.GITHUB_CLIENT_SECRET; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns true when auth is enabled and both GitHub env vars are set', () => { - setAuthEnabledEnv(); - process.env.GITHUB_CLIENT_ID = 'some-id'; - process.env.GITHUB_CLIENT_SECRET = 'some-secret'; - - expect(isGithubAuthEnabled()).toBe(true); - }); -}); diff --git a/tests/unit/auth-config.vitest.spec.ts b/tests/unit/auth-config.vitest.spec.ts new file mode 100644 index 0000000..eac330d --- /dev/null +++ b/tests/unit/auth-config.vitest.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from 'vitest'; +import { getAuthBaseUrl, isAnonymousAuthSessionsEnabled, isAuthEnabled, isGithubAuthEnabled } from '../../src/lib/server/auth/config'; +import { withEnv } from './support/env'; + +describe('auth config contract', () => { + test('auth is enabled only when AUTH_SECRET and BASE_URL are both set', async () => { + await withEnv( + { + AUTH_SECRET: undefined, + BASE_URL: undefined, + }, + async () => { + expect(isAuthEnabled()).toBe(false); + expect(getAuthBaseUrl()).toBeNull(); + }, + ); + + await withEnv( + { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + }, + async () => { + expect(isAuthEnabled()).toBe(true); + expect(getAuthBaseUrl()).toBe('http://localhost:3003'); + }, + ); + }); + + test.each([ + { envValue: undefined, expected: false }, + { envValue: 'true', expected: true }, + { envValue: 'false', expected: false }, + { envValue: '1', expected: false }, + { envValue: 'TRUE', expected: true }, + ])( + 'anonymous sessions honor strict boolean parsing when auth is enabled (value: $envValue)', + async ({ envValue, expected }) => { + await withEnv( + { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + USE_ANONYMOUS_AUTH_SESSIONS: envValue, + }, + async () => { + expect(isAnonymousAuthSessionsEnabled()).toBe(expected); + }, + ); + }, + ); + + test('anonymous sessions are always disabled when auth is disabled', async () => { + await withEnv( + { + AUTH_SECRET: undefined, + BASE_URL: undefined, + USE_ANONYMOUS_AUTH_SESSIONS: 'true', + }, + async () => { + expect(isAnonymousAuthSessionsEnabled()).toBe(false); + }, + ); + }); + + test.each([ + { + title: 'returns false when auth is disabled', + env: { + AUTH_SECRET: undefined, + BASE_URL: undefined, + GITHUB_CLIENT_ID: 'id', + GITHUB_CLIENT_SECRET: 'secret', + }, + expected: false, + }, + { + title: 'returns false when GitHub client id is missing', + env: { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + GITHUB_CLIENT_ID: undefined, + GITHUB_CLIENT_SECRET: 'secret', + }, + expected: false, + }, + { + title: 'returns false when GitHub client secret is missing', + env: { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + GITHUB_CLIENT_ID: 'id', + GITHUB_CLIENT_SECRET: undefined, + }, + expected: false, + }, + { + title: 'returns true when auth is enabled and GitHub credentials are present', + env: { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + GITHUB_CLIENT_ID: 'id', + GITHUB_CLIENT_SECRET: 'secret', + }, + expected: true, + }, + ])('$title', async ({ env, expected }) => { + await withEnv(env, async () => { + expect(isGithubAuthEnabled()).toBe(expected); + }); + }); +}); diff --git a/tests/unit/canonicalize-epub-segment.spec.ts b/tests/unit/canonicalize-epub-segment.vitest.spec.ts similarity index 97% rename from tests/unit/canonicalize-epub-segment.spec.ts rename to tests/unit/canonicalize-epub-segment.vitest.spec.ts index 42a02b8..164d857 100644 --- a/tests/unit/canonicalize-epub-segment.spec.ts +++ b/tests/unit/canonicalize-epub-segment.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { canonicalizeEpubSegmentAgainstSpineText, @@ -6,7 +6,7 @@ import { } from '../../src/lib/client/epub/canonicalize-epub-segment'; import { planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan'; -test.describe('canonicalizeEpubSegmentAgainstSpineText', () => { +describe('canonicalizeEpubSegmentAgainstSpineText', () => { test('maps an exact sentence to the canonical segment identity', () => { const spineText = [ 'First section sentence with enough words to stand alone.', @@ -95,7 +95,7 @@ test.describe('canonicalizeEpubSegmentAgainstSpineText', () => { }); }); -test.describe('canonicalizeEpubSegmentsAgainstSpineText', () => { +describe('canonicalizeEpubSegmentsAgainstSpineText', () => { test('maps overlap-boundary drift sentences to forward canonical segments', () => { const sourceSentences = [ 'The star was particularly bright when the station lights switched off for cycle night.', diff --git a/tests/unit/changelog-check.spec.ts b/tests/unit/changelog-check.vitest.spec.ts similarity index 95% rename from tests/unit/changelog-check.spec.ts rename to tests/unit/changelog-check.vitest.spec.ts index f95bcbc..19d8642 100644 --- a/tests/unit/changelog-check.spec.ts +++ b/tests/unit/changelog-check.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { scheduleChangelogCheck } from '../../src/lib/client/changelog-check'; @@ -6,7 +6,7 @@ function wait(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -test.describe('changelog check scheduling', () => { +describe('changelog check scheduling', () => { test('cancels throwaway mount and runs exactly once on remount', async () => { const completedRef = { current: null as string | null }; const inFlightRef = { current: null as string | null }; diff --git a/tests/unit/changelog.spec.ts b/tests/unit/changelog.vitest.spec.ts similarity index 96% rename from tests/unit/changelog.spec.ts rename to tests/unit/changelog.vitest.spec.ts index 02117d9..3d56744 100644 --- a/tests/unit/changelog.spec.ts +++ b/tests/unit/changelog.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { findCurrentVersionIndex, @@ -10,7 +10,7 @@ import { type ChangelogManifestEntry, } from '../../src/lib/shared/changelog'; -test.describe('changelog utilities', () => { +describe('changelog utilities', () => { test('normalizes version strings with and without v prefix', () => { expect(normalizeVersion('v2.2.0')).toBe('2.2.0'); expect(normalizeVersion('2.2.0')).toBe('2.2.0'); diff --git a/tests/unit/compute-abort-like-error.spec.ts b/tests/unit/compute-abort-like-error.spec.ts deleted file mode 100644 index e31287e..0000000 --- a/tests/unit/compute-abort-like-error.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { isAbortLikeError } from '../../src/lib/server/compute/abort-like-error'; - -test.describe('isAbortLikeError', () => { - test('matches abort-shaped errors', () => { - expect(isAbortLikeError(new DOMException('This operation was aborted', 'AbortError'))).toBe(true); - expect(isAbortLikeError(Object.assign(new Error('random'), { name: 'AbortError' }))).toBe(true); - expect(isAbortLikeError(new Error('This operation was aborted'))).toBe(true); - expect(isAbortLikeError({ name: 'AbortError' })).toBe(true); - }); - - test('does not match non-abort errors', () => { - expect(isAbortLikeError(new Error('boom'))).toBe(false); - expect(isAbortLikeError({ name: 'TypeError' })).toBe(false); - expect(isAbortLikeError(null)).toBe(false); - expect(isAbortLikeError(undefined)).toBe(false); - }); -}); diff --git a/tests/unit/compute-control-plane.spec.ts b/tests/unit/compute-control-plane.spec.ts deleted file mode 100644 index 7a9e338..0000000 --- a/tests/unit/compute-control-plane.spec.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { expect, test } from '@playwright/test'; -import type { WorkerOperationRequest, WorkerOperationState } from '../../compute/core/src/api-contracts'; -import { - encodeSseFrame, - explainReplacementReason, - InMemoryOperationEventStream, - InMemoryOperationQueue, - InMemoryOperationStateStore, - OperationOrchestrator, - parseSseEventId, - parseSsePayload, - shouldReuseExistingOperation, -} from '../../compute/core/src/control-plane'; - -function buildPdfLayoutRequest(opKey: string): WorkerOperationRequest { - return { - kind: 'pdf_layout', - opKey, - payload: { - documentId: `doc-${opKey}`, - documentObjectKey: `s3://bucket/${opKey}.pdf`, - namespace: null, - }, - }; -} - -test.describe('compute control-plane', () => { - test('in-memory queue and state store support enqueue/claim/CAS', async () => { - const queue = new InMemoryOperationQueue(); - const store = new InMemoryOperationStateStore(); - - await queue.enqueue({ - jobId: 'job-1', - opId: 'op-1', - opKey: 'k-1', - kind: 'pdf_layout', - queuedAt: 1000, - payload: { documentId: 'd1', documentObjectKey: 'obj1', namespace: null }, - }); - await queue.enqueue({ - jobId: 'job-2', - opId: 'op-2', - opKey: 'k-2', - kind: 'whisper_align', - queuedAt: 1100, - payload: { text: 'hello', audioObjectKey: 'obj2' }, - }); - - expect(queue.size()).toBe(2); - expect(queue.size('pdf_layout')).toBe(1); - expect(queue.size('whisper_align')).toBe(1); - - const claimedLayout = await queue.claimNext('pdf_layout'); - expect(claimedLayout?.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(0); - - const firstCas = await store.compareAndSetOpIndex({ - opKey: 'k-1', - newOpId: 'op-1', - expectedOpId: null, - }); - const secondCas = await store.compareAndSetOpIndex({ - opKey: 'k-1', - newOpId: 'op-2', - expectedOpId: null, - }); - - expect(firstCas).toBeTruthy(); - expect(secondCas).toBeFalsy(); - expect(await store.getOpIndex('k-1')).toEqual({ opId: 'op-1' }); - }); - - test('in-memory event stream replays from sinceEventId and streams live events', async () => { - const stream = new InMemoryOperationEventStream(); - - const queued: WorkerOperationState = { - opId: 'op-1', - opKey: 'k-1', - kind: 'pdf_layout', - jobId: 'job-1', - status: 'queued', - queuedAt: 1000, - updatedAt: 1000, - }; - const running: WorkerOperationState = { - ...queued, - status: 'running', - startedAt: 1200, - updatedAt: 1200, - }; - const succeeded: WorkerOperationState = { - ...running, - status: 'succeeded', - updatedAt: 1400, - result: { ok: true }, - }; - - await stream.append('op-1', queued); - await stream.append('op-1', running); - - const receivedEventIds: number[] = []; - const unsubscribe = await stream.subscribe({ - opId: 'op-1', - sinceEventId: 1, - onEvent: (event) => { - receivedEventIds.push(event.eventId); - }, - }); - - await stream.append('op-1', succeeded); - unsubscribe(); - - expect(receivedEventIds).toEqual([2, 3]); - }); - - test('orchestrator reuses fresh inflight operations and replaces stale ones', async () => { - let now = 1_000; - let nextId = 1; - const queue = new InMemoryOperationQueue(); - const stateStore = new InMemoryOperationStateStore(); - const eventStream = new InMemoryOperationEventStream(); - const orchestrator = new OperationOrchestrator({ - queue, - stateStore, - eventStream, - config: { opStaleMs: 2_000, maxCasRetries: 5 }, - clock: { now: () => now }, - idFactory: { - opId: () => `op-${nextId}`, - jobId: () => `job-${nextId++}`, - }, - }); - - const request = buildPdfLayoutRequest('same-op-key'); - - const first = await orchestrator.enqueueOrReuse(request); - expect(first.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(1); - - now = 2_000; - const reused = await orchestrator.enqueueOrReuse(request); - expect(reused.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(1); - - await orchestrator.markRunning({ opId: first.opId, updatedAt: 2_100 }); - - now = 6_000; - const replaced = await orchestrator.enqueueOrReuse(request); - expect(replaced.opId).toBe('op-2'); - expect(queue.size('pdf_layout')).toBe(2); - expect(await stateStore.getOpIndex('same-op-key')).toEqual({ opId: 'op-2' }); - - const op1Events = await eventStream.listSince('op-1', 0); - const op2Events = await eventStream.listSince('op-2', 0); - expect(op1Events.map((event) => event.eventId)).toEqual([1, 2]); - expect(op2Events.map((event) => event.eventId)).toEqual([1]); - }); - - test('state-machine helpers return consistent reuse/replacement decisions', () => { - const current: WorkerOperationState = { - opId: 'op-1', - opKey: 'same-op-key', - kind: 'pdf_layout', - jobId: 'job-1', - status: 'running', - queuedAt: 1000, - updatedAt: 2000, - }; - - expect(shouldReuseExistingOperation({ - current, - requestKind: 'pdf_layout', - now: 2500, - opStaleMs: 1_000, - })).toBeTruthy(); - - expect(shouldReuseExistingOperation({ - current, - requestKind: 'pdf_layout', - now: 4005, - opStaleMs: 1_000, - })).toBeFalsy(); - - expect(explainReplacementReason({ - current, - requestKind: 'pdf_layout', - now: 4005, - opStaleMs: 1_000, - })).toBe('stale_running'); - }); - - test('sse helpers encode and decode frame payload and id', () => { - const frame = encodeSseFrame({ - id: 42, - event: 'snapshot', - data: { ok: true }, - }); - expect(parseSseEventId(frame)).toBe(42); - expect(parseSsePayload(frame)).toBe('{"ok":true}'); - }); -}); diff --git a/tests/unit/compute-worker-op-key.spec.ts b/tests/unit/compute-worker-op-key.spec.ts deleted file mode 100644 index 2101af6..0000000 --- a/tests/unit/compute-worker-op-key.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; - -test.describe('compute worker pdf opKey', () => { - test('keeps stable key when no force token is provided', () => { - const base = { - documentId: 'doc-123', - namespace: 'ns-1', - documentObjectKey: 'docs/ns-1/doc-123', - }; - expect(buildPdfOpKey(base)).toBe('pdf_layout|v1|doc-123|ns-1|docs/ns-1/doc-123|'); - expect(buildPdfOpKey(base)).toBe(buildPdfOpKey(base)); - }); - - test('cache-busts key when force token is provided', () => { - const base = { - documentId: 'doc-123', - namespace: 'ns-1', - documentObjectKey: 'docs/ns-1/doc-123', - }; - const opKeyA = buildPdfOpKey({ ...base, forceToken: 'force-a' }); - const opKeyB = buildPdfOpKey({ ...base, forceToken: 'force-b' }); - const normal = buildPdfOpKey(base); - - expect(opKeyA).not.toBe(opKeyB); - expect(opKeyA).not.toBe(normal); - expect(opKeyB).not.toBe(normal); - }); -}); diff --git a/tests/unit/compute-worker-pdf-progress.spec.ts b/tests/unit/compute-worker-pdf-progress.spec.ts deleted file mode 100644 index 284fd04..0000000 --- a/tests/unit/compute-worker-pdf-progress.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { - buildInferProgressForPageParsed, - buildInferProgressForPageStart, -} from '../../compute/worker/src/pdf-progress'; - -test.describe('compute worker pdf progress helpers', () => { - test('page-start progress keeps current page but does not count it as parsed yet', () => { - expect(buildInferProgressForPageStart({ pageNumber: 1, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 0, - currentPage: 1, - phase: 'infer', - }); - - expect(buildInferProgressForPageStart({ pageNumber: 5, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 4, - currentPage: 5, - phase: 'infer', - }); - }); - - test('page-parsed progress counts the current page as parsed', () => { - expect(buildInferProgressForPageParsed({ pageNumber: 5, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 5, - currentPage: 5, - phase: 'infer', - }); - }); -}); diff --git a/tests/unit/docstore.spec.ts b/tests/unit/docstore.vitest.spec.ts similarity index 79% rename from tests/unit/docstore.spec.ts rename to tests/unit/docstore.vitest.spec.ts index 7e9f144..684cc52 100644 --- a/tests/unit/docstore.spec.ts +++ b/tests/unit/docstore.vitest.spec.ts @@ -1,10 +1,10 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { getMigratedDocumentFileName } from '../../src/lib/server/storage/docstore-legacy'; -test.describe('Docstore Filename Safety', () => { +describe('docstore filename safety', () => { const id = 'a'.repeat(64); // Simulate sha256 hex ID - test('should generate standard filename for short names', () => { + test('generates stable file names for short inputs', () => { const name = 'test-file.pdf'; const result = getMigratedDocumentFileName(id, name); expect(result).toBe(`${id}__test-file.pdf`); @@ -50,4 +50,13 @@ test.describe('Docstore Filename Safety', () => { expect(result.length).toBe(240); expect(result).not.toContain('truncated-'); }); + + test('drops path traversal and null-byte fragments from migrated file names', () => { + const dirtyName = '../nested/\u0000financial-report.pdf'; + const result = getMigratedDocumentFileName(id, dirtyName); + + expect(result).toBe(`${id}__financial-report.pdf`); + expect(result).not.toContain('..'); + expect(result).not.toContain('\u0000'); + }); }); diff --git a/tests/unit/document-cache.spec.ts b/tests/unit/document-cache.vitest.spec.ts similarity index 71% rename from tests/unit/document-cache.spec.ts rename to tests/unit/document-cache.vitest.spec.ts index 061db9a..c807c39 100644 --- a/tests/unit/document-cache.spec.ts +++ b/tests/unit/document-cache.vitest.spec.ts @@ -1,16 +1,17 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import type { BaseDocument, EPUBDocument, HTMLDocument, PDFDocument } from '../../src/types/documents'; import { ensureCachedDocumentCore } from '../../src/lib/client/cache/documents'; +import { makeBaseDocument } from './support/document-fixtures'; -test.describe('document-cache-core', () => { +describe('document-cache-core', () => { test('returns cached PDF without downloading', async () => { - const meta: BaseDocument = { + const meta: BaseDocument = makeBaseDocument({ id: 'pdf1', name: 'a.pdf', size: 10, - lastModified: Date.now(), + lastModified: 1_700_000_000_001, type: 'pdf', - }; + }); let downloads = 0; const cached: PDFDocument = { ...meta, type: 'pdf', data: new ArrayBuffer(1) }; @@ -32,13 +33,13 @@ test.describe('document-cache-core', () => { }); test('downloads and stores on cache miss (EPUB)', async () => { - const meta: BaseDocument = { + const meta: BaseDocument = makeBaseDocument({ id: 'epub1', name: 'b.epub', size: 10, - lastModified: Date.now(), + lastModified: 1_700_000_000_002, type: 'epub', - }; + }); const store = new Map(); let downloads = 0; @@ -63,13 +64,13 @@ test.describe('document-cache-core', () => { }); test('downloads, decodes, and stores HTML on cache miss', async () => { - const meta: BaseDocument = { + const meta: BaseDocument = makeBaseDocument({ id: 'html1', name: 'c.txt', size: 5, - lastModified: Date.now(), + lastModified: 1_700_000_000_003, type: 'html', - }; + }); const store = new Map(); let decodedCalls = 0; @@ -92,4 +93,22 @@ test.describe('document-cache-core', () => { expect(result.type).toBe('html'); expect((result as HTMLDocument).data).toBe('hello'); }); + + test('throws deterministic cache error when download succeeds but backend misses persisted PDF', async () => { + const meta = makeBaseDocument({ + id: 'pdf-cache-miss', + type: 'pdf', + }); + + await expect( + ensureCachedDocumentCore(meta, { + get: async () => null, + putPdf: async () => { /* simulate write path without persisted row */ }, + putEpub: async () => { /* unused */ }, + putHtml: async () => { /* unused */ }, + download: async () => new Uint8Array([1, 2, 3]).buffer, + decodeText: () => '', + }), + ).rejects.toThrow('Failed to cache PDF'); + }); }); diff --git a/tests/unit/document-previews-render.spec.ts b/tests/unit/document-previews-render.vitest.spec.ts similarity index 90% rename from tests/unit/document-previews-render.spec.ts rename to tests/unit/document-previews-render.vitest.spec.ts index 85f6a31..c82ed75 100644 --- a/tests/unit/document-previews-render.spec.ts +++ b/tests/unit/document-previews-render.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import path from 'path'; import { readFile } from 'fs/promises'; import { @@ -6,7 +6,7 @@ import { renderPdfFirstPageToJpeg, } from '../../src/lib/server/documents/previews-render'; -test.describe('document-previews-render', () => { +describe('document-previews-render', () => { test('renders first PDF page to JPEG preview', async () => { const pdfPath = path.join(process.cwd(), 'tests/files/sample.pdf'); const bytes = await readFile(pdfPath); diff --git a/tests/unit/epub-audiobook-hook.spec.ts b/tests/unit/epub-audiobook-hook.vitest.spec.ts similarity index 94% rename from tests/unit/epub-audiobook-hook.spec.ts rename to tests/unit/epub-audiobook-hook.vitest.spec.ts index 8efd21b..309eb4e 100644 --- a/tests/unit/epub-audiobook-hook.spec.ts +++ b/tests/unit/epub-audiobook-hook.vitest.spec.ts @@ -1,11 +1,11 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { filterNonEmptySpineTextEntries, resolveLoadedSpineSectionDocument, } from '../../src/hooks/epub/useEPUBAudiobook'; -test.describe('EPUB audiobook hook helpers', () => { +describe('EPUB audiobook hook helpers', () => { test('resolveLoadedSpineSectionDocument prefers ownerDocument when available', () => { const ownerDocument = { body: { textContent: 'from ownerDocument' } } as unknown as Document; const loaded = { ownerDocument } as unknown; diff --git a/tests/unit/epub-location-controller.spec.ts b/tests/unit/epub-location-controller.vitest.spec.ts similarity index 92% rename from tests/unit/epub-location-controller.spec.ts rename to tests/unit/epub-location-controller.vitest.spec.ts index ca165a7..c784eab 100644 --- a/tests/unit/epub-location-controller.spec.ts +++ b/tests/unit/epub-location-controller.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { isDirectionalEpubLocation, @@ -6,7 +6,7 @@ import { shouldPersistEpubLocation, } from '../../src/hooks/epub/useEPUBLocationController'; -test.describe('EPUB location controller helpers', () => { +describe('EPUB location controller helpers', () => { test('detects directional locations', () => { expect(isDirectionalEpubLocation('next')).toBe(true); expect(isDirectionalEpubLocation('prev')).toBe(true); diff --git a/tests/unit/epub-spine-coordinates.spec.ts b/tests/unit/epub-spine-coordinates.vitest.spec.ts similarity index 97% rename from tests/unit/epub-spine-coordinates.spec.ts rename to tests/unit/epub-spine-coordinates.vitest.spec.ts index 2d8b9a5..eca51e3 100644 --- a/tests/unit/epub-spine-coordinates.spec.ts +++ b/tests/unit/epub-spine-coordinates.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import type { Book } from 'epubjs'; import { buildEpubLocator, @@ -71,7 +71,7 @@ function makeFakeBook(items: Array<{ index: number; href: string; cfiBase: strin return { book, sections }; } -test.describe('findSegmentOffset', () => { +describe('findSegmentOffset', () => { test('finds an offset for matching text', () => { const spineText = 'Hello world. This is a test paragraph for offsets.'; const offset = findSegmentOffset(spineText, 'this is a test'); @@ -98,7 +98,7 @@ test.describe('findSegmentOffset', () => { }); }); -test.describe('resolveSpineFromCfi', () => { +describe('resolveSpineFromCfi', () => { test('returns spine identity for a CFI inside a known spine', () => { const { book } = makeFakeBook([ { index: 2, href: 'OEBPS/ch02.xhtml', cfiBase: '/6/4', text: 'chapter 2 contents' }, @@ -125,7 +125,7 @@ test.describe('resolveSpineFromCfi', () => { }); }); -test.describe('buildEpubLocator', () => { +describe('buildEpubLocator', () => { test('produces a stable locator with charOffset within the spine item', async () => { const { book } = makeFakeBook([ { @@ -175,7 +175,7 @@ test.describe('buildEpubLocator', () => { }); }); -test.describe('buildEpubLocatorFromChunk', () => { +describe('buildEpubLocatorFromChunk', () => { test('uses the chunk anchor offset as a hint to disambiguate repeated text', () => { const spineText = 'foo bar foo bar foo bar'; const anchorEarly = { @@ -230,7 +230,7 @@ test.describe('buildEpubLocatorFromChunk', () => { }); }); -test.describe('buildEpubLocator anchored-search regression', () => { +describe('buildEpubLocator anchored-search regression', () => { // Pins the contract for the server-side resolver path: when called with // a `chunkText` argument representing the current rendered page, the // returned locator's `charOffset` is **at or after** the chunk's position @@ -261,7 +261,7 @@ test.describe('buildEpubLocator anchored-search regression', () => { }); }); -test.describe('findSegmentOffset fallback contract', () => { +describe('findSegmentOffset fallback contract', () => { // These tests pin the documented behavior of `findSegmentOffset`. The // from-start fallback is correct for single-shot lookups but **wrong** in // a monotonic per-sentence walk — `resolveMonotonicSentenceOffsets` exists @@ -278,7 +278,7 @@ test.describe('findSegmentOffset fallback contract', () => { }); }); -test.describe('resolveMonotonicSentenceOffsets', () => { +describe('resolveMonotonicSentenceOffsets', () => { test('returns monotonically non-decreasing offsets for sentences in order', () => { const spineText = 'The cat sat on the mat. The dog barked loudly. Then it was quiet.'; const sentences = [ @@ -347,7 +347,7 @@ test.describe('resolveMonotonicSentenceOffsets', () => { }); }); -test.describe('getSpineItemPlainText shape handling', () => { +describe('getSpineItemPlainText shape handling', () => { // Regression: epubjs's `Section.load()` resolves to the spine item's // `` Element (NOT a Document). Reading `loaded.body` directly was // returning undefined for every spine item, so spineText came back '' diff --git a/tests/unit/epub-word-highlight.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts similarity index 95% rename from tests/unit/epub-word-highlight.spec.ts rename to tests/unit/epub-word-highlight.vitest.spec.ts index dc9720a..8899dc8 100644 --- a/tests/unit/epub-word-highlight.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildMonotonicWordToTokenMap, @@ -27,7 +27,7 @@ const alignmentWords = (words: string[]): TTSSentenceAlignment['words'] => charEnd: word.length, })); -test.describe('EPUB word highlight mapping', () => { +describe('EPUB word highlight mapping', () => { test('tokenizes canonical segment words with source offsets', () => { const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); diff --git a/tests/unit/html-audiobook-adapter.spec.ts b/tests/unit/html-audiobook-adapter.vitest.spec.ts similarity index 95% rename from tests/unit/html-audiobook-adapter.spec.ts rename to tests/unit/html-audiobook-adapter.vitest.spec.ts index 9395533..7fda86c 100644 --- a/tests/unit/html-audiobook-adapter.spec.ts +++ b/tests/unit/html-audiobook-adapter.vitest.spec.ts @@ -1,11 +1,11 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { createHtmlAudiobookSourceAdapter } from '../../src/lib/client/audiobooks/adapters/html'; import { parseHtmlBlocks, type HtmlBlock } from '../../src/lib/client/html/blocks'; const blocksFromMd = (src: string): HtmlBlock[] => parseHtmlBlocks(src, false); const blocksFromTxt = (src: string): HtmlBlock[] => parseHtmlBlocks(src, true); -test.describe('createHtmlAudiobookSourceAdapter (markdown chapter splitting)', () => { +describe('createHtmlAudiobookSourceAdapter (markdown chapter splitting)', () => { test('starts a new chapter at each h1/h2 heading by default', async () => { const blocks = blocksFromMd( [ @@ -95,7 +95,7 @@ test.describe('createHtmlAudiobookSourceAdapter (markdown chapter splitting)', ( }); }); -test.describe('createHtmlAudiobookSourceAdapter (txt chapter splitting)', () => { +describe('createHtmlAudiobookSourceAdapter (txt chapter splitting)', () => { test('chunks TXT documents into "Part N" of 50 blocks by default', async () => { const blocks = blocksFromTxt( Array.from({ length: 120 }, (_, i) => `Block ${i + 1}.`).join('\n\n'), @@ -130,7 +130,7 @@ test.describe('createHtmlAudiobookSourceAdapter (txt chapter splitting)', () => }); }); -test.describe('createHtmlAudiobookSourceAdapter — prepareChapter', () => { +describe('createHtmlAudiobookSourceAdapter — prepareChapter', () => { test('returns the same chapter by index that prepareChapters lists', async () => { const blocks = blocksFromMd(['# A', '', 'one', '', '## B', '', 'two'].join('\n')); const adapter = createHtmlAudiobookSourceAdapter({ diff --git a/tests/unit/html-blocks.spec.ts b/tests/unit/html-blocks.vitest.spec.ts similarity index 94% rename from tests/unit/html-blocks.spec.ts rename to tests/unit/html-blocks.vitest.spec.ts index 533d153..02f6b0e 100644 --- a/tests/unit/html-blocks.spec.ts +++ b/tests/unit/html-blocks.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { mdToPlainText, parseHtmlBlocks, @@ -6,7 +6,7 @@ import { splitTxtBlocks, } from '../../src/lib/client/html/blocks'; -test.describe('parseHtmlBlocks (markdown)', () => { +describe('parseHtmlBlocks (markdown)', () => { test('splits headings, paragraphs, and lists into separate blocks', () => { const src = [ '# Title', @@ -55,7 +55,7 @@ test.describe('parseHtmlBlocks (markdown)', () => { }); }); -test.describe('parseHtmlBlocks (txt)', () => { +describe('parseHtmlBlocks (txt)', () => { test('splits on blank-line boundaries and preserves intra-block whitespace', () => { const src = 'First block\nmore.\n\nSecond block.\n\n\nThird block.'; const blocks = parseHtmlBlocks(src, true); @@ -73,7 +73,7 @@ test.describe('parseHtmlBlocks (txt)', () => { }); }); -test.describe('mdToPlainText badge/image stripping', () => { +describe('mdToPlainText badge/image stripping', () => { // The bug: badge alt text was being kept in plainText. Since the rendered // DOM is just an with no text node, the sentence-highlight pattern // matcher couldn't find those words and the WHOLE first-segment match @@ -124,7 +124,7 @@ test.describe('mdToPlainText badge/image stripping', () => { }); }); -test.describe('buildFullDocumentText-style integration (badge-only blocks)', () => { +describe('buildFullDocumentText-style integration (badge-only blocks)', () => { // If a paragraph is composed only of badges, its plainText is empty after // stripping. The reader filters empty plainText out of the TTS source // (`useHtmlDocument#buildFullDocumentText`), so badge blocks don't generate diff --git a/tests/unit/icons-grid.spec.ts b/tests/unit/icons-grid.vitest.spec.ts similarity index 91% rename from tests/unit/icons-grid.spec.ts rename to tests/unit/icons-grid.vitest.spec.ts index 5663c47..c21f95b 100644 --- a/tests/unit/icons-grid.spec.ts +++ b/tests/unit/icons-grid.vitest.spec.ts @@ -1,7 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { iconsGridStyle, maxColumnsForIconGrid } from '../../src/components/doclist/views/iconsGrid'; -test.describe('icons grid layout', () => { +describe('icons grid layout', () => { test('calculates max columns from width and icon size', () => { expect(maxColumnsForIconGrid('md', 136)).toBe(1); expect(maxColumnsForIconGrid('md', 300)).toBe(2); diff --git a/tests/unit/kokoro.spec.ts b/tests/unit/kokoro.vitest.spec.ts similarity index 89% rename from tests/unit/kokoro.spec.ts rename to tests/unit/kokoro.vitest.spec.ts index 3860c1f..5f1950a 100644 --- a/tests/unit/kokoro.spec.ts +++ b/tests/unit/kokoro.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { getMaxVoicesForProvider } from '../../src/lib/shared/kokoro'; -test.describe('kokoro voice limits', () => { +describe('kokoro voice limits', () => { test('keeps Replicate single-voice even for Kokoro models', () => { expect(getMaxVoicesForProvider('replicate', 'kokoro')).toBe(1); expect(getMaxVoicesForProvider('replicate', 'hexgrad/Kokoro-82M')).toBe(1); diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.vitest.spec.ts similarity index 96% rename from tests/unit/nlp.spec.ts rename to tests/unit/nlp.vitest.spec.ts index ce65ef0..4753b20 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { preprocessSentenceForAudio, splitTextToTtsBlocks, @@ -18,7 +18,7 @@ const expectNormalizedBlocks = (blocks: string[], maxLen = Number.POSITIVE_INFIN } }; -test.describe('preprocessSentenceForAudio', () => { +describe('preprocessSentenceForAudio', () => { test('normalizes common extraction artifacts', () => { const cases: Array<{ input: string; expected: string }> = [ { @@ -45,7 +45,7 @@ test.describe('preprocessSentenceForAudio', () => { }); }); -test.describe('splitTextToTtsBlocks (PDF-oriented)', () => { +describe('splitTextToTtsBlocks (PDF-oriented)', () => { test('returns [] for empty input', () => { expect(splitTextToTtsBlocks('')).toEqual([]); expect(splitTextToTtsBlocks(' ')).toEqual([]); @@ -160,7 +160,7 @@ test.describe('splitTextToTtsBlocks (PDF-oriented)', () => { }); }); -test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { +describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { test('returns [] for empty input', () => { expect(splitTextToTtsBlocksEPUB('')).toEqual([]); expect(splitTextToTtsBlocksEPUB(' ')).toEqual([]); @@ -193,7 +193,7 @@ test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { }); }); -test.describe('normalizeTextForTts', () => { +describe('normalizeTextForTts', () => { test('returns a single normalized string without newlines', () => { const input = 'Hello.\nWorld.\n\nNext paragraph.'; const normalized = normalizeTextForTts(input); diff --git a/tests/unit/onboarding-flow.spec.ts b/tests/unit/onboarding-flow.vitest.spec.ts similarity index 94% rename from tests/unit/onboarding-flow.spec.ts rename to tests/unit/onboarding-flow.vitest.spec.ts index 9533aa8..27eaaa6 100644 --- a/tests/unit/onboarding-flow.spec.ts +++ b/tests/unit/onboarding-flow.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { createCoalescedAsyncRunner, resolveNextOnboardingStep } from '../../src/lib/client/onboarding-flow'; -test.describe('onboarding flow resolver', () => { +describe('onboarding flow resolver', () => { test('resolves deterministic order with privacy first', () => { const step = resolveNextOnboardingStep({ privacyRequired: true, @@ -78,7 +78,7 @@ test.describe('onboarding flow resolver', () => { }); }); -test.describe('coalesced onboarding runner', () => { +describe('coalesced onboarding runner', () => { test('coalesces concurrent triggers into one extra rerun', async () => { let runs = 0; let nestedRequested = false; diff --git a/tests/unit/onboarding-state.spec.ts b/tests/unit/onboarding-state.vitest.spec.ts similarity index 90% rename from tests/unit/onboarding-state.spec.ts rename to tests/unit/onboarding-state.vitest.spec.ts index 9379c2a..c457b92 100644 --- a/tests/unit/onboarding-state.spec.ts +++ b/tests/unit/onboarding-state.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { ONBOARDING_STATE_REGISTRY } from '../../src/lib/shared/onboarding-state'; import { SYNCED_PREFERENCE_KEYS } from '../../src/types/user-state'; -test.describe('onboarding state storage scopes', () => { +describe('onboarding state storage scopes', () => { test('keeps local onboarding flags out of synced server preferences', () => { expect(ONBOARDING_STATE_REGISTRY.privacyAccepted.scope).toBe('local-dexie'); expect(ONBOARDING_STATE_REGISTRY.firstVisitSettingsOpened.scope).toBe('local-dexie'); diff --git a/tests/unit/pdf-audiobook-adapter.spec.ts b/tests/unit/pdf-audiobook-adapter.vitest.spec.ts similarity index 96% rename from tests/unit/pdf-audiobook-adapter.spec.ts rename to tests/unit/pdf-audiobook-adapter.vitest.spec.ts index dd110b5..17cebce 100644 --- a/tests/unit/pdf-audiobook-adapter.spec.ts +++ b/tests/unit/pdf-audiobook-adapter.vitest.spec.ts @@ -1,15 +1,15 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { createPdfAudiobookSourceAdapter } from '../../src/lib/client/audiobooks/adapters/pdf'; import type { ParsedPdfDocument } from '../../src/types/parsed-pdf'; import type { DocumentSettings } from '../../src/types/document-settings'; -test.describe('pdf audiobook adapter', () => { +describe('pdf audiobook adapter', () => { test('builds chapters from paragraph titles and filters skipped kinds', async () => { const parsed: ParsedPdfDocument = { schemaVersion: 1, documentId: 'doc-1', parserVersion: 'test', - parsedAt: Date.now(), + parsedAt: 1_700_000_000_000, pages: [ { pageNumber: 1, @@ -77,7 +77,7 @@ test.describe('pdf audiobook adapter', () => { schemaVersion: 1, documentId: 'doc-2', parserVersion: 'test', - parsedAt: Date.now(), + parsedAt: 1_700_000_000_001, pages: [ { pageNumber: 1, diff --git a/tests/unit/pdf-build-page-text.spec.ts b/tests/unit/pdf-build-page-text.vitest.spec.ts similarity index 92% rename from tests/unit/pdf-build-page-text.spec.ts rename to tests/unit/pdf-build-page-text.vitest.spec.ts index 247507c..6604d2d 100644 --- a/tests/unit/pdf-build-page-text.spec.ts +++ b/tests/unit/pdf-build-page-text.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildPageTextFromBlocks } from '../../src/lib/client/pdf-block-text'; import type { ParsedPdfPage } from '../../src/types/parsed-pdf'; -test.describe('buildPageTextFromBlocks', () => { +describe('buildPageTextFromBlocks', () => { test('filters skipped kinds and preserves reading order', () => { const page: ParsedPdfPage = { pageNumber: 1, diff --git a/tests/unit/pdf-force-reparse.spec.ts b/tests/unit/pdf-force-reparse.vitest.spec.ts similarity index 89% rename from tests/unit/pdf-force-reparse.spec.ts rename to tests/unit/pdf-force-reparse.vitest.spec.ts index 95c2ce4..6db012b 100644 --- a/tests/unit/pdf-force-reparse.spec.ts +++ b/tests/unit/pdf-force-reparse.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { FORCE_REPARSE_CONFIRM_MESSAGE, FORCE_REPARSE_CONFIRM_TEXT, @@ -6,7 +6,7 @@ import { isForceReparseDisabled, } from '../../src/lib/client/pdf/force-reparse'; -test.describe('pdf force reparse controls', () => { +describe('pdf force reparse controls', () => { test('disables action while parse is pending or running', () => { expect(isForceReparseDisabled('pending')).toBeTruthy(); expect(isForceReparseDisabled('running')).toBeTruthy(); diff --git a/tests/unit/pdf-merge-text-with-regions.spec.ts b/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts similarity index 59% rename from tests/unit/pdf-merge-text-with-regions.spec.ts rename to tests/unit/pdf-merge-text-with-regions.vitest.spec.ts index 86bd435..135e9c3 100644 --- a/tests/unit/pdf-merge-text-with-regions.spec.ts +++ b/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts @@ -1,7 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { mergeTextWithRegions } from '@openreader/compute-core'; -test.describe('mergeTextWithRegions', () => { +describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { const regions = [ { bbox: [0, 0, 100, 50] as [number, number, number, number], label: 'text' as const }, @@ -20,4 +20,18 @@ test.describe('mergeTextWithRegions', () => { expect(merged[0].text).toBe('hello world'); expect(merged[1].text).toBe('Figure 1.2'); }); + + test('drops text whose centroid is outside every region', () => { + const regions = [ + { bbox: [0, 0, 50, 50] as [number, number, number, number], label: 'text' as const }, + ]; + const textItems = [ + { text: 'inside', x: 10, y: 10, width: 10, height: 8 }, + { text: 'outside', x: 80, y: 80, width: 12, height: 8 }, + ]; + + const merged = mergeTextWithRegions(regions, textItems); + expect(merged).toHaveLength(1); + expect(merged[0].text).toBe('inside'); + }); }); diff --git a/tests/unit/pdf-parse-normalize-text-items.spec.ts b/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts similarity index 69% rename from tests/unit/pdf-parse-normalize-text-items.spec.ts rename to tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts index aac976c..3ceccd0 100644 --- a/tests/unit/pdf-parse-normalize-text-items.spec.ts +++ b/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { normalizeTextItemsForLayout } from '@openreader/compute-core'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; @@ -19,7 +19,7 @@ function makeTextItem( } as unknown as TextItem; } -test.describe('normalizeTextItemsForLayout', () => { +describe('normalizeTextItemsForLayout', () => { test('keeps horizontal body text and drops rotated/skewed margin text', () => { const horizontal = makeTextItem( 'Powered by large language models', @@ -36,4 +36,12 @@ test.describe('normalizeTextItemsForLayout', () => { expect(normalized).toHaveLength(1); expect(normalized[0]?.text).toBe('Powered by large language models'); }); + + test('drops malformed/vertical-only runs so downstream layout planning sees no body text', () => { + const vertical = makeTextItem('Side label', [0, 10, -10, 0, 30, 200]); + const skewed = makeTextItem('Watermark', [10, 5, 2, 10, 200, 500]); + + const normalized = normalizeTextItemsForLayout([vertical, skewed], 800); + expect(normalized).toEqual([]); + }); }); diff --git a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts b/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts similarity index 89% rename from tests/unit/pdf-stitch-cross-page-blocks.spec.ts rename to tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts index 86e6d4a..0aa37e6 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts +++ b/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts @@ -1,6 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { stitchCrossPageBlocks } from '@openreader/compute-core'; import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf'; +import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( id: string, @@ -23,19 +24,13 @@ function makeBlock( } function makeDoc(page1Blocks: ParsedPdfBlock[], page2Blocks: ParsedPdfBlock[]): ParsedPdfDocument { - return { - schemaVersion: 1, - documentId: 'doc', - parserVersion: 'test', - parsedAt: 0, - pages: [ - { pageNumber: 1, width: 100, height: 100, blocks: page1Blocks }, - { pageNumber: 2, width: 100, height: 100, blocks: page2Blocks }, - ], - }; + return makeParsedPdfDocument([ + makeParsedPdfPage(1, page1Blocks), + makeParsedPdfPage(2, page2Blocks), + ]); } -test.describe('stitchCrossPageBlocks', () => { +describe('stitchCrossPageBlocks', () => { test('stitches paragraph continuation across footer/header noise', () => { const doc = makeDoc( [ diff --git a/tests/unit/pdf-tts-planning.spec.ts b/tests/unit/pdf-tts-planning.vitest.spec.ts similarity index 97% rename from tests/unit/pdf-tts-planning.spec.ts rename to tests/unit/pdf-tts-planning.vitest.spec.ts index 1066d7b..c9314dd 100644 --- a/tests/unit/pdf-tts-planning.spec.ts +++ b/tests/unit/pdf-tts-planning.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildPdfPageSourceUnits, buildPdfPrefetchPayload } from '../../src/lib/client/pdf-tts-planning'; import type { ParsedPdfPage } from '../../src/types/parsed-pdf'; @@ -39,7 +39,7 @@ function buildPage(pageNumber: number): ParsedPdfPage { }; } -test.describe('pdf tts planning helpers', () => { +describe('pdf tts planning helpers', () => { test('buildPdfPageSourceUnits uses parsed blocks, honors skip kinds, and pins locator page', () => { const page = buildPage(2); const units = buildPdfPageSourceUnits(page, 2, ['header']); diff --git a/tests/unit/rate-limit-runtime-settings.vitest.spec.ts b/tests/unit/rate-limit-runtime-settings.vitest.spec.ts new file mode 100644 index 0000000..b39f3e9 --- /dev/null +++ b/tests/unit/rate-limit-runtime-settings.vitest.spec.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from 'vitest'; + +import { RUNTIME_CONFIG_SCHEMA } from '../../src/lib/server/admin/settings'; + +describe('TTS rate limit runtime config seeds', () => { + test('defaults disable TTS daily rate limiting', () => { + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.default).toBe(true); + }); + + test('parses disable seed boolean values', () => { + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('true')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('false')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('1')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('0')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('on')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('no')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('invalid')).toBeUndefined(); + }); + + test('daily limit values are runtime-only (no env seed vars)', () => { + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAnonymous.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAuthenticated.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsIpDailyLimitAnonymous.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsIpDailyLimitAuthenticated.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAnonymous.default).toBe(50_000); + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAuthenticated.default).toBe(500_000); + }); +}); diff --git a/tests/unit/request-ip.vitest.spec.ts b/tests/unit/request-ip.vitest.spec.ts new file mode 100644 index 0000000..ee51281 --- /dev/null +++ b/tests/unit/request-ip.vitest.spec.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from 'vitest'; +import type { NextRequest } from 'next/server'; +import { getClientIp } from '../../src/lib/server/rate-limit/request-ip'; +import { withEnv } from './support/env'; + +function makeReq(headers: Record, ip?: string): NextRequest { + const req = { headers: new Headers(headers) } as Partial & { ip?: string }; + if (ip) req.ip = ip; + return req as NextRequest; +} + +describe('getClientIp', () => { + test('never trusts the left-most (client-prependable) X-Forwarded-For entry', async () => { + await withEnv({ VERCEL: undefined }, () => { + // Attacker prepends a spoofed value; the real connecting IP is the right-most hop. + const req = makeReq({ 'x-forwarded-for': '1.2.3.4, 9.9.9.9' }); + expect(getClientIp(req)).toBe('9.9.9.9'); + }); + }); + + test('prefers x-real-ip over X-Forwarded-For', async () => { + await withEnv({ VERCEL: undefined }, () => { + const req = makeReq({ 'x-real-ip': '5.5.5.5', 'x-forwarded-for': '1.2.3.4, 9.9.9.9' }); + expect(getClientIp(req)).toBe('5.5.5.5'); + }); + }); + + test('prefers x-vercel-forwarded-for when running on Vercel', async () => { + await withEnv({ VERCEL: '1' }, () => { + const req = makeReq({ + 'x-vercel-forwarded-for': '8.8.8.8', + 'x-real-ip': '5.5.5.5', + 'x-forwarded-for': '1.2.3.4', + }); + expect(getClientIp(req)).toBe('8.8.8.8'); + }); + }); + + test('ignores x-vercel-forwarded-for when not on Vercel', async () => { + await withEnv({ VERCEL: undefined }, () => { + // A self-hosted deployment must not trust a client-supplied x-vercel-* header. + const req = makeReq({ 'x-vercel-forwarded-for': '8.8.8.8', 'x-real-ip': '5.5.5.5' }); + expect(getClientIp(req)).toBe('5.5.5.5'); + }); + }); + + test('falls back to the connecting address when no proxy headers are present', async () => { + await withEnv({ VERCEL: undefined }, () => { + const req = makeReq({}, '10.0.0.1'); + expect(getClientIp(req)).toBe('10.0.0.1'); + }); + }); +}); diff --git a/tests/unit/server-error-contract.spec.ts b/tests/unit/server-error-contract.vitest.spec.ts similarity index 62% rename from tests/unit/server-error-contract.spec.ts rename to tests/unit/server-error-contract.vitest.spec.ts index 84307d7..e489af3 100644 --- a/tests/unit/server-error-contract.spec.ts +++ b/tests/unit/server-error-contract.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { ServerAppError, createServerAppError, @@ -7,10 +7,12 @@ import { toApiErrorBody, toHttpStatus, } from '../../src/lib/server/errors/contract'; +import { makeServerAppErrorInput } from './support/factories'; -test.describe('server error contract', () => { +describe('server error contract', () => { test('normalizes unknown throwable to fallback shape', () => { const normalized = normalizeServerError('boom'); + expect(normalized.code).toBe('UNKNOWN_SERVER_ERROR'); expect(normalized.errorClass).toBe('unknown'); expect(normalized.httpStatus).toBe(500); @@ -19,15 +21,17 @@ test.describe('server error contract', () => { }); test('preserves ServerAppError metadata', () => { - const err = createServerAppError({ + const err = createServerAppError(makeServerAppErrorInput({ code: 'USER_PROGRESS_UPDATE_FAILED', message: 'Failed to update progress', errorClass: 'db', retryable: true, httpStatus: 500, details: { operation: 'update_progress' }, - }); + })); + const normalized = normalizeServerError(err); + expect(isServerAppError(err)).toBe(true); expect(normalized.code).toBe('USER_PROGRESS_UPDATE_FAILED'); expect(normalized.errorClass).toBe('db'); @@ -36,6 +40,22 @@ test.describe('server error contract', () => { expect(normalized.details?.operation).toBe('update_progress'); }); + test('normalizes partially-shaped thrown errors with code sanitization', () => { + const thrown = Object.assign(new Error('Timed out waiting for storage read'), { + code: 'not_valid_code', + errorClass: 'timeout', + httpStatus: 604, + retryable: true, + }); + + const normalized = normalizeServerError(thrown); + + expect(normalized.code).toBe('UNKNOWN_SERVER_ERROR'); + expect(normalized.errorClass).toBe('timeout'); + expect(normalized.httpStatus).toBe(599); + expect(normalized.retryable).toBe(true); + }); + test('maps normalized errors to API body + status', () => { const normalized = normalizeServerError( new ServerAppError({ @@ -44,7 +64,9 @@ test.describe('server error contract', () => { errorClass: 'upstream', }), ); + const body = toApiErrorBody(normalized, { includeDetails: false }); + expect(body).toEqual({ error: 'Upstream failure', errorCode: 'UPSTREAM_TTS_ERROR', @@ -52,4 +74,12 @@ test.describe('server error contract', () => { }); expect(toHttpStatus(normalized)).toBe(502); }); + + test('rejects invalid server error code at creation boundary', () => { + expect(() => createServerAppError(makeServerAppErrorInput({ + code: 'bad-code', + message: 'Invalid', + errorClass: 'validation', + }))).toThrow(/Invalid ServerAppError code/i); + }); }); diff --git a/tests/unit/server-error-response.spec.ts b/tests/unit/server-error-response.spec.ts deleted file mode 100644 index a5ed779..0000000 --- a/tests/unit/server-error-response.spec.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { errorResponse } from '../../src/lib/server/errors/next-response'; -import { createServerAppError } from '../../src/lib/server/errors/contract'; - -test.describe('server error response helper', () => { - test('returns mapped 4xx response with explicit app code', async () => { - const response = errorResponse( - createServerAppError({ - code: 'AUTH_UNAUTHORIZED', - message: 'Unauthorized', - errorClass: 'auth', - httpStatus: 401, - retryable: false, - }), - { - apiErrorMessage: 'Unauthorized', - }, - ); - - expect(response.status).toBe(401); - await expect(response.json()).resolves.toEqual({ - error: 'Unauthorized', - errorCode: 'AUTH_UNAUTHORIZED', - retryable: false, - }); - }); - - test('normalizes unknown errors to safe 500 without stack leakage', async () => { - const response = errorResponse(new Error('sensitive internal details'), { - apiErrorMessage: 'Internal Server Error', - normalize: { code: 'UNKNOWN_SERVER_ERROR', errorClass: 'unknown' }, - }); - const body = await response.json(); - expect(response.status).toBe(500); - expect(body).toEqual({ - error: 'Internal Server Error', - errorCode: 'UNKNOWN_SERVER_ERROR', - retryable: false, - }); - expect(JSON.stringify(body)).not.toContain('stack'); - expect(JSON.stringify(body)).not.toContain('cause'); - }); -}); diff --git a/tests/unit/server-error-response.vitest.spec.ts b/tests/unit/server-error-response.vitest.spec.ts new file mode 100644 index 0000000..5bfba1d --- /dev/null +++ b/tests/unit/server-error-response.vitest.spec.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'vitest'; +import { createServerAppError } from '../../src/lib/server/errors/contract'; +import { asRouteErrorContext, errorResponse, withErrorBoundary } from '../../src/lib/server/errors/next-response'; +import { createServerLoggerStub } from './support/server-stubs'; +import { makeServerAppErrorInput } from './support/factories'; + +describe('server error response helper', () => { + test('returns mapped 4xx response with explicit app code', async () => { + const response = errorResponse( + createServerAppError(makeServerAppErrorInput({ + code: 'AUTH_UNAUTHORIZED', + message: 'Unauthorized', + errorClass: 'auth', + httpStatus: 401, + retryable: false, + })), + { + apiErrorMessage: 'Unauthorized', + }, + ); + + expect(response.status).toBe(401); + await expect(response.json()).resolves.toEqual({ + error: 'Unauthorized', + errorCode: 'AUTH_UNAUTHORIZED', + retryable: false, + }); + }); + + test('normalizes unknown errors to safe 500 without stack leakage', async () => { + const response = errorResponse(new Error('sensitive internal details'), { + apiErrorMessage: 'Internal Server Error', + normalize: { code: 'UNKNOWN_SERVER_ERROR', errorClass: 'unknown' }, + }); + const body = await response.json(); + expect(response.status).toBe(500); + expect(body).toEqual({ + error: 'Internal Server Error', + errorCode: 'UNKNOWN_SERVER_ERROR', + retryable: false, + }); + expect(JSON.stringify(body)).not.toContain('stack'); + expect(JSON.stringify(body)).not.toContain('cause'); + }); + + test('withErrorBoundary returns normalized error response and emits a structured log', async () => { + const logger = createServerLoggerStub(); + const handler = withErrorBoundary( + async () => { + throw new Error('storage dependency timed out'); + }, + { + route: '/api/unit-test', + logger: logger as never, + event: 'unit_test.route.failed', + msg: 'Route failed', + normalize: { + code: 'DOCUMENT_BLOB_FETCH_FAILED', + errorClass: 'storage', + }, + apiErrorMessage: 'Failed to fetch document blob', + }, + ); + + const response = await handler(); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body).toEqual({ + error: 'Failed to fetch document blob', + errorCode: 'DOCUMENT_BLOB_FETCH_FAILED', + retryable: true, + }); + expect(logger.error).toHaveBeenCalledTimes(1); + }); + + test('asRouteErrorContext derives method and request id consistently', () => { + const context = asRouteErrorContext({ + request: { method: 'PATCH' } as never, + route: '/api/runtime-config', + requestId: 'req-123', + }); + + expect(context).toEqual({ + route: '/api/runtime-config', + method: 'PATCH', + requestId: 'req-123', + }); + }); +}); diff --git a/tests/unit/server-route-error-mappings.spec.ts b/tests/unit/server-route-error-mappings.spec.ts deleted file mode 100644 index ac1a131..0000000 --- a/tests/unit/server-route-error-mappings.spec.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { errorResponse } from '../../src/lib/server/errors/next-response'; -import { AdminProviderError } from '../../src/lib/server/admin/providers'; - -test.describe('route error mapping contract', () => { - test('admin provider validation errors map to 4xx via route normalize policy', async () => { - const adminError = new AdminProviderError('slug is required', 400); - const response = errorResponse(adminError, { - apiErrorMessage: adminError.message, - normalize: { - code: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', - errorClass: 'validation', - httpStatus: adminError.status, - retryable: false, - }, - }); - - expect(response.status).toBe(400); - await expect(response.json()).resolves.toEqual({ - error: 'slug is required', - errorCode: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', - retryable: false, - }); - }); - - test('documents storage failures map to retryable 503', async () => { - const response = errorResponse(new Error('blobstore timeout'), { - apiErrorMessage: 'Failed to fetch document blob', - normalize: { - code: 'DOCUMENT_BLOB_FETCH_FAILED', - errorClass: 'storage', - }, - }); - - expect(response.status).toBe(503); - await expect(response.json()).resolves.toEqual({ - error: 'Failed to fetch document blob', - errorCode: 'DOCUMENT_BLOB_FETCH_FAILED', - retryable: true, - }); - }); - - test('audiobook upstream failures map to retryable 502', async () => { - const response = errorResponse(new Error('provider overloaded'), { - apiErrorMessage: 'Failed to process audio chapter', - normalize: { - code: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', - errorClass: 'upstream', - }, - }); - - expect(response.status).toBe(502); - await expect(response.json()).resolves.toEqual({ - error: 'Failed to process audio chapter', - errorCode: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', - retryable: true, - }); - }); - - test('user export auth initialization failure maps to 500 auth classification', async () => { - const response = errorResponse(new Error('Auth not initialized'), { - apiErrorMessage: 'Auth not initialized', - normalize: { - code: 'USER_EXPORT_AUTH_NOT_INITIALIZED', - errorClass: 'auth', - httpStatus: 500, - retryable: false, - }, - }); - - expect(response.status).toBe(500); - await expect(response.json()).resolves.toEqual({ - error: 'Auth not initialized', - errorCode: 'USER_EXPORT_AUTH_NOT_INITIALIZED', - retryable: false, - }); - }); -}); diff --git a/tests/unit/server-route-error-mappings.vitest.spec.ts b/tests/unit/server-route-error-mappings.vitest.spec.ts new file mode 100644 index 0000000..6d5bb23 --- /dev/null +++ b/tests/unit/server-route-error-mappings.vitest.spec.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from 'vitest'; +import { AdminProviderError } from '../../src/lib/server/admin/providers'; +import { errorResponse } from '../../src/lib/server/errors/next-response'; +import { makeServerErrorContext } from './support/factories'; + +describe('route error mapping contract', () => { + test('admin provider validation errors map to 4xx via route normalize policy', async () => { + const adminError = new AdminProviderError('slug is required', 400); + const response = errorResponse(adminError, { + apiErrorMessage: adminError.message, + normalize: makeServerErrorContext({ + code: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', + errorClass: 'validation', + httpStatus: adminError.status, + retryable: false, + }), + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toEqual({ + error: 'slug is required', + errorCode: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', + retryable: false, + }); + }); + + test.each([ + { + name: 'documents storage failures map to retryable 503', + thrown: new Error('blobstore timeout'), + apiErrorMessage: 'Failed to fetch document blob', + normalize: makeServerErrorContext({ + code: 'DOCUMENT_BLOB_FETCH_FAILED', + errorClass: 'storage', + }), + expectedStatus: 503, + expectedBody: { + error: 'Failed to fetch document blob', + errorCode: 'DOCUMENT_BLOB_FETCH_FAILED', + retryable: true, + }, + }, + { + name: 'audiobook upstream failures map to retryable 502', + thrown: new Error('provider overloaded'), + apiErrorMessage: 'Failed to process audio chapter', + normalize: makeServerErrorContext({ + code: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', + errorClass: 'upstream', + }), + expectedStatus: 502, + expectedBody: { + error: 'Failed to process audio chapter', + errorCode: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', + retryable: true, + }, + }, + { + name: 'user export auth initialization failure maps to 500 auth classification', + thrown: new Error('Auth not initialized'), + apiErrorMessage: 'Auth not initialized', + normalize: makeServerErrorContext({ + code: 'USER_EXPORT_AUTH_NOT_INITIALIZED', + errorClass: 'auth', + httpStatus: 500, + retryable: false, + }), + expectedStatus: 500, + expectedBody: { + error: 'Auth not initialized', + errorCode: 'USER_EXPORT_AUTH_NOT_INITIALIZED', + retryable: false, + }, + }, + ])('$name', async ({ thrown, apiErrorMessage, normalize, expectedStatus, expectedBody }) => { + const response = errorResponse(thrown, { apiErrorMessage, normalize }); + expect(response.status).toBe(expectedStatus); + await expect(response.json()).resolves.toEqual(expectedBody); + }); +}); diff --git a/tests/unit/sha256.spec.ts b/tests/unit/sha256.vitest.spec.ts similarity index 91% rename from tests/unit/sha256.spec.ts rename to tests/unit/sha256.vitest.spec.ts index 64796a3..1a44fec 100644 --- a/tests/unit/sha256.spec.ts +++ b/tests/unit/sha256.vitest.spec.ts @@ -1,10 +1,10 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { sha256HexSoftwareFromBytes, sha256HexFromString } from '../../src/lib/client/sha256'; -test.describe('SHA256 Software Implementation', () => { +describe('SHA256 Software Implementation', () => { // Known test vectors // "" -> e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 // "abc" -> ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad diff --git a/tests/unit/shared-provider-selection.vitest.spec.ts b/tests/unit/shared-provider-selection.vitest.spec.ts new file mode 100644 index 0000000..69d9eb2 --- /dev/null +++ b/tests/unit/shared-provider-selection.vitest.spec.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from 'vitest'; + +import { resolvePreferredSharedProviderSlug } from '../../src/lib/shared/shared-provider-selection'; +import { makeSharedProviders } from './support/factories'; + +const PROVIDERS = makeSharedProviders(['shared-a', 'default-openai', 'shared-b']); + +describe('resolvePreferredSharedProviderSlug', () => { + test('prefers requested shared slug when present', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: 'shared-b', + runtimeDefaultSlug: 'shared-a', + })).toBe('shared-b'); + }); + + test('falls back to runtime default shared slug when requested is missing', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: 'missing-slug', + runtimeDefaultSlug: 'shared-a', + })).toBe('shared-a'); + }); + + test('falls back to default-openai before first provider', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: null, + runtimeDefaultSlug: null, + })).toBe('default-openai'); + }); + + test('ignores built-in provider ids for requested/runtime defaults', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: 'openai', + runtimeDefaultSlug: 'custom-openai', + })).toBe('default-openai'); + }); + + test('falls back to first enabled provider when default-openai is missing', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: [{ slug: 'shared-z' }, { slug: 'shared-y' }], + requestedSlug: null, + runtimeDefaultSlug: null, + })).toBe('shared-z'); + }); +}); diff --git a/tests/unit/signup-policy.spec.ts b/tests/unit/signup-policy.vitest.spec.ts similarity index 76% rename from tests/unit/signup-policy.spec.ts rename to tests/unit/signup-policy.vitest.spec.ts index 190ef67..a74ba83 100644 --- a/tests/unit/signup-policy.spec.ts +++ b/tests/unit/signup-policy.vitest.spec.ts @@ -1,9 +1,9 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { RUNTIME_CONFIG_SCHEMA } from '../../src/lib/server/admin/settings'; import { assertUserSignupAllowed } from '../../src/lib/server/auth/signup-policy'; -test.describe('enableUserSignups runtime config', () => { +describe('enableUserSignups runtime config', () => { test('defaults to enabled', () => { expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.default).toBe(true); }); @@ -13,10 +13,13 @@ test.describe('enableUserSignups runtime config', () => { expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('false')).toBe(false); expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('1')).toBe(true); expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('0')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('yes')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('off')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('maybe')).toBeUndefined(); }); }); -test.describe('signup policy enforcement', () => { +describe('signup policy enforcement', () => { test('allows new non-anonymous users when signups are enabled', () => { expect(() => assertUserSignupAllowed({ enableUserSignups: true, isAnonymous: false })).not.toThrow(); }); diff --git a/tests/unit/support/document-fixtures.ts b/tests/unit/support/document-fixtures.ts new file mode 100644 index 0000000..aefe931 --- /dev/null +++ b/tests/unit/support/document-fixtures.ts @@ -0,0 +1,54 @@ +import type { BaseDocument } from '../../../src/types/documents'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../src/types/parsed-pdf'; + +export function makeBaseDocument(overrides: Partial = {}): BaseDocument { + return { + id: 'doc-1', + name: 'document.pdf', + size: 1_024, + lastModified: 1_700_000_000_000, + type: 'pdf', + ...overrides, + }; +} + +export function makeParsedPdfBlock(input: { + id: string; + kind: ParsedPdfBlockKind; + text: string; + page: number; + readingOrder: number; +}): ParsedPdfBlock { + return { + id: input.id, + kind: input.kind, + text: input.text, + fragments: [ + { + page: input.page, + bbox: [0, 0, 100, 10], + text: input.text, + readingOrder: input.readingOrder, + }, + ], + }; +} + +export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage { + return { + pageNumber, + width: 800, + height: 1_200, + blocks, + }; +} + +export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument { + return { + schemaVersion: 1, + documentId: 'doc-fixture', + parserVersion: 'test', + parsedAt: 1_700_000_000_000, + pages, + }; +} diff --git a/tests/unit/support/env.ts b/tests/unit/support/env.ts new file mode 100644 index 0000000..4f76129 --- /dev/null +++ b/tests/unit/support/env.ts @@ -0,0 +1,30 @@ +export type EnvPatch = Record; + +export function captureEnv(keys: readonly string[]): EnvPatch { + const snapshot: EnvPatch = {}; + for (const key of keys) { + snapshot[key] = process.env[key]; + } + return snapshot; +} + +export function restoreEnv(snapshot: EnvPatch): void { + for (const [key, value] of Object.entries(snapshot)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +} + +export async function withEnv(patch: EnvPatch, run: () => T | Promise): Promise { + const keys = Object.keys(patch); + const snapshot = captureEnv(keys); + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { + return await run(); + } finally { + restoreEnv(snapshot); + } +} diff --git a/tests/unit/support/factories.ts b/tests/unit/support/factories.ts new file mode 100644 index 0000000..cb8835b --- /dev/null +++ b/tests/unit/support/factories.ts @@ -0,0 +1,30 @@ +import type { ServerErrorClass, ServerErrorContext } from '../../../src/lib/server/errors/contract'; + +export function makeServerAppErrorInput(overrides: Partial<{ + code: string; + message: string; + errorClass: ServerErrorClass; + httpStatus: number; + retryable: boolean; + details: Record; + cause: unknown; +}> = {}) { + return { + code: 'UNKNOWN_SERVER_ERROR', + message: 'Unhandled server failure', + errorClass: 'unknown' as const, + ...overrides, + }; +} + +export function makeServerErrorContext(overrides: ServerErrorContext = {}): ServerErrorContext { + return { + code: 'UNKNOWN_SERVER_ERROR', + errorClass: 'unknown', + ...overrides, + }; +} + +export function makeSharedProviders(slugs: readonly string[]): Array<{ slug: string }> { + return slugs.map((slug) => ({ slug })); +} diff --git a/tests/unit/support/server-stubs.ts b/tests/unit/support/server-stubs.ts new file mode 100644 index 0000000..c8b7e29 --- /dev/null +++ b/tests/unit/support/server-stubs.ts @@ -0,0 +1,22 @@ +import type { Mock } from 'vitest'; +import { vi } from 'vitest'; + +export interface ServerLoggerStub { + child: Mock; + debug: Mock; + error: Mock; + info: Mock; + warn: Mock; +} + +export function createServerLoggerStub(): ServerLoggerStub { + const logger = { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + child: vi.fn(), + } as ServerLoggerStub; + logger.child.mockReturnValue(logger); + return logger; +} diff --git a/tests/unit/test-namespace-gate.vitest.spec.ts b/tests/unit/test-namespace-gate.vitest.spec.ts new file mode 100644 index 0000000..b2e4eb4 --- /dev/null +++ b/tests/unit/test-namespace-gate.vitest.spec.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from 'vitest'; +import { getOpenReaderTestNamespace } from '../../src/lib/server/testing/test-namespace'; +import { withEnv } from './support/env'; + +function headers(value: string): Headers { + return new Headers({ 'x-openreader-test-namespace': value }); +} + +describe('getOpenReaderTestNamespace gate', () => { + test('honored in non-production builds', async () => { + await withEnv({ NODE_ENV: 'development', ENABLE_TEST_NAMESPACE: undefined }, () => { + expect(getOpenReaderTestNamespace(headers('chromium'))).toBe('chromium'); + }); + }); + + test('ignored on production builds without the explicit flag', async () => { + await withEnv({ NODE_ENV: 'production', ENABLE_TEST_NAMESPACE: undefined }, () => { + expect(getOpenReaderTestNamespace(headers('attacker'))).toBeNull(); + }); + }); + + test('honored on production builds when ENABLE_TEST_NAMESPACE=true (CI parity)', async () => { + await withEnv({ NODE_ENV: 'production', ENABLE_TEST_NAMESPACE: 'true' }, () => { + expect(getOpenReaderTestNamespace(headers('webkit'))).toBe('webkit'); + }); + }); +}); diff --git a/tests/unit/transfer-user-documents.spec.ts b/tests/unit/transfer-user-documents.vitest.spec.ts similarity index 95% rename from tests/unit/transfer-user-documents.spec.ts rename to tests/unit/transfer-user-documents.vitest.spec.ts index ca58e28..f1c0bed 100644 --- a/tests/unit/transfer-user-documents.spec.ts +++ b/tests/unit/transfer-user-documents.vitest.spec.ts @@ -1,11 +1,11 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; import { eq } from 'drizzle-orm'; import { documents } from '../../src/db/schema_sqlite'; import { transferUserDocuments } from '../../src/lib/server/user/claim-data'; -test.describe('transferUserDocuments', () => { +describe('transferUserDocuments', () => { test('moves document rows to new user without PK conflicts', async () => { process.env.BASE_URL = 'http://localhost:3003'; process.env.AUTH_SECRET = 'test-secret'; diff --git a/tests/unit/tts-audio-warm-cache.spec.ts b/tests/unit/tts-audio-warm-cache.vitest.spec.ts similarity index 97% rename from tests/unit/tts-audio-warm-cache.spec.ts rename to tests/unit/tts-audio-warm-cache.vitest.spec.ts index 06a5d24..4f38c73 100644 --- a/tests/unit/tts-audio-warm-cache.spec.ts +++ b/tests/unit/tts-audio-warm-cache.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { releaseWarmAudio, upsertWarmAudioEntry, @@ -23,7 +23,7 @@ function createTrackedAudio() { return { audio, calls }; } -test.describe('tts warm audio cache helpers', () => { +describe('tts warm audio cache helpers', () => { test('releaseWarmAudio clears source and invokes teardown methods', () => { const tracked = createTrackedAudio(); releaseWarmAudio(tracked.audio); diff --git a/tests/unit/tts-epub-handoff.spec.ts b/tests/unit/tts-epub-handoff.vitest.spec.ts similarity index 97% rename from tests/unit/tts-epub-handoff.spec.ts rename to tests/unit/tts-epub-handoff.vitest.spec.ts index 143e6e1..fa5fb82 100644 --- a/tests/unit/tts-epub-handoff.spec.ts +++ b/tests/unit/tts-epub-handoff.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { completedEpubBoundarySegment, @@ -23,7 +23,7 @@ const segment = ( ...overrides, }); -test.describe('EPUB boundary handoff', () => { +describe('EPUB boundary handoff', () => { test('records only completed segments that span a source boundary', () => { expect(completedEpubBoundarySegment(segment('same page'))).toBeNull(); diff --git a/tests/unit/tts-epub-preload.spec.ts b/tests/unit/tts-epub-preload.vitest.spec.ts similarity index 94% rename from tests/unit/tts-epub-preload.spec.ts rename to tests/unit/tts-epub-preload.vitest.spec.ts index d780e51..f74dd39 100644 --- a/tests/unit/tts-epub-preload.spec.ts +++ b/tests/unit/tts-epub-preload.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import type { CanonicalTtsSourceUnit } from '../../src/lib/shared/tts-segment-plan'; import { @@ -6,7 +6,7 @@ import { selectUpcomingWalkerItems, } from '../../src/lib/client/epub/tts-epub-preload'; -test.describe('EPUB walker preload helpers', () => { +describe('EPUB walker preload helpers', () => { test('selectUpcomingWalkerItems honors depth as current + (depth-1) upcoming', () => { const items = [ { cfi: 'epubcfi(/6/2[;s=a]!/4/2)' }, // same as current after normalization diff --git a/tests/unit/tts-generate.spec.ts b/tests/unit/tts-generate.vitest.spec.ts similarity index 92% rename from tests/unit/tts-generate.spec.ts rename to tests/unit/tts-generate.vitest.spec.ts index 364f6f2..a601952 100644 --- a/tests/unit/tts-generate.spec.ts +++ b/tests/unit/tts-generate.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { extractReplicateAudioUrl } from '../../src/lib/server/tts/generate'; -test.describe('replicate output URL extraction', () => { +describe('replicate output URL extraction', () => { test('returns direct URL string output', () => { expect(extractReplicateAudioUrl('https://replicate.delivery/audio.mp3')).toBe( 'https://replicate.delivery/audio.mp3' diff --git a/tests/unit/tts-locator.spec.ts b/tests/unit/tts-locator.vitest.spec.ts similarity index 96% rename from tests/unit/tts-locator.spec.ts rename to tests/unit/tts-locator.vitest.spec.ts index 19e0eee..6e8b32c 100644 --- a/tests/unit/tts-locator.spec.ts +++ b/tests/unit/tts-locator.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { compareSegmentLocators, locatorGroupKey, @@ -25,7 +25,7 @@ const epubLocator = ( ...(cfi ? { cfi } : {}), }); -test.describe('locatorGroupKey', () => { +describe('locatorGroupKey', () => { test('groups stable EPUB locators by spine index and href', () => { expect(locatorGroupKey(epubLocator('OEBPS/ch02.xhtml', 2, 0))).toBe('epub:2:OEBPS/ch02.xhtml'); // Different charOffset, same spine → same group (sidebar bucket is chapter-sized). @@ -55,7 +55,7 @@ test.describe('locatorGroupKey', () => { }); }); -test.describe('locatorIdentityKey', () => { +describe('locatorIdentityKey', () => { test('two EPUB rows in the same chapter at different charOffsets get distinct identity keys', () => { // This is the bug that caused page-N rows to swallow page-(N+1) rows in // the server-side manifest aggregator: the previous code keyed by the @@ -82,7 +82,7 @@ test.describe('locatorIdentityKey', () => { }); }); -test.describe('compareSegmentLocators', () => { +describe('compareSegmentLocators', () => { test('orders EPUB rows by spineIndex then charOffset numerically', () => { // Spine 10 must come AFTER spine 2 — a lexicographic compare would // wrongly place "10" before "2". @@ -137,7 +137,7 @@ test.describe('compareSegmentLocators', () => { }); }); -test.describe('type guards', () => { +describe('type guards', () => { test('isStableEpubLocator requires all spine fields', () => { expect(isStableEpubLocator(epubLocator('a.xhtml', 0, 0))).toBe(true); expect(isStableEpubLocator({ readerType: 'epub', spineHref: 'a.xhtml' })).toBe(false); @@ -159,7 +159,7 @@ test.describe('type guards', () => { }); }); -test.describe('normalizeEpubLocationToken', () => { +describe('normalizeEpubLocationToken', () => { test('strips whitespace and step markers but preserves identity', () => { const a = normalizeEpubLocationToken('epubcfi(/6/8!/4:0)'); const b = normalizeEpubLocationToken(' epubcfi(/6/8[;s=a]!/4:0) '); diff --git a/tests/unit/tts-provider-catalog.spec.ts b/tests/unit/tts-provider-catalog.vitest.spec.ts similarity index 99% rename from tests/unit/tts-provider-catalog.spec.ts rename to tests/unit/tts-provider-catalog.vitest.spec.ts index 3c40372..334ea16 100644 --- a/tests/unit/tts-provider-catalog.spec.ts +++ b/tests/unit/tts-provider-catalog.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { REPLICATE_KOKORO_82M_VERSIONED_MODEL, @@ -13,7 +13,7 @@ import { resolveReplicateVoiceInputKey, resolveVoices } from '../../src/lib/serv import { applyConfigUpdate, getVoicePreferenceKey } from '../../src/lib/client/config/updates'; import { buildSyncedPreferencePatch } from '../../src/lib/client/config/preferences'; -test.describe('tts provider catalog', () => { +describe('tts provider catalog', () => { test('resolves provider models with fixed Deepinfra catalog', () => { expect(resolveProviderModels('openai').map((model) => model.id)).toEqual([ 'tts-1', @@ -389,7 +389,7 @@ test.describe('tts provider catalog', () => { }); }); -test.describe('config helpers', () => { +describe('config helpers', () => { test('persists voice preferences by provider/model pair', () => { expect(getVoicePreferenceKey('openai', 'tts-1')).toBe('openai:tts-1'); diff --git a/tests/unit/tts-segment-audio-urls.spec.ts b/tests/unit/tts-segment-audio-urls.vitest.spec.ts similarity index 94% rename from tests/unit/tts-segment-audio-urls.spec.ts rename to tests/unit/tts-segment-audio-urls.vitest.spec.ts index 93b8d92..742d53a 100644 --- a/tests/unit/tts-segment-audio-urls.spec.ts +++ b/tests/unit/tts-segment-audio-urls.vitest.spec.ts @@ -1,10 +1,10 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildSegmentAudioFallbackUrl, resolveSegmentAudioUrls, } from '../../src/lib/server/tts/segment-audio-urls'; -test.describe('tts segment audio url resolution', () => { +describe('tts segment audio url resolution', () => { test('builds fallback URL with encoded params', () => { const url = buildSegmentAudioFallbackUrl('doc 1', 'segment/2'); expect(url).toBe('/api/tts/segments/audio/fallback?documentId=doc%201&segmentId=segment%2F2'); diff --git a/tests/unit/tts-segment-plan-leading.spec.ts b/tests/unit/tts-segment-plan-leading.vitest.spec.ts similarity index 98% rename from tests/unit/tts-segment-plan-leading.spec.ts rename to tests/unit/tts-segment-plan-leading.vitest.spec.ts index 689d36a..3114733 100644 --- a/tests/unit/tts-segment-plan-leading.spec.ts +++ b/tests/unit/tts-segment-plan-leading.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan'; -test.describe('planCanonicalTtsSegments – leading context handling', () => { +describe('planCanonicalTtsSegments – leading context handling', () => { test('clean boundary: splits block so current source keeps its first sentence', () => { const currentSourceKey = 'str:epubcfi(/6/10!/4/2)'; const plan = planCanonicalTtsSegments([ diff --git a/tests/unit/tts-segment-plan.spec.ts b/tests/unit/tts-segment-plan.vitest.spec.ts similarity index 98% rename from tests/unit/tts-segment-plan.spec.ts rename to tests/unit/tts-segment-plan.vitest.spec.ts index 8f03797..c9a3740 100644 --- a/tests/unit/tts-segment-plan.spec.ts +++ b/tests/unit/tts-segment-plan.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildSegmentKey, buildSegmentKeyPrefix, planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan'; -test.describe('planCanonicalTtsSegments', () => { +describe('planCanonicalTtsSegments', () => { test('emits a cross-boundary segment once and assigns it to the source where it starts', () => { const plan = planCanonicalTtsSegments([ { @@ -223,7 +223,7 @@ test.describe('planCanonicalTtsSegments', () => { }); }); -test.describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => { +describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => { // These keys are the bridge between persistence and the sidebar's merge of // synth rows with manifest rows. Both sides must produce identical keys for // the same `(documentId, readerType, text)` triple, or the sidebar will diff --git a/tests/unit/tts-segment-retry-policy.spec.ts b/tests/unit/tts-segment-retry-policy.vitest.spec.ts similarity index 93% rename from tests/unit/tts-segment-retry-policy.spec.ts rename to tests/unit/tts-segment-retry-policy.vitest.spec.ts index 0dd6e29..140e518 100644 --- a/tests/unit/tts-segment-retry-policy.spec.ts +++ b/tests/unit/tts-segment-retry-policy.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { isRetryableSegmentStatus, @@ -6,7 +6,7 @@ import { shouldDeferSegmentRetry, } from '../../src/lib/client/tts/segment-retry-policy'; -test.describe('tts segment retry policy', () => { +describe('tts segment retry policy', () => { test('marks pending/error statuses as retryable', () => { expect(isRetryableSegmentStatus('pending')).toBe(true); expect(isRetryableSegmentStatus('error')).toBe(true); diff --git a/tests/unit/tts-segments-audio-cache.spec.ts b/tests/unit/tts-segments-audio-cache.vitest.spec.ts similarity index 91% rename from tests/unit/tts-segments-audio-cache.spec.ts rename to tests/unit/tts-segments-audio-cache.vitest.spec.ts index ce09173..a1dd8d9 100644 --- a/tests/unit/tts-segments-audio-cache.spec.ts +++ b/tests/unit/tts-segments-audio-cache.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildSegmentAudioCacheHeaders, normalizeAudioByteRangeHeader, @@ -6,7 +6,7 @@ import { TTS_SEGMENT_FALLBACK_CACHE_CONTROL, } from '../../src/lib/server/tts/segments-audio'; -test.describe('tts segment audio cache headers', () => { +describe('tts segment audio cache headers', () => { test('builds fallback cache headers', () => { expect(buildSegmentAudioCacheHeaders()).toEqual({ 'Cache-Control': TTS_SEGMENT_FALLBACK_CACHE_CONTROL, diff --git a/tests/unit/tts-segments-manifest.spec.ts b/tests/unit/tts-segments-manifest.vitest.spec.ts similarity index 94% rename from tests/unit/tts-segments-manifest.spec.ts rename to tests/unit/tts-segments-manifest.vitest.spec.ts index 513ca5c..ad02218 100644 --- a/tests/unit/tts-segments-manifest.spec.ts +++ b/tests/unit/tts-segments-manifest.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { compareManifestSegments, decodeManifestCursor, @@ -6,8 +6,9 @@ import { encodeManifestCursor, parseManifestPageSize, } from '../../src/lib/server/tts/segments-manifest'; +import { locatorGroupKey, locatorIdentityKey } from '../../src/lib/shared/tts-locator'; -test.describe('tts segments manifest helpers', () => { +describe('tts segments manifest helpers', () => { test('dedupe prefers completed variant over newer pending for same settings key', () => { const variants = dedupeManifestVariants([ { @@ -125,8 +126,6 @@ test.describe('tts segments manifest helpers', () => { const sorted = rows.sort(compareManifestSegments); // Recompute groupKey using the production helper to assert the actual // grouping behavior rather than the test's hand-labeled groupKey. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { locatorGroupKey } = require('../../src/lib/shared/tts-locator'); const groupKeys = sorted.map((row) => locatorGroupKey(row.locator)); expect(groupKeys[0]).toBe(groupKeys[1]); // both ch02 expect(groupKeys[2]).not.toBe(groupKeys[0]); // ch03 is its own group @@ -141,9 +140,6 @@ test.describe('tts segments manifest helpers', () => { // rows visually "disappear" when later pages were generated. // // The fix uses `locatorIdentityKey` which includes charOffset for EPUB. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { locatorIdentityKey, locatorGroupKey } = require('../../src/lib/shared/tts-locator'); - const page1Row0 = { readerType: 'epub' as const, spineHref: 'OEBPS/ch02.xhtml', spineIndex: 2, charOffset: 0 }; const page2Row0 = { readerType: 'epub' as const, spineHref: 'OEBPS/ch02.xhtml', spineIndex: 2, charOffset: 2048 }; diff --git a/tests/unit/tts-segments.spec.ts b/tests/unit/tts-segments.vitest.spec.ts similarity index 98% rename from tests/unit/tts-segments.spec.ts rename to tests/unit/tts-segments.vitest.spec.ts index ddd2125..def8e9f 100644 --- a/tests/unit/tts-segments.spec.ts +++ b/tests/unit/tts-segments.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildProportionalAlignment, buildTtsSegmentEntryId, @@ -11,7 +11,7 @@ import { projectSegmentLocator, } from '../../src/lib/server/tts/segments'; -test.describe('tts segment helpers', () => { +describe('tts segment helpers', () => { test('builds stable settings hash', () => { const a = buildTtsSegmentSettingsHash({ providerRef: 'openai', diff --git a/tests/unit/tts-settings-admin-view-model.spec.ts b/tests/unit/tts-settings-admin-view-model.vitest.spec.ts similarity index 97% rename from tests/unit/tts-settings-admin-view-model.spec.ts rename to tests/unit/tts-settings-admin-view-model.vitest.spec.ts index c207ce6..8b1f32f 100644 --- a/tests/unit/tts-settings-admin-view-model.spec.ts +++ b/tests/unit/tts-settings-admin-view-model.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { resolveTtsSettingsViewModel } from '../../src/lib/client/settings/tts-settings'; import type { SharedProviderEntry } from '../../src/hooks/useSharedProviders'; @@ -20,7 +20,7 @@ const SHARED: SharedProviderEntry[] = [ }, ]; -test.describe('resolveTtsSettingsViewModel (admin/shared modes)', () => { +describe('resolveTtsSettingsViewModel (admin/shared modes)', () => { test('keeps default-openai selection when that shared provider exists', () => { const vm = resolveTtsSettingsViewModel({ providerRef: 'default-openai', diff --git a/tests/unit/walker-theme.spec.ts b/tests/unit/walker-theme.vitest.spec.ts similarity index 93% rename from tests/unit/walker-theme.spec.ts rename to tests/unit/walker-theme.vitest.spec.ts index c5f5000..777b963 100644 --- a/tests/unit/walker-theme.spec.ts +++ b/tests/unit/walker-theme.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildWalkerThemeRules } from '../../src/lib/client/epub/walker-theme'; -test.describe('walker theme rules', () => { +describe('walker theme rules', () => { test('always includes foreground/background colors', () => { const rules = buildWalkerThemeRules({ foreground: 'rgb(1, 2, 3)', diff --git a/tests/unit/whisper-alignment-mapping.spec.ts b/tests/unit/whisper-alignment-mapping.vitest.spec.ts similarity index 88% rename from tests/unit/whisper-alignment-mapping.spec.ts rename to tests/unit/whisper-alignment-mapping.vitest.spec.ts index 5f91973..b53a4f6 100644 --- a/tests/unit/whisper-alignment-mapping.spec.ts +++ b/tests/unit/whisper-alignment-mapping.vitest.spec.ts @@ -1,9 +1,9 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, } from '@openreader/compute-core'; -test.describe('whisper alignment mapping', () => { +describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { const aligned = mapWordsToSentenceOffsets('Hello, world again.', [ { word: 'Hello', start: 0, end: 0.25 }, diff --git a/tests/unit/whisper-spectral.spec.ts b/tests/unit/whisper-spectral.vitest.spec.ts similarity index 91% rename from tests/unit/whisper-spectral.spec.ts rename to tests/unit/whisper-spectral.vitest.spec.ts index 90857a2..2b87a04 100644 --- a/tests/unit/whisper-spectral.spec.ts +++ b/tests/unit/whisper-spectral.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core'; function dftPower(samples: Float32Array, k: number): number { @@ -13,7 +13,7 @@ function dftPower(samples: Float32Array, k: number): number { return (re * re) + (im * im); } -test.describe('whisper spectral helpers', () => { +describe('whisper spectral helpers', () => { test('goertzel power matches direct DFT for non-power-of-two frame size', () => { const frameSize = 400; const bins = 201; diff --git a/tests/unit/whisper-token-timestamps.spec.ts b/tests/unit/whisper-token-timestamps.vitest.spec.ts similarity index 95% rename from tests/unit/whisper-token-timestamps.spec.ts rename to tests/unit/whisper-token-timestamps.vitest.spec.ts index a584d6f..c2aded0 100644 --- a/tests/unit/whisper-token-timestamps.spec.ts +++ b/tests/unit/whisper-token-timestamps.vitest.spec.ts @@ -1,11 +1,11 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, } from '@openreader/compute-core'; -test.describe('whisper token timestamp alignment', () => { +describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { const seqLen = 6; const frames = 10; diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..2bf3656 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,55 @@ +import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'node:url'; + +const srcDir = fileURLToPath(new URL('./src/', import.meta.url)); +const computeCoreIndex = fileURLToPath(new URL('./compute/core/src/index.ts', import.meta.url)); +const computeCoreApiContracts = fileURLToPath(new URL('./compute/core/src/api-contracts/index.ts', import.meta.url)); +const computeCoreControlPlane = fileURLToPath(new URL('./compute/core/src/control-plane/index.ts', import.meta.url)); +const computeCoreLocalRuntime = fileURLToPath(new URL('./compute/core/src/local-runtime.ts', import.meta.url)); +const computeCoreTypes = fileURLToPath(new URL('./compute/core/src/types/index.ts', import.meta.url)); +const alias = [ + { find: /^@openreader\/compute-core$/, replacement: computeCoreIndex }, + { find: /^@openreader\/compute-core\/api-contracts$/, replacement: computeCoreApiContracts }, + { find: /^@openreader\/compute-core\/control-plane$/, replacement: computeCoreControlPlane }, + { find: /^@openreader\/compute-core\/local-runtime$/, replacement: computeCoreLocalRuntime }, + { find: /^@openreader\/compute-core\/types$/, replacement: computeCoreTypes }, + { find: /^@\//, replacement: `${srcDir}` }, + { find: '@', replacement: srcDir }, +]; + +export default defineConfig({ + resolve: { + alias, + }, + test: { + alias, + reporters: process.env.CI ? ['default', 'github-actions'] : ['default'], + projects: [ + { + resolve: { + alias, + }, + test: { + name: 'openreader', + environment: 'node', + include: ['tests/unit/**/*.vitest.spec.ts'], + }, + }, + { + test: { + name: 'compute-core', + environment: 'node', + include: ['compute/core/tests/control-plane/**/*.test.ts'], + }, + }, + { + test: { + name: 'compute-worker', + environment: 'node', + include: ['compute/worker/tests/{unit,api}/**/*.test.ts'], + setupFiles: ['compute/worker/tests/setup-env.ts'], + }, + }, + ], + }, +});