From a0f2d008e0ac1928f6e82ad02752e762f9ce7685 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 09:55:16 -0600 Subject: [PATCH 01/16] refactor(tts): transition rate limiting to runtime config and admin panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove legacy TTS rate limit environment variables and migrate all related configuration to runtime settings managed via the Admin UI - Add runtime config keys for TTS rate limiting enable/disable and per-user/IP daily quotas - Refactor rate limiter and API routes to use runtime config for thresholds and enablement - Update AdminFeaturesPanel to allow editing TTS rate limiting and quota values in the UI - Add shared-provider-selection utility for consistent provider selection logic - Update documentation to reflect new runtime/admin configuration and remove obsolete env var docs - Add unit tests for rate limit runtime settings and provider selection BREAKING CHANGE: TTS rate limiting is now controlled via Admin → Site features; environment variables for TTS rate limiting are no longer supported and will be ignored. --- .env.example | 62 +++--- docs-site/docs/configure/tts-rate-limiting.md | 20 +- .../docs/reference/environment-variables.md | 43 ++-- src/app/api/audiobook/chapter/route.ts | 18 +- src/app/api/rate-limit/status/route.ts | 24 ++- src/app/api/tts/segments/ensure/route.ts | 17 +- src/components/admin/AdminFeaturesPanel.tsx | 114 ++++++++-- src/components/admin/AdminProvidersPanel.tsx | 198 +++++++++++++++--- src/components/formPrimitives.tsx | 63 +++++- src/components/icons/Icons.tsx | 21 ++ src/contexts/RuntimeConfigContext.tsx | 10 + src/lib/server/admin/providers.ts | 83 +++++++- src/lib/server/admin/resolve-credentials.ts | 37 +--- src/lib/server/admin/settings.ts | 23 ++ src/lib/server/audiobooks/settings.ts | 14 +- src/lib/server/rate-limit/problem-response.ts | 8 +- src/lib/server/rate-limit/rate-limiter.ts | 106 +++++----- src/lib/shared/shared-provider-selection.ts | 34 +++ .../unit/rate-limit-runtime-settings.spec.ts | 25 +++ tests/unit/shared-provider-selection.spec.ts | 51 +++++ 20 files changed, 744 insertions(+), 227 deletions(-) create mode 100644 src/lib/shared/shared-provider-selection.ts create mode 100644 tests/unit/rate-limit-runtime-settings.spec.ts create mode 100644 tests/unit/shared-provider-selection.spec.ts diff --git a/.env.example b/.env.example index 9de5c4d..d3c5c47 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,7 @@ API_BASE=http://localhost:8880/v1 API_KEY=api_key_optional -# (Optional) TTS request/cache tuning (leave unset to use defaults) +# (Optional) TTS request/cache tuning (deaults shown below; leave unset to use defaults) # TTS_CACHE_MAX_SIZE_BYTES=268435456 # 256MB # TTS_CACHE_TTL_MS=1800000 # 30 minutes # TTS_MAX_RETRIES=2 @@ -24,27 +24,18 @@ API_KEY=api_key_optional # TTS_RETRY_BACKOFF=2 # TTS_UPSTREAM_TIMEOUT_MS=285000 # 285 seconds -# (Optional) Enable TTS character rate limiting (default is `false`) -# TTS_ENABLE_RATE_LIMIT=true -# (Optional) TTS per-user daily limits (leave unset to use defaults) -# TTS_DAILY_LIMIT_ANONYMOUS=50000 -# TTS_DAILY_LIMIT_AUTHENTICATED=500000 -# (Optional) TTS IP backstop daily limits (leave unset to use defaults) -# TTS_IP_DAILY_LIMIT_ANONYMOUS=100000 -# TTS_IP_DAILY_LIMIT_AUTHENTICATED=1000000 - -# Auth configuration (recommended for contributors and public instances) +# Auth configuration (recommended; required for admin features) # (Optional) Auth is only enabled when AUTH_SECRET and BASE_URL are set BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network) AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -hex 32` AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional trusted origins (BASE_URL is always trusted) # (Optional) Allow anonymous auth sessions when auth is enabled (default: `false`) -USE_ANONYMOUS_AUTH_SESSIONS= +# USE_ANONYMOUS_AUTH_SESSIONS=false # (Optional) Sign in w/ GitHub Configuration -GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= -# (Optional) Disable Better Auth built-in rate limiting (useful for testing) -DISABLE_AUTH_RATE_LIMIT= +# GITHUB_CLIENT_ID= +# GITHUB_CLIENT_SECRET= +# (Optional) Disable Better Auth built-in rate limiting (useful for testing; default: `false`) +# DISABLE_AUTH_RATE_LIMIT=false # (Optional) Comma-separated list of emails that are auto-promoted to admin. # Admins see the "Admin" tab in Settings (TTS shared providers + site features). @@ -54,34 +45,28 @@ ADMIN_EMAILS= # (Optional) Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables. # Defaults to SQLite at docstore/sqlite3.db when not set. -POSTGRES_URL= +# POSTGRES_URL= # Embedded SeaweedFS weed mini config -# (Optional) Enable embedded weed mini for local S3-compatible storage (default: `true`) -USE_EMBEDDED_WEED_MINI= -WEED_MINI_DIR= -WEED_MINI_WAIT_SEC= +# (Optional) Enable embedded weed mini for local S3-compatible storage (defaults shown below) +# USE_EMBEDDED_WEED_MINI=true +# WEED_MINI_DIR=docstore/seaweedfs +# WEED_MINI_WAIT_SEC=20 # S3 storage config (use with embedded weed mini or external S3-compatible storage) # (Optional) For embedded weed mini, set explicit keys if you want stable credentials across restarts. S3_ACCESS_KEY_ID= S3_SECRET_ACCESS_KEY= S3_BUCKET= -S3_REGION= +# S3_REGION= # (Optional) If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host. -S3_ENDPOINT= -S3_FORCE_PATH_STYLE= -S3_PREFIX= +# S3_ENDPOINT= +# S3_FORCE_PATH_STYLE= +# S3_PREFIX= -# Migrations configuration -# (Optional) Skip automatic startup migrations when set to `false` (default: `true`) -RUN_DRIZZLE_MIGRATIONS= -# (Optional) Skip automatic filesystem->S3/DB migration pass when set to `false` (default: `true`) -RUN_FS_MIGRATIONS= - -# (Optional) Server library import roots (uses /docstore/library by default) -IMPORT_LIBRARY_DIR= -IMPORT_LIBRARY_DIRS= +# (Optional) Library import mode directory (uses /docstore/library by default) +# IMPORT_LIBRARY_DIR= +# IMPORT_LIBRARY_DIRS= # Compute # Embedded/local (default): leave COMPUTE_WORKER_URL empty. @@ -118,7 +103,7 @@ IMPORT_LIBRARY_DIRS= # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main # (Optional) Override ffmpeg binary path used for audiobook processing -FFMPEG_BIN= +# FFMPEG_BIN= # (Optional) Client feature flags — seeded into the admin-managed runtime # config on first boot, then ignored. Edit values from Settings → Admin → @@ -132,3 +117,10 @@ FFMPEG_BIN= # RUNTIME_SEED_DEFAULT_TTS_PROVIDER=custom-openai # RUNTIME_SEED_CHANGELOG_FEED_URL=https://docs.openreader.richardr.dev/changelog/manifest.json # RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT=true +# RUNTIME_SEED_DISABLE_TTS_LIMIT=true + +# Migrations configuration +# (Optional) Skip automatic startup migrations when set to `false` (default: `true`) +# RUN_DRIZZLE_MIGRATIONS=true +# (Optional) Skip automatic filesystem->S3/DB migration pass when set to `false` (default: `true`) +# RUN_FS_MIGRATIONS=true diff --git a/docs-site/docs/configure/tts-rate-limiting.md b/docs-site/docs/configure/tts-rate-limiting.md index ed507d1..b9cd801 100644 --- a/docs-site/docs/configure/tts-rate-limiting.md +++ b/docs-site/docs/configure/tts-rate-limiting.md @@ -7,7 +7,8 @@ This page explains OpenReader's TTS character rate limiting controls. ## Overview - TTS rate limiting is disabled by default. -- To enable it, set `TTS_ENABLE_RATE_LIMIT=true`. +- Primary control is **Settings → Admin → Site features → Disable TTS daily rate limiting**. +- Optional first-boot seed: `RUNTIME_SEED_DISABLE_TTS_LIMIT=true`. - Limits are enforced per day in UTC. - Enforcement applies only when auth is enabled. @@ -28,21 +29,10 @@ If a request exceeds the active limit, the TTS API returns `429` with reset meta - `DISABLE_AUTH_RATE_LIMIT` only affects Better Auth's own request throttling. - `DISABLE_AUTH_RATE_LIMIT` does not disable TTS character limits. -## Environment variables +## Runtime config + seed var -Enable/disable: - -- `TTS_ENABLE_RATE_LIMIT` (default: `false`) - -Per-user daily limits: - -- `TTS_DAILY_LIMIT_ANONYMOUS` (default: `50000`) -- `TTS_DAILY_LIMIT_AUTHENTICATED` (default: `500000`) - -IP backstop daily limits: - -- `TTS_IP_DAILY_LIMIT_ANONYMOUS` (default: `100000`) -- `TTS_IP_DAILY_LIMIT_AUTHENTICATED` (default: `1000000`) +- First-boot seed toggle: `RUNTIME_SEED_DISABLE_TTS_LIMIT` (default: `true`) +- Per-user and IP backstop limit values are configured in **Settings → Admin → Site features** and stored in DB runtime settings. ## Related docs diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index b4a4078..4db5083 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -24,11 +24,6 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `TTS_RETRY_MAX_MS` | TTS retry | `2000` | Tune max retry delay | | `TTS_RETRY_BACKOFF` | TTS retry | `2` | Tune exponential backoff factor | | `TTS_UPSTREAM_TIMEOUT_MS` | TTS request timeout | `285000` | Set max upstream TTS request duration before fail-fast | -| `TTS_ENABLE_RATE_LIMIT` | Rate limiting | `false` | Set `true` to enable TTS per-user/IP daily character limits | -| `TTS_DAILY_LIMIT_ANONYMOUS` | Rate limiting | `50000` | Override anonymous per-user daily character limit | -| `TTS_DAILY_LIMIT_AUTHENTICATED` | Rate limiting | `500000` | Override authenticated per-user daily character limit | -| `TTS_IP_DAILY_LIMIT_ANONYMOUS` | Rate limiting | `100000` | Override anonymous IP backstop daily limit | -| `TTS_IP_DAILY_LIMIT_AUTHENTICATED` | Rate limiting | `1000000` | Override authenticated IP backstop daily limit | | `BASE_URL` | Auth | unset | Required (with `AUTH_SECRET`) to enable auth | | `AUTH_SECRET` | Auth | unset | Required (with `BASE_URL`) to enable auth | | `AUTH_TRUSTED_ORIGINS` | Auth | empty | Add extra allowed origins | @@ -76,6 +71,7 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `RUNTIME_SEED_RESTRICT_USER_API_KEYS` | Legacy bootstrap seed | runtime-dependent | Optional first-boot seed to restrict per-user BYOK | | `RUNTIME_SEED_DEFAULT_TTS_PROVIDER` | Legacy bootstrap seed | `custom-openai` | Optional first-boot seed for default TTS provider slug | | `RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT` | Legacy bootstrap seed | `true` | Optional first-boot seed to enable audiobook export UI | +| `RUNTIME_SEED_DISABLE_TTS_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps TTS daily rate limiting disabled | @@ -158,37 +154,22 @@ Maximum upstream TTS request timeout in milliseconds. - Applies to outbound provider calls from server routes using shared TTS generation - Increase for slower providers/models; decrease to fail fast and surface retryable errors sooner -### TTS_ENABLE_RATE_LIMIT +### TTS Daily Rate Limiting (Runtime Settings) -Controls TTS character rate limiting in the TTS API. +TTS character rate limiting is now managed from **Settings → Admin → Site features**. -- Default: `false` (TTS char limits disabled) -- Set to `true` to enforce `TTS_DAILY_LIMIT_*` and `TTS_IP_DAILY_LIMIT_*` -- For behavior details and examples, see [TTS Rate Limiting](../configure/tts-rate-limiting) +- `disableTtsRateLimit` default: `true` (rate limiting disabled) +- Daily limit defaults: + - Anonymous per-user: `50000` + - Authenticated per-user: `500000` + - Anonymous IP backstop: `100000` + - Authenticated IP backstop: `1000000` -### TTS_DAILY_LIMIT_ANONYMOUS +Optional first-boot seeds: -Anonymous per-user daily character limit. +- `RUNTIME_SEED_DISABLE_TTS_LIMIT` -- Default: `50000` - -### TTS_DAILY_LIMIT_AUTHENTICATED - -Authenticated per-user daily character limit. - -- Default: `500000` - -### TTS_IP_DAILY_LIMIT_ANONYMOUS - -Anonymous IP backstop daily character limit. - -- Default: `100000` - -### TTS_IP_DAILY_LIMIT_AUTHENTICATED - -Authenticated IP backstop daily character limit. - -- Default: `1000000` +After first boot, these values are DB-backed admin runtime settings. ## Auth and Identity diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index c2a2f27..f0875de 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -8,7 +8,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { audiobooks, audiobookChapters } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -532,7 +532,15 @@ export async function POST(request: NextRequest) { sharedDefaultInstructions: credResolved.adminRecord?.defaultInstructions, }); - if (authEnabled && userId && isTtsRateLimitEnabled()) { + const ttsRateLimitEnabled = !runtimeConfig.disableTtsRateLimit; + const limits = resolveRateLimitThresholds({ + anonymous: runtimeConfig.ttsDailyLimitAnonymous, + authenticated: runtimeConfig.ttsDailyLimitAuthenticated, + ipAnonymous: runtimeConfig.ttsIpDailyLimitAnonymous, + ipAuthenticated: runtimeConfig.ttsIpDailyLimitAuthenticated, + }); + + if (authEnabled && userId && ttsRateLimitEnabled) { const isAnonymous = Boolean(user?.isAnonymous); const charCount = data.text.length; const ip = getClientIp(request); @@ -549,6 +557,10 @@ export async function POST(request: NextRequest) { deviceId: device?.deviceId ?? null, ip, }, + { + enabled: ttsRateLimitEnabled, + limits, + }, ); if (!rateLimitResult.allowed) { @@ -570,7 +582,7 @@ export async function POST(request: NextRequest) { resetTimeMs, userType: isAnonymous ? 'anonymous' : 'authenticated', upgradeHint: isAnonymous - ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + ? `Sign up to increase your limit from ${formatLimitForHint(limits.anonymous)} to ${formatLimitForHint(limits.authenticated)} characters per day` : undefined, instance: request.nextUrl.pathname, }; diff --git a/src/app/api/rate-limit/status/route.ts b/src/app/api/rate-limit/status/route.ts index e541f9f..f21d8dc 100644 --- a/src/app/api/rate-limit/status/route.ts +++ b/src/app/api/rate-limit/status/route.ts @@ -1,11 +1,12 @@ import { NextResponse, type NextRequest } from 'next/server'; import { auth } from '@/lib/server/auth/auth'; -import { rateLimiter, RATE_LIMITS, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { headers } from 'next/headers'; import { isAuthEnabled } from '@/lib/server/auth/config'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id'; import { nextUtcMidnightTimestampMs } from '@/lib/shared/timestamps'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; @@ -17,7 +18,14 @@ function getUtcResetTimeMs(): number { export async function GET(req: NextRequest) { try { - const ttsRateLimitEnabled = isTtsRateLimitEnabled(); + const runtimeConfig = await getResolvedRuntimeConfig(); + const limits = resolveRateLimitThresholds({ + anonymous: runtimeConfig.ttsDailyLimitAnonymous, + authenticated: runtimeConfig.ttsDailyLimitAuthenticated, + ipAnonymous: runtimeConfig.ttsIpDailyLimitAnonymous, + ipAuthenticated: runtimeConfig.ttsIpDailyLimitAuthenticated, + }); + const ttsRateLimitEnabled = !runtimeConfig.disableTtsRateLimit; // If auth is not enabled, return unlimited status if (!isAuthEnabled() || !auth) { @@ -46,8 +54,8 @@ export async function GET(req: NextRequest) { return NextResponse.json({ allowed: true, currentCount: 0, - limit: ttsRateLimitEnabled ? RATE_LIMITS.ANONYMOUS : Number.MAX_SAFE_INTEGER, - remainingChars: ttsRateLimitEnabled ? RATE_LIMITS.ANONYMOUS : Number.MAX_SAFE_INTEGER, + limit: ttsRateLimitEnabled ? limits.anonymous : Number.MAX_SAFE_INTEGER, + remainingChars: ttsRateLimitEnabled ? limits.anonymous : Number.MAX_SAFE_INTEGER, resetTimeMs, userType: 'unauthenticated', authEnabled: true @@ -57,7 +65,7 @@ export async function GET(req: NextRequest) { const isAnonymous = Boolean((session.user as { isAnonymous?: boolean }).isAnonymous); const ip = getClientIp(req); - const device = isTtsRateLimitEnabled() ? (isAnonymous ? getOrCreateDeviceId(req) : null) : null; + const device = ttsRateLimitEnabled ? (isAnonymous ? getOrCreateDeviceId(req) : null) : null; const result = await rateLimiter.getCurrentUsage( { @@ -67,7 +75,11 @@ export async function GET(req: NextRequest) { { deviceId: device?.deviceId ?? null, ip, - } + }, + { + enabled: ttsRateLimitEnabled, + limits, + }, ); const response = NextResponse.json({ diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index a3dc204..80178fb 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -23,7 +23,7 @@ import { } from '@/lib/server/tts/segments'; import { isBuiltInTtsProviderId, isTtsProviderType } from '@/lib/shared/tts-provider-catalog'; import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; -import { rateLimiter, isTtsRateLimitEnabled } from '@/lib/server/rate-limit/rate-limiter'; +import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { resolveTtsCredentials } from '@/lib/server/admin/resolve-credentials'; import { resolveEffectiveTtsInstructions } from '@/lib/server/admin/tts-instructions'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; @@ -169,6 +169,13 @@ export async function POST(request: NextRequest) { const scope = await resolveSegmentDocumentScope(request, parsed.documentId); if (scope instanceof Response) return scope; const runtimeConfig = await getResolvedRuntimeConfig(); + const ttsRateLimitEnabled = !runtimeConfig.disableTtsRateLimit; + const limits = resolveRateLimitThresholds({ + anonymous: runtimeConfig.ttsDailyLimitAnonymous, + authenticated: runtimeConfig.ttsDailyLimitAuthenticated, + ipAnonymous: runtimeConfig.ttsIpDailyLimitAnonymous, + ipAuthenticated: runtimeConfig.ttsIpDailyLimitAuthenticated, + }); const requestCreds = await resolveTtsCredentials({ providerHeader: parsed.settings.providerRef, apiKeyHeader: request.headers.get('x-openai-key'), @@ -461,7 +468,7 @@ export async function POST(request: NextRequest) { segmentId: segment.segmentId, }); - if (scope.authEnabled && scope.userId && isTtsRateLimitEnabled()) { + if (scope.authEnabled && scope.userId && ttsRateLimitEnabled) { const charCount = segment.text.length; const ip = getClientIp(request); const device = scope.isAnonymousUser ? getOrCreateDeviceId(request) : null; @@ -477,6 +484,10 @@ export async function POST(request: NextRequest) { deviceId: device?.deviceId ?? null, ip, }, + { + enabled: ttsRateLimitEnabled, + limits, + }, ); if (!rateLimitResult.allowed) { @@ -484,6 +495,8 @@ export async function POST(request: NextRequest) { rateLimitResult, isAnonymousUser: scope.isAnonymousUser, pathname: request.nextUrl.pathname, + anonymousLimit: limits.anonymous, + authenticatedLimit: limits.authenticated, }); attachDeviceIdCookie(response, deviceIdToSet, didCreateDeviceIdCookie); return response; diff --git a/src/components/admin/AdminFeaturesPanel.tsx b/src/components/admin/AdminFeaturesPanel.tsx index 41aca85..3c3c6da 100644 --- a/src/components/admin/AdminFeaturesPanel.tsx +++ b/src/components/admin/AdminFeaturesPanel.tsx @@ -7,6 +7,7 @@ import toast from 'react-hot-toast'; import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; import { Badge, + EditableRow, Section, ToggleRow, buttonClass, @@ -56,12 +57,14 @@ export function AdminFeaturesPanel() { }); const [draft, setDraft] = useState>({}); const [dirty, setDirty] = useState>(new Set()); + const [editingChangelogUrl, setEditingChangelogUrl] = useState(false); const { providers: sharedProviders } = useSharedProviders(); useEffect(() => { if (!data) return; setDraft({ ...data.values }); setDirty(new Set()); + setEditingChangelogUrl(false); }, [data]); useEffect(() => { @@ -104,11 +107,19 @@ export function AdminFeaturesPanel() { setDraft((d) => ({ ...d, [key]: value })); setDirty((s) => { const next = new Set(s); - next.add(key); + const baselineValue = data?.values?.[key]; + if (Object.is(value, baselineValue)) next.delete(key); + else next.add(key); return next; }); }; + const updatePositiveIntDraft = (key: string, raw: string) => { + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return; + updateDraft(key, Math.max(1, Math.floor(parsed))); + }; + const resetField = (key: string) => { if (saving) return; resetMutation.mutate(key); @@ -151,6 +162,7 @@ export function AdminFeaturesPanel() { } as ProviderOption : fallbackShared; const selectedProviderOption = effectiveSelectedProvider; + const showTtsDailyLimitInputs = !Boolean(draft.disableTtsRateLimit); const handleProviderChange = (opt: ProviderOption) => { updateDraft('defaultTtsProvider', opt.id); @@ -270,6 +282,85 @@ export function AdminFeaturesPanel() { right={renderSource('showAllProviderModels')} variant="flat" /> + updateDraft('disableTtsRateLimit', checked)} + right={renderSource('disableTtsRateLimit')} + variant="flat" + /> + +
+
+
+
+ + {renderSource('ttsDailyLimitAnonymous')} +
+ updatePositiveIntDraft('ttsDailyLimitAnonymous', event.target.value)} + /> +
+
+
+ + {renderSource('ttsDailyLimitAuthenticated')} +
+ updatePositiveIntDraft('ttsDailyLimitAuthenticated', event.target.value)} + /> +
+
+
+ + {renderSource('ttsIpDailyLimitAnonymous')} +
+ updatePositiveIntDraft('ttsIpDailyLimitAnonymous', event.target.value)} + /> +
+
+
+ + {renderSource('ttsIpDailyLimitAuthenticated')} +
+ updatePositiveIntDraft('ttsIpDailyLimitAuthenticated', event.target.value)} + /> +
+
+
+
Feature Flags} > -
-
-
-

Changelog feed URL

-

- Public URL to the changelog manifest JSON used by Settings. -

-
-
{renderSource('changelogFeedUrl')}
-
+ updateDraft('changelogFeedUrl', event.target.value)} placeholder="https://docs.openreader.richardr.dev/changelog/manifest.json" /> -
+ { + const res = await fetch('/api/admin/settings'); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = (await res.json()) as { values?: { defaultTtsProvider?: unknown } }; + return typeof data.values?.defaultTtsProvider === 'string' ? data.values.defaultTtsProvider : ''; +} + +async function patchDefaultProviderSlug(slug: string): Promise { + const res = await fetch('/api/admin/settings', { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ updates: { defaultTtsProvider: slug } }), + }); + if (!res.ok && res.status !== 207) throw new Error(`HTTP ${res.status}`); +} + +async function patchProviderEnabled(input: { id: string; enabled: boolean }): Promise { + const res = await fetch(`/api/admin/providers/${input.id}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ enabled: input.enabled }), + }); + if (!res.ok) { + const data = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(data.error || `HTTP ${res.status}`); + } +} function createEmptyForm(): FormState { return { @@ -70,6 +99,11 @@ function createEmptyForm(): FormState { }; } +function truncateModelLabel(value: string, maxLength = 56): string { + if (value.length <= maxLength) return value; + return `${value.slice(0, maxLength - 3)}...`; +} + async function fetchAdminProviders(): Promise { const res = await fetch('/api/admin/providers'); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -122,19 +156,34 @@ export function AdminProvidersPanel() { queryKey: ADMIN_PROVIDERS_QUERY_KEY, queryFn: fetchAdminProviders, }); + const { + data: defaultProviderSlug = '', + error: defaultProviderError, + } = useQuery({ + queryKey: ADMIN_SETTINGS_QUERY_KEY, + queryFn: fetchDefaultProviderSlug, + }); useEffect(() => { if (!error) return; console.error('[AdminProvidersPanel] load failed:', error); toast.error('Failed to load admin providers'); }, [error]); + useEffect(() => { + if (!defaultProviderError) return; + console.error('[AdminProvidersPanel] default provider load failed:', defaultProviderError); + toast.error('Failed to load default provider'); + }, [defaultProviderError]); const saveMutation = useMutation({ mutationFn: upsertAdminProvider, onSuccess: async (_data, variables) => { toast.success(variables.editingId === '__new' ? 'Provider created' : 'Provider updated'); cancelEdit(); - await queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + ]); }, onError: (mutationError) => { console.error('[AdminProvidersPanel] save failed:', mutationError); @@ -146,13 +195,41 @@ export function AdminProvidersPanel() { mutationFn: deleteAdminProvider, onSuccess: async () => { toast.success('Provider deleted'); - await queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + ]); }, onError: (mutationError) => { console.error('[AdminProvidersPanel] delete failed:', mutationError); toast.error((mutationError as Error).message || 'Delete failed'); }, }); + const toggleEnabledMutation = useMutation({ + mutationFn: patchProviderEnabled, + onSuccess: async (_data, vars) => { + toast.success(vars.enabled ? 'Provider enabled' : 'Provider disabled'); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + ]); + }, + onError: (mutationError) => { + console.error('[AdminProvidersPanel] toggle enabled failed:', mutationError); + toast.error((mutationError as Error).message || 'Update failed'); + }, + }); + const setDefaultMutation = useMutation({ + mutationFn: patchDefaultProviderSlug, + onSuccess: async () => { + toast.success('Default provider updated'); + await queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }); + }, + onError: (mutationError) => { + console.error('[AdminProvidersPanel] set default failed:', mutationError); + toast.error((mutationError as Error).message || 'Failed to set default'); + }, + }); const startCreate = () => { setForm(createEmptyForm()); @@ -191,6 +268,14 @@ export function AdminProvidersPanel() { if (deleteMutation.isPending) return; deleteMutation.mutate(id); }; + const toggleEnabled = (provider: AdminProviderMasked) => { + if (toggleEnabledMutation.isPending) return; + toggleEnabledMutation.mutate({ id: provider.id, enabled: !provider.enabled }); + }; + const setDefault = (slug: string) => { + if (setDefaultMutation.isPending) return; + setDefaultMutation.mutate(slug); + }; const isEditingExisting = editingId !== null && editingId !== '__new'; const editingProvider = isEditingExisting @@ -504,38 +589,101 @@ export function AdminProvidersPanel() {

No shared providers configured yet.

) : ( providers.map((p) => ( -
-
+
+
-
+
{p.displayName} {p.slug} + {defaultProviderSlug === p.slug && Default} {!p.enabled && Disabled}
-
+
{p.providerType} - {p.defaultModel ? ` · ${p.defaultModel}` : ''} + {p.defaultModel ? ( + <> + {' · '} + {truncateModelLabel(p.defaultModel)} + + ) : ( + ' · no default model' + )} {p.defaultInstructions ? ' · instructions' : ''} - {p.baseUrl ? ` · ${p.baseUrl}` : ''} - {' · '}key {p.apiKeyMask} +
+
+ {p.baseUrl ? p.baseUrl : 'provider base URL default'} · key {p.apiKeyMask}
-
- - -
+ + + {({ active }) => ( + + )} + + + {({ active, disabled }) => ( + + )} + + + {({ active }) => ( + + )} + + + {({ active }) => ( + + )} + + + +
)) diff --git a/src/components/formPrimitives.tsx b/src/components/formPrimitives.tsx index 6095591..d332594 100644 --- a/src/components/formPrimitives.tsx +++ b/src/components/formPrimitives.tsx @@ -1,6 +1,8 @@ 'use client'; -import { useId, type ReactNode } from 'react'; +import { Fragment, useId, type ReactNode } from 'react'; +import { Transition } from '@headlessui/react'; +import { EditIcon } from '@/components/icons/Icons'; /** * Shared compact form primitives used by settings-like surfaces across @@ -239,6 +241,65 @@ export function ToggleRow({ ); } +export function EditableRow({ + label, + description, + value, + expanded, + onExpandedChange, + right, + children, + editLabel = 'Edit', +}: { + label: string; + description: string; + value: ReactNode; + expanded: boolean; + onExpandedChange: (expanded: boolean) => void; + right?: ReactNode; + children: ReactNode; + editLabel?: string; +}) { + const rowClass = + 'px-0.5 pt-1 pb-2 border-b border-offbase last:border-b-0 transition-transform duration-200 ease-out hover:scale-[1.003]'; + return ( +
+
+
+ {label} + {description} + {value} +
+ {right ?
{right}
: null} + +
+ +
+ {children} +
+ +
+ ); +} + export function CheckItem({ label, checked, diff --git a/src/components/icons/Icons.tsx b/src/components/icons/Icons.tsx index 74c6e8f..a780481 100644 --- a/src/components/icons/Icons.tsx +++ b/src/components/icons/Icons.tsx @@ -193,6 +193,27 @@ export function PlusIcon(props: React.SVGProps) { ); } +export function EditIcon(props: React.SVGProps) { + return ( + + + + + ); +} + export function UploadIcon(props: React.SVGProps) { return ( { return (rows as Array>).map(rowToRecord); } +export async function listEnabledAdminProviders(): Promise { + const rows = await db + .select() + .from(adminProviders) + .where(eq(adminProviders.enabled, 1)) + .orderBy( + desc(adminProviders.updatedAt), + desc(adminProviders.createdAt), + asc(adminProviders.slug), + ); + return (rows as Array>).map(rowToRecord); +} + export async function getAdminProviderBySlug(slug: string): Promise { const rows = await db.select().from(adminProviders).where(eq(adminProviders.slug, slug)).limit(1); const arr = rows as Array>; @@ -258,6 +273,8 @@ export async function updateAdminProvider( ): Promise { const current = await getAdminProviderById(id); if (!current) throw new AdminProviderError('provider not found', 404); + const runtimeConfigBefore = await getRuntimeConfig(); + const wasDefaultProvider = runtimeConfigBefore.defaultTtsProvider === current.slug; const update: Record = { updatedAt: Date.now() }; const nextModel = @@ -309,13 +326,27 @@ export async function updateAdminProvider( await db.update(adminProviders).set(update).where(eq(adminProviders.id, id)); const updated = await getAdminProviderById(id); if (!updated) throw new AdminProviderError('failed to load updated provider', 500); + if (wasDefaultProvider) { + if (updated.enabled && updated.slug !== current.slug) { + await setRuntimeConfigKey('defaultTtsProvider', updated.slug); + } else if (!updated.enabled) { + await swapDefaultSharedProvider(updated.slug); + } + } + await ensureDefaultSharedProviderValidity(); return updated; } export async function deleteAdminProvider(id: string): Promise { const existing = await getAdminProviderById(id); if (!existing) throw new AdminProviderError('provider not found', 404); + const runtimeConfigBefore = await getRuntimeConfig(); + const wasDefaultProvider = runtimeConfigBefore.defaultTtsProvider === existing.slug; await db.delete(adminProviders).where(eq(adminProviders.id, id)); + if (wasDefaultProvider) { + await swapDefaultSharedProvider(existing.slug); + } + await ensureDefaultSharedProviderValidity(); } /** Lookup helper used by TTS routes: returns null if not found or disabled. */ @@ -333,11 +364,49 @@ export async function getEnabledAdminProviderBySlug( } export async function getFirstEnabledAdminProvider(): Promise { - const rows = await db - .select() - .from(adminProviders) - .where(eq(adminProviders.enabled, 1)) - .limit(1); - const arr = rows as Array>; - return arr[0] ? rowToRecord(arr[0]) : null; + const rows = await listEnabledAdminProviders(); + return rows[0] ?? null; +} + +export async function resolvePreferredEnabledAdminProvider(input: { + requestedSlug?: string | null; + runtimeDefaultSlug?: string | null; +}): Promise { + const providers = await listEnabledAdminProviders(); + const selectedSlug = resolvePreferredSharedProviderSlug({ + providers, + requestedSlug: input.requestedSlug, + runtimeDefaultSlug: input.runtimeDefaultSlug, + }); + if (!selectedSlug) return null; + return providers.find((provider) => provider.slug === selectedSlug) ?? null; +} + +async function ensureDefaultSharedProviderValidity(): Promise { + const runtimeConfig = await getRuntimeConfig(); + const currentDefaultSlug = runtimeConfig.defaultTtsProvider; + if (!currentDefaultSlug || BUILT_IN_PROVIDER_IDS.includes(currentDefaultSlug as TtsProviderId)) { + return; + } + + const currentDefaultEnabled = await getEnabledAdminProviderBySlug(currentDefaultSlug); + if (currentDefaultEnabled) return; + + const nextProvider = await resolvePreferredEnabledAdminProvider({ + runtimeDefaultSlug: currentDefaultSlug, + }); + if (!nextProvider) return; + + await setRuntimeConfigKey('defaultTtsProvider', nextProvider.slug); +} + +async function swapDefaultSharedProvider(excludedSlug: string): Promise { + const providers = await listEnabledAdminProviders(); + const filtered = providers.filter((provider) => provider.slug !== excludedSlug); + const selectedSlug = resolvePreferredSharedProviderSlug({ + providers: filtered, + runtimeDefaultSlug: null, + }); + if (!selectedSlug) return; + await setRuntimeConfigKey('defaultTtsProvider', selectedSlug); } diff --git a/src/lib/server/admin/resolve-credentials.ts b/src/lib/server/admin/resolve-credentials.ts index 95dd7a3..2193b39 100644 --- a/src/lib/server/admin/resolve-credentials.ts +++ b/src/lib/server/admin/resolve-credentials.ts @@ -1,7 +1,7 @@ import { getEnabledAdminProviderBySlug, - getFirstEnabledAdminProvider, decryptedKeyFor, + resolvePreferredEnabledAdminProvider, type AdminProviderRecord, } from '@/lib/server/admin/providers'; @@ -46,37 +46,20 @@ export async function resolveTtsCredentials(opts: { if (opts.restrictUserApiKeys) { const requestedIsBuiltIn = isBuiltInProviderId(requestedProvider); const fallback = opts.fallbackProvider || ''; - const fallbackIsBuiltIn = isBuiltInProviderId(fallback); - const requestedSharedSlug = requestedIsBuiltIn ? '' : requestedProvider; - const fallbackSharedSlug = fallbackIsBuiltIn ? '' : fallback; - const preferredSharedSlug = requestedSharedSlug || fallbackSharedSlug; - - if (preferredSharedSlug) { - const admin = await getEnabledAdminProviderBySlug(preferredSharedSlug); - if (!admin) { - return { error: 'provider_unknown', slug: preferredSharedSlug }; - } - const apiKey = await decryptedKeyFor(admin); - return { - provider: admin.providerType, - apiKey, - baseUrl: admin.baseUrl || undefined, - fromAdmin: true, - adminRecord: admin, - }; - } - - const firstShared = await getFirstEnabledAdminProvider(); - if (!firstShared) { + const selected = await resolvePreferredEnabledAdminProvider({ + requestedSlug: requestedIsBuiltIn ? null : requestedProvider, + runtimeDefaultSlug: fallback, + }); + if (!selected) { return { error: 'no_shared_provider_configured', slug: requestedProvider }; } - const apiKey = await decryptedKeyFor(firstShared); + const apiKey = await decryptedKeyFor(selected); return { - provider: firstShared.providerType, + provider: selected.providerType, apiKey, - baseUrl: firstShared.baseUrl || undefined, + baseUrl: selected.baseUrl || undefined, fromAdmin: true, - adminRecord: firstShared, + adminRecord: selected, }; } diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index f10dffe..c92b718 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -76,6 +76,24 @@ function stringValue(defaultValue: string, envVar: string): RuntimeConfigKeyDef< }; } +function positiveIntValue(defaultValue: number, envVar?: string): RuntimeConfigKeyDef { + return { + default: defaultValue, + envVar, + parseEnv(raw) { + const trimmed = raw.trim(); + if (!trimmed) return undefined; + const parsed = Number(trimmed); + if (!Number.isFinite(parsed) || parsed <= 0) return undefined; + return Math.floor(parsed); + }, + validate(value) { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined; + return Math.floor(value); + }, + }; +} + export const RUNTIME_CONFIG_SCHEMA = { defaultTtsProvider: stringValue('custom-openai', 'RUNTIME_SEED_DEFAULT_TTS_PROVIDER'), changelogFeedUrl: stringValue('https://docs.openreader.richardr.dev/changelog/manifest.json', 'RUNTIME_SEED_CHANGELOG_FEED_URL'), @@ -88,6 +106,11 @@ export const RUNTIME_CONFIG_SCHEMA = { enableDocxConversion: booleanFlag(true, 'RUNTIME_SEED_ENABLE_DOCX_CONVERSION'), enableDestructiveDeleteActions: booleanFlag(true, 'RUNTIME_SEED_ENABLE_DESTRUCTIVE_DELETE_ACTIONS'), showAllProviderModels: runtimeBoolean(true), + disableTtsRateLimit: booleanFlag(true, 'RUNTIME_SEED_DISABLE_TTS_LIMIT'), + ttsDailyLimitAnonymous: positiveIntValue(50_000), + ttsDailyLimitAuthenticated: positiveIntValue(500_000), + ttsIpDailyLimitAnonymous: positiveIntValue(100_000), + ttsIpDailyLimitAuthenticated: positiveIntValue(1_000_000), } as const satisfies Record>; export type RuntimeConfigKey = keyof typeof RUNTIME_CONFIG_SCHEMA; diff --git a/src/lib/server/audiobooks/settings.ts b/src/lib/server/audiobooks/settings.ts index 1e61ef7..cf74673 100644 --- a/src/lib/server/audiobooks/settings.ts +++ b/src/lib/server/audiobooks/settings.ts @@ -8,6 +8,7 @@ import { isBuiltInTtsProviderId, isTtsProviderType, type TtsProviderId } from '@ import type { AudiobookGenerationSettings } from '@/types/client'; import type { TTSAudiobookFormat } from '@/types/tts'; import { resolveEffectiveTtsInstructions } from '@/lib/server/admin/tts-instructions'; +import { resolvePreferredSharedProviderSlug } from '@/lib/shared/shared-provider-selection'; function isAudiobookFormat(value: unknown): value is TTSAudiobookFormat { return value === 'mp3' || value === 'm4b'; @@ -101,13 +102,12 @@ function resolveRestrictedProviderRef( fallbackProviderRef: string, sharedProviders: SharedProviderPolicyEntry[], ): string { - const requestedIsBuiltIn = isBuiltInTtsProviderId(providerRef); - const fallbackIsBuiltIn = isBuiltInTtsProviderId(fallbackProviderRef); - const requestedSharedSlug = requestedIsBuiltIn ? '' : providerRef; - const fallbackSharedSlug = fallbackIsBuiltIn ? '' : fallbackProviderRef; - const preferredSharedSlug = requestedSharedSlug || fallbackSharedSlug; - if (preferredSharedSlug) return preferredSharedSlug; - return sharedProviders[0]?.slug || providerRef; + const preferred = resolvePreferredSharedProviderSlug({ + providers: sharedProviders, + requestedSlug: isBuiltInTtsProviderId(providerRef) ? null : providerRef, + runtimeDefaultSlug: isBuiltInTtsProviderId(fallbackProviderRef) ? null : fallbackProviderRef, + }); + return preferred || providerRef; } export function canonicalizeAudiobookSettingsForRuntime(input: { diff --git a/src/lib/server/rate-limit/problem-response.ts b/src/lib/server/rate-limit/problem-response.ts index 9f7e2ad..69b08ab 100644 --- a/src/lib/server/rate-limit/problem-response.ts +++ b/src/lib/server/rate-limit/problem-response.ts @@ -1,5 +1,5 @@ import { NextResponse } from 'next/server'; -import { RATE_LIMITS, type RateLimitResult } from '@/lib/server/rate-limit/rate-limiter'; +import { type RateLimitResult } from '@/lib/server/rate-limit/rate-limiter'; function formatLimitForHint(limit: number): string { if (!Number.isFinite(limit) || limit <= 0) return String(limit); @@ -15,8 +15,10 @@ export function buildDailyQuotaExceededResponse(input: { rateLimitResult: RateLimitResult; isAnonymousUser: boolean; pathname: string; + anonymousLimit: number; + authenticatedLimit: number; }): NextResponse { - const { rateLimitResult, isAnonymousUser, pathname } = input; + const { rateLimitResult, isAnonymousUser, pathname, anonymousLimit, authenticatedLimit } = input; const resetTimeMs = rateLimitResult.resetTimeMs; const retryAfterSeconds = Math.max(0, Math.ceil((resetTimeMs - Date.now()) / 1000)); @@ -32,7 +34,7 @@ export function buildDailyQuotaExceededResponse(input: { resetTimeMs, userType: isAnonymousUser ? 'anonymous' : 'authenticated', upgradeHint: isAnonymousUser - ? `Sign up to increase your limit from ${formatLimitForHint(RATE_LIMITS.ANONYMOUS)} to ${formatLimitForHint(RATE_LIMITS.AUTHENTICATED)} characters per day` + ? `Sign up to increase your limit from ${formatLimitForHint(anonymousLimit)} to ${formatLimitForHint(authenticatedLimit)} characters per day` : undefined, instance: pathname, }), { diff --git a/src/lib/server/rate-limit/rate-limiter.ts b/src/lib/server/rate-limit/rate-limiter.ts index 053cb8d..0a9dd68 100644 --- a/src/lib/server/rate-limit/rate-limiter.ts +++ b/src/lib/server/rate-limit/rate-limiter.ts @@ -3,53 +3,40 @@ import { userTtsChars } from '@/db/schema'; import { isAuthEnabled } from '@/lib/server/auth/config'; import { eq, and, lt, sql } from 'drizzle-orm'; import { nextUtcMidnightTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; -import { serverLogger } from '@/lib/server/logger'; -import { logDegraded } from '@/lib/server/errors/logging'; -function readPositiveIntEnv(name: string, fallback: number): number { - const raw = process.env[name]; - if (!raw || raw.trim() === '') return fallback; - - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - logDegraded(serverLogger, { - event: 'rate_limit.config.invalid_env', - msg: 'Invalid rate limiter env value; using default', - step: 'read_limit_env', - context: { - envVar: name, - envValue: raw, - fallbackValue: fallback, - }, - }); - return fallback; - } - - return Math.floor(parsed); +export interface RateLimitThresholds { + anonymous: number; + authenticated: number; + ipAnonymous: number; + ipAuthenticated: number; } -function readBooleanEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]; - if (!raw || raw.trim() === '') return fallback; - const normalized = raw.trim().toLowerCase(); - if (normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on') return true; - if (normalized === '0' || normalized === 'false' || normalized === 'no' || normalized === 'off') return false; - return fallback; +export interface RateLimitRuntimeOptions { + enabled: boolean; + limits: RateLimitThresholds; } -export function isTtsRateLimitEnabled(): boolean { - return readBooleanEnv('TTS_ENABLE_RATE_LIMIT', false); -} +export const DEFAULT_RATE_LIMITS: RateLimitThresholds = { + anonymous: 50_000, + authenticated: 500_000, + ipAnonymous: 100_000, + ipAuthenticated: 1_000_000, +}; -// Rate limits configuration - character counts per day -export const RATE_LIMITS = { - ANONYMOUS: readPositiveIntEnv('TTS_DAILY_LIMIT_ANONYMOUS', 50_000), - AUTHENTICATED: readPositiveIntEnv('TTS_DAILY_LIMIT_AUTHENTICATED', 500_000), - // IP-based backstop limits to make it harder to reset limits by creating new accounts - // or clearing storage/cookies - IP_ANONYMOUS: readPositiveIntEnv('TTS_IP_DAILY_LIMIT_ANONYMOUS', 100_000), - IP_AUTHENTICATED: readPositiveIntEnv('TTS_IP_DAILY_LIMIT_AUTHENTICATED', 1_000_000), -} as const; +export function resolveRateLimitThresholds(input?: Partial): RateLimitThresholds { + const source = input ?? {}; + const normalize = (value: unknown, fallback: number): number => { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return fallback; + return Math.floor(value); + }; + + return { + anonymous: normalize(source.anonymous, DEFAULT_RATE_LIMITS.anonymous), + authenticated: normalize(source.authenticated, DEFAULT_RATE_LIMITS.authenticated), + ipAnonymous: normalize(source.ipAnonymous, DEFAULT_RATE_LIMITS.ipAnonymous), + ipAuthenticated: normalize(source.ipAuthenticated, DEFAULT_RATE_LIMITS.ipAuthenticated), + }; +} // Helper to ensure DB is strictly typed when we know it exists // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -163,8 +150,15 @@ export class RateLimiter { /** * Check if a user can use TTS and increment their char count if allowed */ - async checkAndIncrementLimit(user: UserInfo, charCount: number, backstops?: RateLimitBackstops): Promise { - if (!isAuthEnabled() || !isTtsRateLimitEnabled()) { + async checkAndIncrementLimit( + user: UserInfo, + charCount: number, + backstops?: RateLimitBackstops, + options?: RateLimitRuntimeOptions, + ): Promise { + const limits = resolveRateLimitThresholds(options?.limits); + const enabled = options?.enabled ?? true; + if (!isAuthEnabled() || !enabled) { return { allowed: true, currentCount: 0, @@ -176,7 +170,7 @@ export class RateLimiter { const today = new Date().toISOString().split('T')[0]; const dateValue = today as unknown as UserTtsCharsDateValue; - const userLimit = user.isAnonymous ? RATE_LIMITS.ANONYMOUS : RATE_LIMITS.AUTHENTICATED; + const userLimit = user.isAnonymous ? limits.anonymous : limits.authenticated; const buckets: Bucket[] = [{ key: user.id, limit: userLimit }]; @@ -184,13 +178,13 @@ export class RateLimiter { const ip = backstops?.ip?.toString() || null; if (user.isAnonymous && deviceId) { - buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: RATE_LIMITS.ANONYMOUS }); + buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: limits.anonymous }); } if (ip) { buckets.push({ key: normalizeBackstopKey('ip', ip), - limit: user.isAnonymous ? RATE_LIMITS.IP_ANONYMOUS : RATE_LIMITS.IP_AUTHENTICATED, + limit: user.isAnonymous ? limits.ipAnonymous : limits.ipAuthenticated, }); } @@ -254,7 +248,7 @@ export class RateLimiter { }); } catch (error) { if (error instanceof RateLimitExceeded) { - const current = await this.getCurrentUsage(user, backstops); + const current = await this.getCurrentUsage(user, backstops, options); return { ...current, allowed: false }; } throw error; @@ -264,8 +258,14 @@ export class RateLimiter { /** * Get current usage for a user without incrementing */ - async getCurrentUsage(user: UserInfo, backstops?: RateLimitBackstops): Promise { - if (!isAuthEnabled() || !isTtsRateLimitEnabled()) { + async getCurrentUsage( + user: UserInfo, + backstops?: RateLimitBackstops, + options?: RateLimitRuntimeOptions, + ): Promise { + const limits = resolveRateLimitThresholds(options?.limits); + const enabled = options?.enabled ?? true; + if (!isAuthEnabled() || !enabled) { return { allowed: true, currentCount: 0, @@ -276,7 +276,7 @@ export class RateLimiter { } const today = new Date().toISOString().split('T')[0]; - const userLimit = user.isAnonymous ? RATE_LIMITS.ANONYMOUS : RATE_LIMITS.AUTHENTICATED; + const userLimit = user.isAnonymous ? limits.anonymous : limits.authenticated; const buckets: Bucket[] = [{ key: user.id, limit: userLimit }]; @@ -284,13 +284,13 @@ export class RateLimiter { const ip = backstops?.ip?.toString() || null; if (user.isAnonymous && deviceId) { - buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: RATE_LIMITS.ANONYMOUS }); + buckets.push({ key: normalizeBackstopKey('device', deviceId), limit: limits.anonymous }); } if (ip) { buckets.push({ key: normalizeBackstopKey('ip', ip), - limit: user.isAnonymous ? RATE_LIMITS.IP_ANONYMOUS : RATE_LIMITS.IP_AUTHENTICATED, + limit: user.isAnonymous ? limits.ipAnonymous : limits.ipAuthenticated, }); } @@ -321,7 +321,7 @@ export class RateLimiter { * Transfer char counts when anonymous user creates an account */ async transferAnonymousUsage(anonymousUserId: string, authenticatedUserId: string): Promise { - if (!isAuthEnabled() || !isTtsRateLimitEnabled()) return; + if (!isAuthEnabled()) return; const today = new Date().toISOString().split('T')[0]; const dateValue = today as unknown as UserTtsCharsDateValue; diff --git a/src/lib/shared/shared-provider-selection.ts b/src/lib/shared/shared-provider-selection.ts new file mode 100644 index 0000000..8dbe87f --- /dev/null +++ b/src/lib/shared/shared-provider-selection.ts @@ -0,0 +1,34 @@ +import { isBuiltInTtsProviderId } from '@/lib/shared/tts-provider-catalog'; + +export interface SharedProviderSlugEntry { + slug: string; +} + +function normalizeSharedSlug(value: string | null | undefined): string { + const trimmed = typeof value === 'string' ? value.trim() : ''; + if (!trimmed) return ''; + return isBuiltInTtsProviderId(trimmed) ? '' : trimmed; +} + +export function resolvePreferredSharedProviderSlug(input: { + providers: readonly T[]; + requestedSlug?: string | null; + runtimeDefaultSlug?: string | null; +}): string | null { + const providers = input.providers; + if (providers.length === 0) return null; + + const bySlug = new Map(); + for (const provider of providers) { + bySlug.set(provider.slug, provider); + } + + const requested = normalizeSharedSlug(input.requestedSlug); + if (requested && bySlug.has(requested)) return requested; + + const runtimeDefault = normalizeSharedSlug(input.runtimeDefaultSlug); + if (runtimeDefault && bySlug.has(runtimeDefault)) return runtimeDefault; + + if (bySlug.has('default-openai')) return 'default-openai'; + return providers[0]?.slug ?? null; +} diff --git a/tests/unit/rate-limit-runtime-settings.spec.ts b/tests/unit/rate-limit-runtime-settings.spec.ts new file mode 100644 index 0000000..df7748d --- /dev/null +++ b/tests/unit/rate-limit-runtime-settings.spec.ts @@ -0,0 +1,25 @@ +import { expect, test } from '@playwright/test'; + +import { RUNTIME_CONFIG_SCHEMA } from '../../src/lib/server/admin/settings'; + +test.describe('TTS rate limit runtime config seeds', () => { + test('defaults disable TTS daily rate limiting', () => { + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.default).toBe(true); + }); + + test('parses disable seed boolean values', () => { + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('true')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('false')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('1')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('0')).toBe(false); + }); + + test('daily limit values are runtime-only (no env seed vars)', () => { + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAnonymous.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAuthenticated.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsIpDailyLimitAnonymous.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsIpDailyLimitAuthenticated.envVar).toBeUndefined(); + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAnonymous.default).toBe(50_000); + expect(RUNTIME_CONFIG_SCHEMA.ttsDailyLimitAuthenticated.default).toBe(500_000); + }); +}); diff --git a/tests/unit/shared-provider-selection.spec.ts b/tests/unit/shared-provider-selection.spec.ts new file mode 100644 index 0000000..a371989 --- /dev/null +++ b/tests/unit/shared-provider-selection.spec.ts @@ -0,0 +1,51 @@ +import { expect, test } from '@playwright/test'; + +import { resolvePreferredSharedProviderSlug } from '../../src/lib/shared/shared-provider-selection'; + +const PROVIDERS = [ + { slug: 'shared-a' }, + { slug: 'default-openai' }, + { slug: 'shared-b' }, +] as const; + +test.describe('resolvePreferredSharedProviderSlug', () => { + test('prefers requested shared slug when present', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: 'shared-b', + runtimeDefaultSlug: 'shared-a', + })).toBe('shared-b'); + }); + + test('falls back to runtime default shared slug when requested is missing', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: 'missing-slug', + runtimeDefaultSlug: 'shared-a', + })).toBe('shared-a'); + }); + + test('falls back to default-openai before first provider', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: null, + runtimeDefaultSlug: null, + })).toBe('default-openai'); + }); + + test('ignores built-in provider ids for requested/runtime defaults', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: PROVIDERS, + requestedSlug: 'openai', + runtimeDefaultSlug: 'custom-openai', + })).toBe('default-openai'); + }); + + test('falls back to first enabled provider when default-openai is missing', () => { + expect(resolvePreferredSharedProviderSlug({ + providers: [{ slug: 'shared-z' }, { slug: 'shared-y' }], + requestedSlug: null, + runtimeDefaultSlug: null, + })).toBe('shared-z'); + }); +}); From 267e2ce41161b71232b729d2601bcd885ee4ae85 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 10:05:24 -0600 Subject: [PATCH 02/16] fix admin features tab reset/flash on tab switches --- src/components/admin/AdminFeaturesPanel.tsx | 39 +++++++++++---------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/components/admin/AdminFeaturesPanel.tsx b/src/components/admin/AdminFeaturesPanel.tsx index 3c3c6da..96fc72c 100644 --- a/src/components/admin/AdminFeaturesPanel.tsx +++ b/src/components/admin/AdminFeaturesPanel.tsx @@ -64,7 +64,6 @@ export function AdminFeaturesPanel() { if (!data) return; setDraft({ ...data.values }); setDirty(new Set()); - setEditingChangelogUrl(false); }, [data]); useEffect(() => { @@ -145,10 +144,14 @@ export function AdminFeaturesPanel() { providerType: entry.providerType, })); }, [sharedProviders]); + const valueFor = (key: string) => + Object.prototype.hasOwnProperty.call(draft, key) + ? draft[key] + : data?.values?.[key]; const currentProviderId = - typeof draft.defaultTtsProvider === 'string' - ? draft.defaultTtsProvider + typeof valueFor('defaultTtsProvider') === 'string' + ? (valueFor('defaultTtsProvider') as string) : ''; const currentSharedEntry: SharedProviderEntry | undefined = sharedProviders.find( (p) => p.slug === currentProviderId, @@ -162,7 +165,7 @@ export function AdminFeaturesPanel() { } as ProviderOption : fallbackShared; const selectedProviderOption = effectiveSelectedProvider; - const showTtsDailyLimitInputs = !Boolean(draft.disableTtsRateLimit); + const showTtsDailyLimitInputs = !Boolean(valueFor('disableTtsRateLimit')); const handleProviderChange = (opt: ProviderOption) => { updateDraft('defaultTtsProvider', opt.id); @@ -253,7 +256,7 @@ export function AdminFeaturesPanel() { { if (!checked) { const ok = confirm( @@ -269,7 +272,7 @@ export function AdminFeaturesPanel() { updateDraft('enableTtsProvidersTab', checked)} right={renderSource('enableTtsProvidersTab')} variant="flat" @@ -277,7 +280,7 @@ export function AdminFeaturesPanel() { updateDraft('showAllProviderModels', checked)} right={renderSource('showAllProviderModels')} variant="flat" @@ -285,7 +288,7 @@ export function AdminFeaturesPanel() { updateDraft('disableTtsRateLimit', checked)} right={renderSource('disableTtsRateLimit')} variant="flat" @@ -312,7 +315,7 @@ export function AdminFeaturesPanel() { min={1} step={1} className={inputClass} - value={String(draft.ttsDailyLimitAnonymous ?? '')} + value={String(valueFor('ttsDailyLimitAnonymous') ?? '')} onChange={(event) => updatePositiveIntDraft('ttsDailyLimitAnonymous', event.target.value)} />
@@ -326,7 +329,7 @@ export function AdminFeaturesPanel() { min={1} step={1} className={inputClass} - value={String(draft.ttsDailyLimitAuthenticated ?? '')} + value={String(valueFor('ttsDailyLimitAuthenticated') ?? '')} onChange={(event) => updatePositiveIntDraft('ttsDailyLimitAuthenticated', event.target.value)} />
@@ -340,7 +343,7 @@ export function AdminFeaturesPanel() { min={1} step={1} className={inputClass} - value={String(draft.ttsIpDailyLimitAnonymous ?? '')} + value={String(valueFor('ttsIpDailyLimitAnonymous') ?? '')} onChange={(event) => updatePositiveIntDraft('ttsIpDailyLimitAnonymous', event.target.value)} />
@@ -354,7 +357,7 @@ export function AdminFeaturesPanel() { min={1} step={1} className={inputClass} - value={String(draft.ttsIpDailyLimitAuthenticated ?? '')} + value={String(valueFor('ttsIpDailyLimitAuthenticated') ?? '')} onChange={(event) => updatePositiveIntDraft('ttsIpDailyLimitAuthenticated', event.target.value)} />
@@ -371,7 +374,7 @@ export function AdminFeaturesPanel() { updateDraft('changelogFeedUrl', event.target.value)} placeholder="https://docs.openreader.richardr.dev/changelog/manifest.json" /> @@ -388,7 +391,7 @@ export function AdminFeaturesPanel() { updateDraft('enableUserSignups', checked)} right={renderSource('enableUserSignups')} variant="flat" @@ -396,7 +399,7 @@ export function AdminFeaturesPanel() { updateDraft('enableAudiobookExport', checked)} right={renderSource('enableAudiobookExport')} variant="flat" @@ -404,7 +407,7 @@ export function AdminFeaturesPanel() { updateDraft('enableDocxConversion', checked)} right={renderSource('enableDocxConversion')} variant="flat" @@ -412,7 +415,7 @@ export function AdminFeaturesPanel() { updateDraft('enableDestructiveDeleteActions', checked)} right={renderSource('enableDestructiveDeleteActions')} variant="flat" From b0a4bccf4c8fc8bc941be89976a9e5f75f5128d7 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 10:33:11 -0600 Subject: [PATCH 03/16] refactor(admin): remove unused EditableRow and EditIcon components - Delete EditableRow from formPrimitives and its usage in AdminFeaturesPanel - Remove EditIcon from icon set as it is no longer referenced - Simplify AdminFeaturesPanel state and draft handling - Update AdminProvidersPanel to use distinct query key for default provider - Ensure query invalidation covers new default provider key This streamlines the admin codebase by eliminating obsolete editing UI and related icon, and clarifies provider query logic. --- src/components/admin/AdminFeaturesPanel.tsx | 170 +++++++++---------- src/components/admin/AdminProvidersPanel.tsx | 11 +- src/components/formPrimitives.tsx | 63 +------ src/components/icons/Icons.tsx | 21 --- 4 files changed, 87 insertions(+), 178 deletions(-) diff --git a/src/components/admin/AdminFeaturesPanel.tsx b/src/components/admin/AdminFeaturesPanel.tsx index 96fc72c..575c875 100644 --- a/src/components/admin/AdminFeaturesPanel.tsx +++ b/src/components/admin/AdminFeaturesPanel.tsx @@ -7,7 +7,6 @@ import toast from 'react-hot-toast'; import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; import { Badge, - EditableRow, Section, ToggleRow, buttonClass, @@ -57,7 +56,6 @@ export function AdminFeaturesPanel() { }); const [draft, setDraft] = useState>({}); const [dirty, setDirty] = useState>(new Set()); - const [editingChangelogUrl, setEditingChangelogUrl] = useState(false); const { providers: sharedProviders } = useSharedProviders(); useEffect(() => { @@ -144,14 +142,10 @@ export function AdminFeaturesPanel() { providerType: entry.providerType, })); }, [sharedProviders]); - const valueFor = (key: string) => - Object.prototype.hasOwnProperty.call(draft, key) - ? draft[key] - : data?.values?.[key]; const currentProviderId = - typeof valueFor('defaultTtsProvider') === 'string' - ? (valueFor('defaultTtsProvider') as string) + typeof draft.defaultTtsProvider === 'string' + ? draft.defaultTtsProvider : ''; const currentSharedEntry: SharedProviderEntry | undefined = sharedProviders.find( (p) => p.slug === currentProviderId, @@ -165,7 +159,7 @@ export function AdminFeaturesPanel() { } as ProviderOption : fallbackShared; const selectedProviderOption = effectiveSelectedProvider; - const showTtsDailyLimitInputs = !Boolean(valueFor('disableTtsRateLimit')); + const shouldRenderRateLimitInputs = draft.disableTtsRateLimit === false; const handleProviderChange = (opt: ProviderOption) => { updateDraft('defaultTtsProvider', opt.id); @@ -256,7 +250,7 @@ export function AdminFeaturesPanel() { { if (!checked) { const ok = confirm( @@ -272,7 +266,7 @@ export function AdminFeaturesPanel() { updateDraft('enableTtsProvidersTab', checked)} right={renderSource('enableTtsProvidersTab')} variant="flat" @@ -280,7 +274,7 @@ export function AdminFeaturesPanel() { updateDraft('showAllProviderModels', checked)} right={renderSource('showAllProviderModels')} variant="flat" @@ -288,82 +282,71 @@ export function AdminFeaturesPanel() { updateDraft('disableTtsRateLimit', checked)} right={renderSource('disableTtsRateLimit')} variant="flat" /> - -
-
-
-
- - {renderSource('ttsDailyLimitAnonymous')} -
- updatePositiveIntDraft('ttsDailyLimitAnonymous', event.target.value)} - /> + {shouldRenderRateLimitInputs ? ( +
+
+
+ + {renderSource('ttsDailyLimitAnonymous')}
-
-
- - {renderSource('ttsDailyLimitAuthenticated')} -
- updatePositiveIntDraft('ttsDailyLimitAuthenticated', event.target.value)} - /> + updatePositiveIntDraft('ttsDailyLimitAnonymous', event.target.value)} + /> +
+
+
+ + {renderSource('ttsDailyLimitAuthenticated')}
-
-
- - {renderSource('ttsIpDailyLimitAnonymous')} -
- updatePositiveIntDraft('ttsIpDailyLimitAnonymous', event.target.value)} - /> + updatePositiveIntDraft('ttsDailyLimitAuthenticated', event.target.value)} + /> +
+
+
+ + {renderSource('ttsIpDailyLimitAnonymous')}
-
-
- - {renderSource('ttsIpDailyLimitAuthenticated')} -
- updatePositiveIntDraft('ttsIpDailyLimitAuthenticated', event.target.value)} - /> + updatePositiveIntDraft('ttsIpDailyLimitAnonymous', event.target.value)} + /> +
+
+
+ + {renderSource('ttsIpDailyLimitAuthenticated')}
+ updatePositiveIntDraft('ttsIpDailyLimitAuthenticated', event.target.value)} + />
- + ) : null}
Feature Flags} > - +
+
+
+

Changelog feed URL

+

+ Public URL to the changelog manifest JSON used by Settings. +

+
+
{renderSource('changelogFeedUrl')}
+
updateDraft('changelogFeedUrl', event.target.value)} placeholder="https://docs.openreader.richardr.dev/changelog/manifest.json" /> - +
updateDraft('enableUserSignups', checked)} right={renderSource('enableUserSignups')} variant="flat" @@ -399,7 +383,7 @@ export function AdminFeaturesPanel() { updateDraft('enableAudiobookExport', checked)} right={renderSource('enableAudiobookExport')} variant="flat" @@ -407,7 +391,7 @@ export function AdminFeaturesPanel() { updateDraft('enableDocxConversion', checked)} right={renderSource('enableDocxConversion')} variant="flat" @@ -415,7 +399,7 @@ export function AdminFeaturesPanel() { updateDraft('enableDestructiveDeleteActions', checked)} right={renderSource('enableDestructiveDeleteActions')} variant="flat" diff --git a/src/components/admin/AdminProvidersPanel.tsx b/src/components/admin/AdminProvidersPanel.tsx index 88ad1b9..b53b15f 100644 --- a/src/components/admin/AdminProvidersPanel.tsx +++ b/src/components/admin/AdminProvidersPanel.tsx @@ -57,6 +57,7 @@ interface FormState { const providerDefaultModel = defaultModelForProviderType; const ADMIN_PROVIDERS_QUERY_KEY = ['admin-providers'] as const; const ADMIN_SETTINGS_QUERY_KEY = ['admin-settings'] as const; +const ADMIN_DEFAULT_PROVIDER_QUERY_KEY = ['admin-settings', 'default-provider-slug'] as const; async function fetchDefaultProviderSlug(): Promise { const res = await fetch('/api/admin/settings'); @@ -160,7 +161,7 @@ export function AdminProvidersPanel() { data: defaultProviderSlug = '', error: defaultProviderError, } = useQuery({ - queryKey: ADMIN_SETTINGS_QUERY_KEY, + queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY, queryFn: fetchDefaultProviderSlug, }); @@ -183,6 +184,7 @@ export function AdminProvidersPanel() { await Promise.all([ queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), ]); }, onError: (mutationError) => { @@ -198,6 +200,7 @@ export function AdminProvidersPanel() { await Promise.all([ queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), ]); }, onError: (mutationError) => { @@ -212,6 +215,7 @@ export function AdminProvidersPanel() { await Promise.all([ queryClient.invalidateQueries({ queryKey: ADMIN_PROVIDERS_QUERY_KEY }), queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), ]); }, onError: (mutationError) => { @@ -223,7 +227,10 @@ export function AdminProvidersPanel() { mutationFn: patchDefaultProviderSlug, onSuccess: async () => { toast.success('Default provider updated'); - await queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ADMIN_SETTINGS_QUERY_KEY }), + queryClient.invalidateQueries({ queryKey: ADMIN_DEFAULT_PROVIDER_QUERY_KEY }), + ]); }, onError: (mutationError) => { console.error('[AdminProvidersPanel] set default failed:', mutationError); diff --git a/src/components/formPrimitives.tsx b/src/components/formPrimitives.tsx index d332594..6095591 100644 --- a/src/components/formPrimitives.tsx +++ b/src/components/formPrimitives.tsx @@ -1,8 +1,6 @@ 'use client'; -import { Fragment, useId, type ReactNode } from 'react'; -import { Transition } from '@headlessui/react'; -import { EditIcon } from '@/components/icons/Icons'; +import { useId, type ReactNode } from 'react'; /** * Shared compact form primitives used by settings-like surfaces across @@ -241,65 +239,6 @@ export function ToggleRow({ ); } -export function EditableRow({ - label, - description, - value, - expanded, - onExpandedChange, - right, - children, - editLabel = 'Edit', -}: { - label: string; - description: string; - value: ReactNode; - expanded: boolean; - onExpandedChange: (expanded: boolean) => void; - right?: ReactNode; - children: ReactNode; - editLabel?: string; -}) { - const rowClass = - 'px-0.5 pt-1 pb-2 border-b border-offbase last:border-b-0 transition-transform duration-200 ease-out hover:scale-[1.003]'; - return ( -
-
-
- {label} - {description} - {value} -
- {right ?
{right}
: null} - -
- -
- {children} -
- -
- ); -} - export function CheckItem({ label, checked, diff --git a/src/components/icons/Icons.tsx b/src/components/icons/Icons.tsx index a780481..74c6e8f 100644 --- a/src/components/icons/Icons.tsx +++ b/src/components/icons/Icons.tsx @@ -193,27 +193,6 @@ export function PlusIcon(props: React.SVGProps) { ); } -export function EditIcon(props: React.SVGProps) { - return ( - - - - - ); -} - export function UploadIcon(props: React.SVGProps) { return ( Date: Sat, 30 May 2026 11:00:01 -0600 Subject: [PATCH 04/16] refactor(worker): modularize worker entrypoint and add vitest infra move worker startup logic from server.ts to runtime.ts for better modularity add worker-loop-policy.ts for loop control abstraction delete legacy unit tests under tests/unit/ related to compute worker and control-plane add vitest config and test directories for compute-core and compute-worker add vitest workflow for CI update package.json scripts for new test commands and add vitest as dev dependency BREAKING CHANGE: worker entrypoint is now compute/worker/src/runtime.ts instead of server.ts; legacy Playwright-based unit tests for compute worker are removed in favor of Vitest --- .github/workflows/vitest.yml | 23 + .../tests/control-plane/orchestrator.test.ts | 95 ++ compute/core/tests/control-plane/sse.test.ts | 26 + .../tests/control-plane/state-machine.test.ts | 76 + compute/worker/src/runtime.ts | 1428 +++++++++++++++++ compute/worker/src/server.ts | 1336 +-------------- compute/worker/src/worker-loop-policy.ts | 22 + compute/worker/tests/api/routes.test.ts | 122 ++ .../tests/fixtures/fake-control-plane.ts | 119 ++ compute/worker/tests/setup-env.ts | 8 + compute/worker/tests/tsconfig.json | 9 + .../tests/unit/jetstream-adapters.test.ts | 134 +- .../tests/unit/key-and-progress.test.ts | 37 + .../tests/unit/worker-loop-policy.test.ts | 18 + compute/worker/tsconfig.json | 2 +- package.json | 6 +- pnpm-lock.yaml | 636 +++++++- tests/unit/compute-abort-like-error.spec.ts | 18 - tests/unit/compute-control-plane.spec.ts | 201 --- tests/unit/compute-worker-op-key.spec.ts | 29 - .../unit/compute-worker-pdf-progress.spec.ts | 32 - vitest.config.ts | 24 + 22 files changed, 2701 insertions(+), 1700 deletions(-) create mode 100644 .github/workflows/vitest.yml create mode 100644 compute/core/tests/control-plane/orchestrator.test.ts create mode 100644 compute/core/tests/control-plane/sse.test.ts create mode 100644 compute/core/tests/control-plane/state-machine.test.ts create mode 100644 compute/worker/src/runtime.ts create mode 100644 compute/worker/src/worker-loop-policy.ts create mode 100644 compute/worker/tests/api/routes.test.ts create mode 100644 compute/worker/tests/fixtures/fake-control-plane.ts create mode 100644 compute/worker/tests/setup-env.ts create mode 100644 compute/worker/tests/tsconfig.json rename tests/unit/compute-worker-control-plane-jetstream.spec.ts => compute/worker/tests/unit/jetstream-adapters.test.ts (55%) create mode 100644 compute/worker/tests/unit/key-and-progress.test.ts create mode 100644 compute/worker/tests/unit/worker-loop-policy.test.ts delete mode 100644 tests/unit/compute-abort-like-error.spec.ts delete mode 100644 tests/unit/compute-control-plane.spec.ts delete mode 100644 tests/unit/compute-worker-op-key.spec.ts delete mode 100644 tests/unit/compute-worker-pdf-progress.spec.ts create mode 100644 vitest.config.ts diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml new file mode 100644 index 0000000..652663a --- /dev/null +++ b/.github/workflows/vitest.yml @@ -0,0 +1,23 @@ +name: Vitest Tests + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + vitest-testing: + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: lts/* + package-manager-cache: false + - uses: pnpm/action-setup@v6 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Run Vitest suites + run: pnpm test:vitest diff --git a/compute/core/tests/control-plane/orchestrator.test.ts b/compute/core/tests/control-plane/orchestrator.test.ts new file mode 100644 index 0000000..2a03adf --- /dev/null +++ b/compute/core/tests/control-plane/orchestrator.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, test } from 'vitest'; +import type { WorkerOperationRequest } from '../../src/api-contracts'; +import { + InMemoryOperationEventStream, + InMemoryOperationQueue, + InMemoryOperationStateStore, + OperationOrchestrator, +} from '../../src/control-plane'; + +function buildRequest(opKey: string): WorkerOperationRequest { + return { + kind: 'pdf_layout', + opKey, + payload: { + documentId: `doc-${opKey}`, + namespace: null, + documentObjectKey: `s3://bucket/${opKey}.pdf`, + }, + }; +} + +describe('operation orchestrator', () => { + test('reuses fresh operation and replaces stale operation', async () => { + let now = 1_000; + let nextId = 1; + const queue = new InMemoryOperationQueue(); + const stateStore = new InMemoryOperationStateStore(); + const eventStream = new InMemoryOperationEventStream(); + + const orchestrator = new OperationOrchestrator({ + queue, + stateStore, + eventStream, + config: { opStaleMs: 2_000, maxCasRetries: 5 }, + clock: { now: () => now }, + idFactory: { + opId: () => `op-${nextId}`, + jobId: () => `job-${nextId++}`, + }, + }); + + const request = buildRequest('shared-op'); + + const first = await orchestrator.enqueueOrReuse(request); + expect(first.opId).toBe('op-1'); + + now = 2_000; + const reused = await orchestrator.enqueueOrReuse(request); + expect(reused.opId).toBe('op-1'); + + await orchestrator.markRunning({ opId: first.opId, updatedAt: 2_100 }); + + now = 6_000; + const replaced = await orchestrator.enqueueOrReuse(request); + expect(replaced.opId).toBe('op-2'); + expect(await stateStore.getOpIndex('shared-op')).toEqual({ opId: 'op-2' }); + expect(queue.size('pdf_layout')).toBe(2); + }); + + test('survives transient CAS conflict and eventually creates operation', async () => { + const queue = new InMemoryOperationQueue(); + const eventStream = new InMemoryOperationEventStream(); + const store = new InMemoryOperationStateStore(); + + let firstAttempt = true; + const conflictStore = { + getOpState: store.getOpState.bind(store), + putOpState: store.putOpState.bind(store), + getOpIndex: store.getOpIndex.bind(store), + compareAndSetOpIndex: async (input: { opKey: string; newOpId: string; expectedOpId: string | null }) => { + if (firstAttempt && input.expectedOpId === null) { + firstAttempt = false; + return false; + } + return store.compareAndSetOpIndex(input); + }, + }; + + let id = 1; + const orchestrator = new OperationOrchestrator({ + queue, + stateStore: conflictStore, + eventStream, + config: { opStaleMs: 2_000, maxCasRetries: 4 }, + idFactory: { + opId: () => `op-${id}`, + jobId: () => `job-${id++}`, + }, + }); + + const created = await orchestrator.enqueueOrReuse(buildRequest('cas-key')); + expect(created.opId).toMatch(/^op-/); + expect(await store.getOpIndex('cas-key')).toEqual({ opId: created.opId }); + }); +}); diff --git a/compute/core/tests/control-plane/sse.test.ts b/compute/core/tests/control-plane/sse.test.ts new file mode 100644 index 0000000..e207950 --- /dev/null +++ b/compute/core/tests/control-plane/sse.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from 'vitest'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse'; + +describe('sse codec', () => { + test('encodes event id and payload and decodes both reliably', () => { + const frame = encodeSseFrame({ + id: 42, + event: 'snapshot', + data: { ok: true, opId: 'op-1' }, + }); + + expect(frame).toContain('event: snapshot'); + expect(parseSseEventId(frame)).toBe(42); + expect(parseSsePayload(frame)).toBe('{"ok":true,"opId":"op-1"}'); + }); + + test('supports multiline data payload', () => { + const frame = encodeSseFrame({ + id: 5, + data: 'line1\nline2', + }); + + expect(parseSseEventId(frame)).toBe(5); + expect(parseSsePayload(frame)).toBe('line1\nline2'); + }); +}); diff --git a/compute/core/tests/control-plane/state-machine.test.ts b/compute/core/tests/control-plane/state-machine.test.ts new file mode 100644 index 0000000..e9617cb --- /dev/null +++ b/compute/core/tests/control-plane/state-machine.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from 'vitest'; +import type { WorkerOperationState } from '../../src/api-contracts'; +import { + explainReplacementReason, + isInflightStatus, + isTerminalStatus, + shouldReuseExistingOperation, +} from '../../src/control-plane/state-machine'; + +function runningState(overrides: Partial = {}): WorkerOperationState { + return { + opId: 'op-1', + opKey: 'key-1', + kind: 'pdf_layout', + jobId: 'job-1', + status: 'running', + queuedAt: 1_000, + updatedAt: 2_000, + ...overrides, + }; +} + +describe('state-machine decisions', () => { + test('identifies terminal and inflight states', () => { + expect(isTerminalStatus('succeeded')).toBe(true); + expect(isTerminalStatus('failed')).toBe(true); + expect(isTerminalStatus('queued')).toBe(false); + + expect(isInflightStatus('queued')).toBe(true); + expect(isInflightStatus('running')).toBe(true); + expect(isInflightStatus('failed')).toBe(false); + }); + + test('reuses fresh inflight operation and rejects stale inflight operation', () => { + const current = runningState(); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 2_900, + opStaleMs: 1_000, + })).toBe(true); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 3_100, + opStaleMs: 1_000, + })).toBe(false); + + expect(explainReplacementReason({ + current, + requestKind: 'pdf_layout', + now: 3_100, + opStaleMs: 1_000, + })).toBe('stale_running'); + }); + + test('never reuses kind-mismatched operation', () => { + const current = runningState({ kind: 'whisper_align' }); + + expect(shouldReuseExistingOperation({ + current, + requestKind: 'pdf_layout', + now: 2_100, + opStaleMs: 10_000, + })).toBe(false); + + expect(explainReplacementReason({ + current, + requestKind: 'pdf_layout', + now: 2_100, + opStaleMs: 10_000, + })).toBe('kind_mismatch'); + }); +}); diff --git a/compute/worker/src/runtime.ts b/compute/worker/src/runtime.ts new file mode 100644 index 0000000..1804855 --- /dev/null +++ b/compute/worker/src/runtime.ts @@ -0,0 +1,1428 @@ +import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import { + connect, + nanos, + credsAuthenticator, + type NatsConnection, +} from '@nats-io/transport-node'; +import { + AckPolicy, + DeliverPolicy, + ReplayPolicy, + RetentionPolicy, + StorageType, + jetstream, + jetstreamManager, + type Consumer, + type JetStreamClient, + type JetStreamManager, + type JsMsg, +} from '@nats-io/jetstream'; +import { Kvm } from '@nats-io/kv'; +import { + ensureComputeModels, + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, +} from '@openreader/compute-core/local-runtime'; +import { + getComputeTimeoutConfig, + getComputeOpStaleMs, + getAvailableCpuCores, + getOnnxThreadsPerJob, + withIdleTimeoutAndHardCap, + withTimeout, +} from '@openreader/compute-core'; +import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + WorkerOperationEvent, + WhisperAlignJobRequest, + WhisperAlignJobResult, + WorkerJobState, + WorkerJobTiming, + WorkerOperationKind, + WorkerOperationRequest, + WorkerOperationState, + PdfLayoutProgress, +} from '@openreader/compute-core/api-contracts'; +import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + JetStreamOperationEventStream, + OP_EVENTS_SUBJECT_WILDCARD, + JetStreamOperationQueue, + JetStreamOperationStateStore, + hashOpKey, +} from './control-plane/jetstream'; +import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; +import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; +import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; + +const JOBS_STREAM_NAME = 'compute_jobs'; +const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; +const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; +const WHISPER_CONSUMER_NAME = 'compute_whisper'; +const LAYOUT_CONSUMER_NAME = 'compute_layout'; +const EVENTS_STREAM_NAME = 'compute_events'; +const COMPUTE_STATE_BUCKET = 'compute_state'; +const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; +const LOOP_ERROR_BACKOFF_MS = 500; +const RUNNING_HEARTBEAT_MS = 5000; +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const WHISPER_MAX_DELIVER = 1; +const NATS_API_TIMEOUT_MS = 60_000; +// Disconnect from NATS after this much continuous idle so the worker stops +// generating outbound traffic (pull polling + keepalive PINGs) and Railway can +// put it to sleep. Reconnect happens lazily on the next inbound request. +const IDLE_DISCONNECT_MS = 120_000; +const IDLE_CHECK_INTERVAL_MS = 5_000; +// Bounded pull window so consumer loops yield periodically and can be stopped +// cleanly when going idle, instead of blocking on a long-lived pull. +const PULL_EXPIRES_MS = 5_000; +const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); +const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); +const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { + whisper_align: 15_000, + pdf_layout: 120_000, +}; + +interface QueuedJob { + jobId: string; + opId: string; + opKey: string; + kind: WorkerOperationKind; + queuedAt: number; + payload: TPayload; +} + +interface NatsSession { + nc: NatsConnection; + js: JetStreamClient; + jsm: JetStreamManager; + kv: Awaited>; + whisperConsumer: Consumer; + layoutConsumer: Consumer; +} + +type StreamedOperationState = WorkerOperationState; + +interface OperationEventStreamLike { + subscribe(input: { + opId: string; + sinceEventId?: number; + onEvent: (event: WorkerOperationEvent) => void | Promise; + onError?: (error: unknown) => void; + }): Promise<() => void>; +} + +interface OperationStateStoreLike { + getOpState(opId: string): Promise; +} + +interface OrchestratorLike { + enqueueOrReuse(request: WorkerOperationRequest): Promise; + markRunning(input: { + opId: string; + startedAt?: number; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markProgress(input: { + opId: string; + progress: WorkerOperationState['progress']; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { + opId: string; + result: unknown; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +export interface ComputeWorkerRouteDeps { + orchestrator: OrchestratorLike; + operationStateStore: OperationStateStoreLike; + operationEventStream: OperationEventStreamLike; +} + +export interface CreateComputeWorkerAppOptions { + host?: string; + port?: number; + workerToken?: string; + routeDeps?: ComputeWorkerRouteDeps; + disableWorkers?: boolean; +} + +export interface ComputeWorkerApp { + app: FastifyInstance; + host: string; + port: number; + start(options?: { registerSignalHandlers?: boolean }): Promise; + close(): Promise; +} + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function readIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +function normalizeNatsReplicas(value: number): number { + if (value === 3 || value === 5) return value; + return 1; +} + +function parseBoolEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const normalized = raw.toLowerCase(); + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; +} + +function buildLoggerConfig(): boolean | Record { + const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); + const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; + if (format === 'json') { + return { + level, + base: null, + }; + } + return { + level, + base: null, + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + }, + }; +} + +function normalizeS3Prefix(prefix: string | undefined): string { + const value = (prefix || 'openreader').trim(); + return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; +} + +function sanitizeNamespace(namespace: string | null): string | null { + if (!namespace) return null; + return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; +} + +function documentParsedKey(id: string, namespace: string | null, prefix: string): string { + if (!DOCUMENT_ID_REGEX.test(id)) { + throw new Error(`Invalid document id: ${id}`); + } + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + return `${prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; +} + +function buildS3Client(): S3Client { + const bucket = requireEnv('S3_BUCKET'); + const region = requireEnv('S3_REGION'); + const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); + const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); + const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; + const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); + + void bucket; + + return new S3Client({ + region, + endpoint, + forcePathStyle, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + if (typeof body === 'object' && body !== null && 'on' in body) { + const stream = body as NodeJS.ReadableStream; + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); + } + return Buffer.concat(chunks); + } + throw new Error('Unsupported S3 response body type'); +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function isAuthed(request: FastifyRequest, expectedToken: string): boolean { + const auth = request.headers.authorization; + if (!auth?.startsWith('Bearer ')) return false; + const token = auth.slice('Bearer '.length).trim(); + return token === expectedToken; +} + +function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { + if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; + return Math.max(0, Math.floor((end as number) - (start as number))); +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + return String(error); +} + +function requestPath(request: FastifyRequest): string { + return request.url.split('?')[0] ?? request.url; +} + +function isHealthPath(path: string): boolean { + return path === '/health/live' || path === '/health/ready'; +} + +function extractTraceId(request: FastifyRequest): string | null { + const header = request.headers['x-openreader-trace-id']; + if (Array.isArray(header)) return header[0] ?? null; + return typeof header === 'string' ? header : null; +} + +function extractOpId(request: FastifyRequest, path: string): string | null { + const params = request.params as { opId?: unknown } | undefined; + if (params && typeof params.opId === 'string' && params.opId.trim()) { + return params.opId.trim(); + } + const match = path.match(/^\/ops\/([^/]+)/); + if (!match?.[1]) return null; + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } +} + +function isAlreadyExistsError(error: unknown): boolean { + const message = toErrorMessage(error).toLowerCase(); + return message.includes('already in use') || message.includes('already exists'); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +class ConcurrencyGate { + private readonly maxInFlight: number; + private inFlight = 0; + private readonly queue: Array<() => void> = []; + + constructor(limit: number) { + this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; + } + + async acquire(): Promise { + if (this.inFlight < this.maxInFlight) { + this.inFlight += 1; + return; + } + + await new Promise((resolve) => { + this.queue.push(() => { + this.inFlight += 1; + resolve(); + }); + }); + } + + release(): void { + this.inFlight = Math.max(0, this.inFlight - 1); + const next = this.queue.shift(); + if (next) next(); + } +} + +function isTerminalStatus(status: WorkerJobState): boolean { + return status === 'succeeded' || status === 'failed'; +} + +function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { + if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; + const maybe = result as { parsedObjectKey?: unknown }; + return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; +} + +const alignSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +const layoutSchema = z.object({ + documentId: z.string().trim().min(1), + namespace: z.string().trim().min(1).max(128).nullable(), + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +const operationCreateSchema = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('whisper_align'), + opKey: z.string().trim().min(1).max(1024), + payload: alignSchema, + }), + z.object({ + kind: z.literal('pdf_layout'), + opKey: z.string().trim().min(1).max(1024), + payload: layoutSchema, + }), +]); + +async function ensureJetStreamResources( + jsm: JetStreamManager, + whisperTimeoutMs: number, + pdfTimeoutMs: number, + pdfAttempts: number, + jobsMaxBytes: number, + eventsMaxBytes: number, + natsReplicas: number, +): Promise { + const streamConfig = { + name: JOBS_STREAM_NAME, + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + retention: RetentionPolicy.Workqueue, + storage: StorageType.File, + max_bytes: jobsMaxBytes, + num_replicas: natsReplicas, + }; + + try { + await jsm.streams.add(streamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.streams.update(JOBS_STREAM_NAME, { + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + max_bytes: jobsMaxBytes, + num_replicas: natsReplicas, + }); + } + + const eventsStreamConfig = { + name: EVENTS_STREAM_NAME, + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + retention: RetentionPolicy.Limits, + storage: StorageType.File, + max_bytes: eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: natsReplicas, + }; + + try { + await jsm.streams.add(eventsStreamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.streams.update(EVENTS_STREAM_NAME, { + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + max_bytes: eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: natsReplicas, + }); + } + + const ensureConsumer = async ( + name: string, + subject: string, + ackWaitMs: number, + maxDeliver: number, + ): Promise => { + const config = { + durable_name: name, + ack_policy: AckPolicy.Explicit, + deliver_policy: DeliverPolicy.All, + replay_policy: ReplayPolicy.Instant, + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }; + + try { + await jsm.consumers.add(JOBS_STREAM_NAME, config); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await jsm.consumers.update(JOBS_STREAM_NAME, name, { + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }); + } + }; + + await Promise.all([ + ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), + ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), + ]); +} + +export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { + const port = options.port ?? readIntEnv('PORT', 8081); + const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); + const workerToken = options.workerToken ?? requireEnv('COMPUTE_WORKER_TOKEN'); + const disableWorkers = options.disableWorkers ?? false; + const natsUrl = requireEnv('NATS_URL'); + const timeoutConfig = getComputeTimeoutConfig(); + + const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); + const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; + const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; + const pdfHardCapMs = timeoutConfig.pdfHardCapMs; + const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); + const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); + const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); + const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); + const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); + const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); + const opStaleMs = getComputeOpStaleMs(); + + const connectOpts: Parameters[0] = { servers: natsUrl }; + const natsCreds = process.env.NATS_CREDS?.trim(); + const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); + + if (natsCreds) { + console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); + connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); + } else if (natsCredsFile) { + console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); + const { readFileSync } = await import('node:fs'); + const credsData = readFileSync(natsCredsFile); + connectOpts.authenticator = credsAuthenticator(credsData); + } + + // Lazy NATS connection lifecycle. The worker connects on demand (first request + // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle + // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is + // transparent: any inbound /ops request both wakes the container and re-establishes + // the session via ensureConnected(). + let session: NatsSession | null = null; + let connecting: Promise | null = null; + let workerLoops: Promise[] = []; + let idleTimer: NodeJS.Timeout | null = null; + let stopping = false; + let loopStopRequested = false; + + // Activity accounting feeding the idle detector. The worker is considered idle + // only when no HTTP request is in flight, no SSE stream is open, no job is + // processing, and nothing has happened for IDLE_DISCONNECT_MS. + let inFlightHttp = 0; + let activeSse = 0; + let inFlightJobs = 0; + let lastActivityAt = Date.now(); + const jobGate = new ConcurrencyGate(jobConcurrency); + + const markActivity = (): void => { + lastActivityAt = Date.now(); + }; + + function startIdleTimer(): void { + if (idleTimer) return; + idleTimer = setInterval(() => { + if (!session || stopping) return; + if (inFlightHttp > 0 || activeSse > 0 || inFlightJobs > 0) return; + if (Date.now() - lastActivityAt < IDLE_DISCONNECT_MS) return; + void disconnect('idle'); + }, IDLE_CHECK_INTERVAL_MS); + // Don't let the idle checker keep the process alive on its own. + idleTimer.unref?.(); + } + + async function disconnect(reason: string): Promise { + const current = session; + if (!current) return; + // Clear synchronously (before any await) so concurrent requests reconnect a + // fresh session instead of using the connection we're about to close. + session = null; + loopStopRequested = true; + if (idleTimer) { + clearInterval(idleTimer); + idleTimer = null; + } + try { + await current.nc.close(); + } catch { + // ignore close errors + } + await Promise.allSettled(workerLoops); + workerLoops = []; + loopStopRequested = false; + app.log.info({ reason }, 'nats disconnected'); + } + + async function ensureConnected(): Promise { + if (session) return session; + if (connecting) return connecting; + connecting = (async () => { + const nc: NatsConnection = await connect(connectOpts); + const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); + const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); + await ensureJetStreamResources( + jsm, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + jobsStreamMaxBytes, + eventsStreamMaxBytes, + natsReplicas, + ); + const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { + replicas: natsReplicas, + history: 1, + ttl: COMPUTE_STATE_TTL_MS, + max_bytes: jobStatesMaxBytes, + }); + const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); + const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); + const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; + session = next; + markActivity(); + startWorkerLoops(next); + startIdleTimer(); + // Safety net: if the connection closes for any reason (network drop after + // exhausting reconnects, or our own disconnect), drop the stale session so + // the next request reconnects cleanly. + void nc.closed().then(() => { + if (session?.nc === nc) session = null; + }); + app.log.info('nats connected'); + return next; + })(); + try { + return await connecting; + } finally { + connecting = null; + } + } + + const s3 = disableWorkers ? null : buildS3Client(); + const s3Bucket = disableWorkers ? '' : requireEnv('S3_BUCKET'); + const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); + + const ensureSafeKey = (key: string): string => { + const trimmed = key.trim(); + if (!trimmed.startsWith(`${s3Prefix}/`)) { + throw new Error('Object key prefix mismatch'); + } + return trimmed; + }; + + const readObjectByKey = async (key: string): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const safeKey = ensureSafeKey(key); + const response = await s3.send(new GetObjectCommand({ + Bucket: s3Bucket, + Key: safeKey, + })); + const bytes = await bodyToBuffer(response.Body); + return toArrayBuffer(new Uint8Array(bytes)); + }; + + const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const key = documentParsedKey(documentId, namespace, s3Prefix); + const body = Buffer.from(JSON.stringify(parsed)); + await s3.send(new PutObjectCommand({ + Bucket: s3Bucket, + Key: key, + Body: body, + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + })); + return key; + }; + + if (prewarmModels && !disableWorkers) { + await ensureComputeModels(); + } + + const app = Fastify({ + logger: buildLoggerConfig(), + disableRequestLogging: true, + }); + app.log.info({ + jobConcurrency, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + opStaleMs, + availableCpuCores: getAvailableCpuCores(), + onnxThreadsPerJob: getOnnxThreadsPerJob(), + natsApiTimeoutMs: NATS_API_TIMEOUT_MS, + natsReplicas, + eventsStreamMaxBytes, + pdfLayoutHardCapMs: pdfHardCapMs, + }, 'compute runtime config'); + + const whisperJobCodec = createJsonCodec>(); + const layoutJobCodec = createJsonCodec>(); + + const defaultOperationStateStore = new JetStreamOperationStateStore({ + getKv: async () => (await ensureConnected()).kv, + }); + + const defaultOperationEventStream = new JetStreamOperationEventStream({ + getJs: async () => (await ensureConnected()).js, + getJsm: async () => (await ensureConnected()).jsm, + eventsStreamName: EVENTS_STREAM_NAME, + }); + + const operationQueue = new JetStreamOperationQueue({ + getJs: async () => (await ensureConnected()).js, + whisperSubject: WHISPER_JOBS_SUBJECT, + layoutSubject: LAYOUT_JOBS_SUBJECT, + }); + + const defaultOrchestrator = new OperationOrchestrator({ + queue: operationQueue, + stateStore: defaultOperationStateStore, + eventStream: defaultOperationEventStream, + config: { + opStaleMs, + maxCasRetries: 10, + }, + }); + + const operationStateStore = options.routeDeps?.operationStateStore ?? defaultOperationStateStore; + const operationEventStream = options.routeDeps?.operationEventStream ?? defaultOperationEventStream; + const orchestrator = options.routeDeps?.orchestrator ?? defaultOrchestrator; + + const getOpState = async (opId: string): Promise => { + return await operationStateStore.getOpState(opId); + }; + + const releaseHttp = (request: FastifyRequest): void => { + const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; + if (!counted[REQUEST_COUNTED_KEY]) return; + counted[REQUEST_COUNTED_KEY] = false; + inFlightHttp = Math.max(0, inFlightHttp - 1); + markActivity(); + }; + + app.addHook('onRequest', async (request, reply) => { + const path = requestPath(request); + (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); + // Count every request as in-flight activity so the idle detector never + // disconnects mid-request. Released in onResponse, or manually after hijack + // for SSE streams (where onResponse does not fire). + (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; + inFlightHttp += 1; + markActivity(); + if (isHealthPath(path)) return; + if (!isAuthed(request, workerToken)) { + return reply.code(401).send({ error: 'Unauthorized' }); + } + return; + }); + + app.addHook('onResponse', async (request, reply) => { + releaseHttp(request); + const path = requestPath(request); + if (isHealthPath(path)) return; + if (reply.statusCode >= 500) { + const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; + const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; + app.log.error({ + reqId: request.id, + method: request.method, + path, + statusCode: reply.statusCode, + durationMs, + traceId: extractTraceId(request) ?? null, + opId: extractOpId(request, path), + }, 'http.error'); + } + }); + + app.get('/health/live', async () => ({ ok: true })); + + // Reports readiness without forcing a NATS round-trip. Probing NATS here would + // reconnect (and keep) the connection open, defeating idle sleep, so we only + // report the current connection state. The worker reconnects lazily on the next + // /ops request regardless of what this returns. + app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); + + app.post('/ops', async (request, reply) => { + const parsed = operationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { + error: 'Invalid request body', + issues: parsed.error.issues, + }; + } + + const requestOp = parsed.data as WorkerOperationRequest; + const op = await orchestrator.enqueueOrReuse(requestOp); + app.log.info({ + kind: requestOp.kind, + opId: op.opId, + jobId: op.jobId, + status: op.status, + opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), + }, 'op.accepted'); + reply.code(202); + return op; + }); + + app.get('/ops/:opId', async (request, reply) => { + const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const state = await getOpState(params.data.opId); + if (!state) { + reply.code(404); + return { error: 'Operation not found' }; + } + + return state; + }); + + app.get('/ops/:opId/events', async (request, reply) => { + const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const initial = await getOpState(params.data.opId); + if (!initial) { + reply.code(404); + return { error: 'Operation not found' }; + } + + const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; + const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); + const lastEventIdHeader = request.headers['last-event-id']; + const cursorFromHeader = Number( + Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), + ); + const sinceEventId = Math.max( + 0, + Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, + Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, + ); + + reply.hijack(); + // onResponse will not fire for a hijacked reply, so release the HTTP in-flight + // count here and track the long-lived stream via activeSse instead. + releaseHttp(request); + activeSse += 1; + markActivity(); + reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); + reply.raw.setHeader('Connection', 'keep-alive'); + reply.raw.setHeader('X-Accel-Buffering', 'no'); + + let closed = false; + let unsubscribe: (() => void) | null = null; + + const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { + if (closed || reply.raw.writableEnded) return; + const frameEvent: WorkerOperationEvent = { + eventId, + snapshot, + }; + reply.raw.write(encodeSseFrame({ + id: eventId, + event: 'snapshot', + data: frameEvent, + })); + }; + + const closeStream = (): void => { + if (closed) return; + closed = true; + if (unsubscribe) { + unsubscribe(); + unsubscribe = null; + } + activeSse = Math.max(0, activeSse - 1); + markActivity(); + if (!reply.raw.writableEnded) { + reply.raw.end(); + } + }; + + request.raw.on('close', () => { + closeStream(); + }); + + try { + let current = initial; + let signature = JSON.stringify(current); + writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); + if (isTerminalStatus(current.status)) { + return reply; + } + + unsubscribe = await operationEventStream.subscribe({ + opId: params.data.opId, + sinceEventId, + onEvent: (event) => { + if (closed) return; + if (event.snapshot.opId !== params.data.opId) return; + const nextSignature = JSON.stringify(event.snapshot); + if (nextSignature !== signature) { + current = event.snapshot; + signature = nextSignature; + writeSnapshot(current, event.eventId); + } + if (isTerminalStatus(event.snapshot.status)) { + closeStream(); + } + }, + onError: (error) => { + app.log.warn({ + opId: params.data.opId, + error: toErrorMessage(error), + }, 'op events stream loop error'); + closeStream(); + }, + }); + + await new Promise((resolve) => { + request.raw.once('close', () => resolve()); + }); + } catch (error) { + app.log.warn({ + opId: params.data.opId, + error: toErrorMessage(error), + }, 'op events stream loop error'); + } finally { + closeStream(); + } + + return reply; + }); + + const runWhisper = async ( + payload: WhisperAlignJobRequest, + queueWaitMs: number, + ): Promise => { + const parsed = alignSchema.parse(payload); + + const s3FetchStartedAt = Date.now(); + const audioBuffer = await withTimeout( + readObjectByKey(parsed.audioObjectKey), + whisperTimeoutMs, + 'whisper s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + + const computeStartedAt = Date.now(); + const result = await withTimeout( + runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: parsed.text, + cacheKey: parsed.cacheKey, + lang: parsed.lang, + }), + whisperTimeoutMs, + 'whisper alignment job', + ); + + const computeMs = Date.now() - computeStartedAt; + return { + ...result, + timing: { + queueWaitMs, + s3FetchMs, + computeMs, + }, + }; + }; + + const runLayout = async ( + payload: PdfLayoutJobRequest, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ): Promise => { + const parsed = layoutSchema.parse(payload); + + const s3FetchStartedAt = Date.now(); + const pdfBytes = await withTimeout( + readObjectByKey(parsed.documentObjectKey), + Math.max(pdfTimeoutMs, 1_000), + 'pdf s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + + let lastTotalPages = 0; + let lastPagesParsed = 0; + const computeStartedAt = Date.now(); + const result = await withIdleTimeoutAndHardCap({ + idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), + hardCapMs: pdfHardCapMs, + label: 'pdf layout job', + run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ + documentId: parsed.documentId, + pdfBytes, + onPageStarted: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + if (!hooks?.onProgress) return; + await hooks.onProgress(buildInferProgressForPageStart({ + pageNumber, + totalPages, + })); + }, + onPageParsed: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + lastPagesParsed = pageNumber; + if (!hooks?.onProgress) return; + await hooks.onProgress(buildInferProgressForPageParsed({ + pageNumber, + totalPages, + })); + }, + }), + }); + + const computeMs = Date.now() - computeStartedAt; + if (hooks?.onProgress && lastTotalPages > 0) { + await hooks.onProgress({ + totalPages: lastTotalPages, + pagesParsed: lastPagesParsed, + currentPage: lastPagesParsed || undefined, + phase: 'merge', + }); + } + const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); + return { + parsedObjectKey, + timing: { + queueWaitMs, + s3FetchMs, + computeMs, + }, + }; + }; + + type ProcessMessageInput = { + msg: JsMsg; + codec: JsonCodec>; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + workerLabel: string; + }; + + type ProcessMessageContext = { + decoded: QueuedJob; + workerLabel: string; + startedAt: number; + queueWaitTiming?: { queueWaitMs: number }; + latestProgress?: PdfLayoutProgress; + }; + + function extractTiming(result: unknown): WorkerJobTiming | undefined { + if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; + return (result as { timing?: WorkerJobTiming }).timing; + } + + async function markRunning( + context: ProcessMessageContext, + updatedAt: number, + options?: { includeStartedAt?: boolean }, + ): Promise { + if (context.latestProgress) { + await orchestrator.markProgress({ + opId: context.decoded.opId, + progress: context.latestProgress, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + return; + } + + await orchestrator.markRunning({ + opId: context.decoded.opId, + ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + } + + async function markProgress( + context: ProcessMessageContext, + progress: PdfLayoutProgress, + updatedAt: number, + ): Promise { + context.latestProgress = progress; + await markRunning(context, updatedAt); + } + + async function markTerminal(input: { + context: ProcessMessageContext; + status: 'succeeded' | 'failed'; + result?: TResult; + errorMessage?: string; + timing?: WorkerJobTiming; + updatedAt: number; + }): Promise { + if (input.status === 'succeeded') { + await orchestrator.markSucceeded({ + opId: input.context.decoded.opId, + result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, + updatedAt: input.updatedAt, + ...(input.timing ? { timing: input.timing } : {}), + }); + return; + } + + await orchestrator.markFailed({ + opId: input.context.decoded.opId, + error: { message: input.errorMessage ?? 'unknown worker failure' }, + updatedAt: input.updatedAt, + ...(input.timing ? { timing: input.timing } : {}), + }); + } + + async function handleRetry(input: { + context: ProcessMessageContext | null; + msg: JsMsg; + errorMessage: string; + }): Promise { + const deliveryCount = input.msg.info.deliveryCount; + const kind = input.context?.decoded.kind ?? 'pdf_layout'; + const retryAction = decideRetryAction({ + kind, + deliveryCount, + pdfAttempts, + whisperMaxDeliver: WHISPER_MAX_DELIVER, + }); + const hasRetriesLeft = retryAction === 'nak_retry'; + const maxAttempts = kind === 'whisper_align' ? WHISPER_MAX_DELIVER : pdfAttempts; + + if (input.context) { + const now = Date.now(); + const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); + const persistOpUpdate = hasRetriesLeft + ? (input.context.latestProgress + ? orchestrator.markProgress({ + opId: input.context.decoded.opId, + progress: input.context.latestProgress, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + }) + : orchestrator.markRunning({ + opId: input.context.decoded.opId, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + })) + : markTerminal({ + context: input.context, + status: 'failed', + errorMessage: input.errorMessage, + updatedAt: now, + ...(retryTiming ? { timing: retryTiming } : {}), + }); + + await persistOpUpdate.catch((stateError) => { + app.log.error({ + worker: input.context?.workerLabel, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation state'); + }); + } + + if (hasRetriesLeft) { + input.msg.nak(); + app.log.error({ + worker: input.context?.workerLabel, + kind: input.context?.decoded.kind, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + status: 'running', + error: input.errorMessage, + deliveryCount, + maxAttempts, + retryAction: 'nack_retry', + }, 'job.terminal'); + return; + } + + input.msg.term(input.errorMessage); + app.log.error({ + worker: input.context?.workerLabel, + kind: input.context?.decoded.kind, + opId: input.context?.decoded.opId, + jobId: input.context?.decoded.jobId, + status: 'failed', + error: input.errorMessage, + deliveryCount, + maxAttempts, + retrySuppressed: kind === 'whisper_align' ? 'whisper_align' : undefined, + retryAction: 'term', + }, 'job.terminal'); + } + + async function processMessage(input: ProcessMessageInput): Promise { + let context: ProcessMessageContext | null = null; + let heartbeat: NodeJS.Timeout | null = null; + try { + const decoded = input.codec.decode(input.msg.data); + const startedAt = Date.now(); + context = { + decoded, + workerLabel: input.workerLabel, + startedAt, + queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), + }; + + await markRunning(context, startedAt); + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, + deliveryCount: input.msg.info.deliveryCount, + }, 'job.started'); + + heartbeat = setInterval(() => { + const now = Date.now(); + void markRunning(context!, now).catch((stateError) => { + app.log.error({ + worker: input.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation heartbeat state'); + }); + }, RUNNING_HEARTBEAT_MS); + + const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { + onProgress: async (progress) => { + await markProgress(context!, progress, Date.now()); + }, + }); + const resultTiming = extractTiming(result); + const now = Date.now(); + + await markTerminal({ + context, + status: 'succeeded', + result, + timing: resultTiming, + updatedAt: now, + }); + + input.msg.ack(); + const terminalDurationMs = safeDurationMs(context.startedAt, now); + const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; + if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + durationMs: terminalDurationMs ?? null, + timing: resultTiming ?? null, + }, 'job.stage'); + } + app.log.info({ + worker: input.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + status: 'succeeded', + durationMs: terminalDurationMs ?? null, + resultRef: extractResultRef(decoded.kind, result), + timing: resultTiming ?? null, + }, 'job.terminal'); + } catch (error) { + await handleRetry({ + context, + msg: input.msg, + errorMessage: toErrorMessage(error), + }); + } finally { + if (heartbeat) clearInterval(heartbeat); + } + } + + async function createWorkerLoop(input: { + owner: NatsSession; + consumer: Consumer; + codec: JsonCodec>; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + workerLabel: string; + }): Promise { + // Exit when the loop's connection is no longer the active session (idle + // disconnect, unexpected close, or replaced by a reconnect). + const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; + while (!detached()) { + let msg: JsMsg | null = null; + try { + try { + // Bounded pull so the loop yields periodically and exits promptly when + // the session is torn down for idle (nc.close() rejects the pending pull). + msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); + } catch (error) { + if (detached()) return; + app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); + await sleep(LOOP_ERROR_BACKOFF_MS); + continue; + } + + // An empty pull is not activity; let the idle window advance. + if (!msg) continue; + markActivity(); + inFlightJobs += 1; + await jobGate.acquire(); + if (detached()) { + return; + } + await processMessage({ + msg, + codec: input.codec, + run: input.run, + workerLabel: input.workerLabel, + }); + } finally { + if (msg) { + jobGate.release(); + inFlightJobs = Math.max(0, inFlightJobs - 1); + markActivity(); + } + } + } + } + + function startWorkerLoops(active: NatsSession): void { + if (disableWorkers) return; + // Always starts a fresh set bound to the new session. Any loops from a prior + // session self-terminate once they observe session !== their owner. + workerLoops = []; + loopStopRequested = false; + for (let i = 0; i < jobConcurrency; i += 1) { + workerLoops.push(createWorkerLoop({ + owner: active, + consumer: active.whisperConsumer, + codec: whisperJobCodec, + run: runWhisper, + workerLabel: `whisper-${i + 1}`, + })); + } + for (let i = 0; i < jobConcurrency; i += 1) { + workerLoops.push(createWorkerLoop({ + owner: active, + consumer: active.layoutConsumer, + codec: layoutJobCodec, + run: runLayout, + workerLabel: `layout-${i + 1}`, + })); + } + } + + const close = async (): Promise => { + if (stopping) return; + stopping = true; + if (idleTimer) { + clearInterval(idleTimer); + idleTimer = null; + } + await app.close(); + await Promise.allSettled(workerLoops); + const current = session; + session = null; + if (current) { + try { + await current.nc.drain(); + } catch { + try { + await current.nc.close(); + } catch { + // ignore close errors + } + } + } + }; + + const registerSignalHandlers = (): void => { + process.once('SIGINT', () => { + void close().finally(() => process.exit(0)); + }); + + process.once('SIGTERM', () => { + void close().finally(() => process.exit(0)); + }); + }; + + return { + app, + host, + port, + async start(startOptions) { + if (startOptions?.registerSignalHandlers) { + registerSignalHandlers(); + } + await app.listen({ host, port }); + app.log.info({ host, port }, 'compute worker listening'); + }, + close, + }; +} + +export async function startComputeWorkerFromEnv(): Promise { + const runtime = await createComputeWorkerApp(); + await runtime.start({ registerSignalHandlers: true }); +} diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts index 8c62d03..1bf176c 100644 --- a/compute/worker/src/server.ts +++ b/compute/worker/src/server.ts @@ -1,1338 +1,6 @@ -import Fastify, { type FastifyRequest } from 'fastify'; -import { z } from 'zod'; -import { - connect, - nanos, - credsAuthenticator, - type NatsConnection, -} from '@nats-io/transport-node'; -import { - AckPolicy, - DeliverPolicy, - ReplayPolicy, - RetentionPolicy, - StorageType, - jetstream, - jetstreamManager, - type Consumer, - type JetStreamClient, - type JetStreamManager, - type JsMsg, -} from '@nats-io/jetstream'; -import { Kvm } from '@nats-io/kv'; -import { - ensureComputeModels, - runPdfLayoutFromPdfBuffer, - runWhisperAlignmentFromAudioBuffer, -} from '@openreader/compute-core/local-runtime'; -import { - getComputeTimeoutConfig, - getComputeOpStaleMs, - getAvailableCpuCores, - getOnnxThreadsPerJob, - withIdleTimeoutAndHardCap, - withTimeout, -} from '@openreader/compute-core'; -import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerJobState, - WorkerJobTiming, - WorkerOperationKind, - WorkerOperationRequest, - WorkerOperationState, - PdfLayoutProgress, -} from '@openreader/compute-core/api-contracts'; -import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { - JetStreamOperationEventStream, - OP_EVENTS_SUBJECT_WILDCARD, - JetStreamOperationQueue, - JetStreamOperationStateStore, - hashOpKey, -} from './control-plane/jetstream'; -import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; -import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; +import { startComputeWorkerFromEnv } from './runtime'; -const JOBS_STREAM_NAME = 'compute_jobs'; -const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; -const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; -const WHISPER_CONSUMER_NAME = 'compute_whisper'; -const LAYOUT_CONSUMER_NAME = 'compute_layout'; -const EVENTS_STREAM_NAME = 'compute_events'; -const COMPUTE_STATE_BUCKET = 'compute_state'; -const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; -const LOOP_ERROR_BACKOFF_MS = 500; -const RUNNING_HEARTBEAT_MS = 5000; -const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; -const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; -const WHISPER_MAX_DELIVER = 1; -const NATS_API_TIMEOUT_MS = 60_000; -// Disconnect from NATS after this much continuous idle so the worker stops -// generating outbound traffic (pull polling + keepalive PINGs) and Railway can -// put it to sleep. Reconnect happens lazily on the next inbound request. -const IDLE_DISCONNECT_MS = 120_000; -const IDLE_CHECK_INTERVAL_MS = 5_000; -// Bounded pull window so consumer loops yield periodically and can be stopped -// cleanly when going idle, instead of blocking on a long-lived pull. -const PULL_EXPIRES_MS = 5_000; -const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); -const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); -const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { - whisper_align: 15_000, - pdf_layout: 120_000, -}; - -interface QueuedJob { - jobId: string; - opId: string; - opKey: string; - kind: WorkerOperationKind; - queuedAt: number; - payload: TPayload; -} - -interface NatsSession { - nc: NatsConnection; - js: JetStreamClient; - jsm: JetStreamManager; - kv: Awaited>; - whisperConsumer: Consumer; - layoutConsumer: Consumer; -} - -type StreamedOperationState = WorkerOperationState; - -function requireEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} - -function readIntEnv(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -function normalizeNatsReplicas(value: number): number { - if (value === 3 || value === 5) return value; - return 1; -} - -function parseBoolEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const normalized = raw.toLowerCase(); - return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; -} - -function buildLoggerConfig(): boolean | Record { - const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); - const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; - if (format === 'json') { - return { - level, - base: null, - }; - } - return { - level, - base: null, - transport: { - target: 'pino-pretty', - options: { - colorize: true, - translateTime: 'SYS:standard', - ignore: 'pid,hostname', - }, - }, - }; -} - -function normalizeS3Prefix(prefix: string | undefined): string { - const value = (prefix || 'openreader').trim(); - return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; -} - -function sanitizeNamespace(namespace: string | null): string | null { - if (!namespace) return null; - return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; -} - -function documentParsedKey(id: string, namespace: string | null, prefix: string): string { - if (!DOCUMENT_ID_REGEX.test(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`; -} - -function buildS3Client(): S3Client { - const bucket = requireEnv('S3_BUCKET'); - const region = requireEnv('S3_REGION'); - const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); - const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); - const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; - const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); - - void bucket; - - return new S3Client({ - region, - endpoint, - forcePathStyle, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - credentials: { - accessKeyId, - secretAccessKey, - }, - }); -} - -async function bodyToBuffer(body: unknown): Promise { - if (!body) return Buffer.alloc(0); - if (body instanceof Uint8Array) return Buffer.from(body); - if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); - if (body instanceof ArrayBuffer) return Buffer.from(body); - if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { - const maybe = body as { transformToByteArray?: () => Promise }; - if (typeof maybe.transformToByteArray === 'function') { - return Buffer.from(await maybe.transformToByteArray()); - } - } - if (typeof body === 'object' && body !== null && 'on' in body) { - const stream = body as NodeJS.ReadableStream; - const chunks: Buffer[] = []; - for await (const chunk of stream) { - if (Buffer.isBuffer(chunk)) chunks.push(chunk); - else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); - else chunks.push(Buffer.from(chunk as Uint8Array)); - } - return Buffer.concat(chunks); - } - throw new Error('Unsupported S3 response body type'); -} - -function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { - const copy = new Uint8Array(bytes.byteLength); - copy.set(bytes); - return copy.buffer; -} - -function isAuthed(request: FastifyRequest, expectedToken: string): boolean { - const auth = request.headers.authorization; - if (!auth?.startsWith('Bearer ')) return false; - const token = auth.slice('Bearer '.length).trim(); - return token === expectedToken; -} - -function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { - if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; - return Math.max(0, Math.floor((end as number) - (start as number))); -} - -function toErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - return String(error); -} - -function requestPath(request: FastifyRequest): string { - return request.url.split('?')[0] ?? request.url; -} - -function isHealthPath(path: string): boolean { - return path === '/health/live' || path === '/health/ready'; -} - -function extractTraceId(request: FastifyRequest): string | null { - const header = request.headers['x-openreader-trace-id']; - if (Array.isArray(header)) return header[0] ?? null; - return typeof header === 'string' ? header : null; -} - -function extractOpId(request: FastifyRequest, path: string): string | null { - const params = request.params as { opId?: unknown } | undefined; - if (params && typeof params.opId === 'string' && params.opId.trim()) { - return params.opId.trim(); - } - const match = path.match(/^\/ops\/([^/]+)/); - if (!match?.[1]) return null; - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; - } -} - -function isAlreadyExistsError(error: unknown): boolean { - const message = toErrorMessage(error).toLowerCase(); - return message.includes('already in use') || message.includes('already exists'); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -class ConcurrencyGate { - private readonly maxInFlight: number; - private inFlight = 0; - private readonly queue: Array<() => void> = []; - - constructor(limit: number) { - this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; - } - - async acquire(): Promise { - if (this.inFlight < this.maxInFlight) { - this.inFlight += 1; - return; - } - - await new Promise((resolve) => { - this.queue.push(() => { - this.inFlight += 1; - resolve(); - }); - }); - } - - release(): void { - this.inFlight = Math.max(0, this.inFlight - 1); - const next = this.queue.shift(); - if (next) next(); - } -} - -function isTerminalStatus(status: WorkerJobState): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { - if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; - const maybe = result as { parsedObjectKey?: unknown }; - return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; -} - -const alignSchema = z.object({ - text: z.string().trim().min(1), - lang: z.string().trim().min(1).max(16).optional(), - cacheKey: z.string().trim().min(1).max(256).optional(), - audioObjectKey: z.string().trim().min(1).max(2048), -}); - -const layoutSchema = z.object({ - documentId: z.string().trim().min(1), - namespace: z.string().trim().min(1).max(128).nullable(), - documentObjectKey: z.string().trim().min(1).max(2048), -}); - -const operationCreateSchema = z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('whisper_align'), - opKey: z.string().trim().min(1).max(1024), - payload: alignSchema, - }), - z.object({ - kind: z.literal('pdf_layout'), - opKey: z.string().trim().min(1).max(1024), - payload: layoutSchema, - }), -]); - -async function ensureJetStreamResources( - jsm: JetStreamManager, - whisperTimeoutMs: number, - pdfTimeoutMs: number, - pdfAttempts: number, - jobsMaxBytes: number, - eventsMaxBytes: number, - natsReplicas: number, -): Promise { - const streamConfig = { - name: JOBS_STREAM_NAME, - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - retention: RetentionPolicy.Workqueue, - storage: StorageType.File, - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(streamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(JOBS_STREAM_NAME, { - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }); - } - - const eventsStreamConfig = { - name: EVENTS_STREAM_NAME, - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - retention: RetentionPolicy.Limits, - storage: StorageType.File, - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(eventsStreamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(EVENTS_STREAM_NAME, { - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }); - } - - const ensureConsumer = async ( - name: string, - subject: string, - ackWaitMs: number, - maxDeliver: number, - ): Promise => { - const config = { - durable_name: name, - ack_policy: AckPolicy.Explicit, - deliver_policy: DeliverPolicy.All, - replay_policy: ReplayPolicy.Instant, - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }; - - try { - await jsm.consumers.add(JOBS_STREAM_NAME, config); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.consumers.update(JOBS_STREAM_NAME, name, { - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }); - } - }; - - await Promise.all([ - ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), - ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), - ]); -} - -async function main(): Promise { - const port = readIntEnv('PORT', 8081); - const host = process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'; - const workerToken = requireEnv('COMPUTE_WORKER_TOKEN'); - const natsUrl = requireEnv('NATS_URL'); - const timeoutConfig = getComputeTimeoutConfig(); - - const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); - const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; - const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; - const pdfHardCapMs = timeoutConfig.pdfHardCapMs; - const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); - const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); - const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); - const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); - const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); - const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); - const opStaleMs = getComputeOpStaleMs(); - - const connectOpts: Parameters[0] = { servers: natsUrl }; - const natsCreds = process.env.NATS_CREDS?.trim(); - const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); - - if (natsCreds) { - console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); - connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); - } else if (natsCredsFile) { - console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); - const { readFileSync } = await import('node:fs'); - const credsData = readFileSync(natsCredsFile); - connectOpts.authenticator = credsAuthenticator(credsData); - } - - // Lazy NATS connection lifecycle. The worker connects on demand (first request - // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle - // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound /ops request both wakes the container and re-establishes - // the session via ensureConnected(). - let session: NatsSession | null = null; - let connecting: Promise | null = null; - let workerLoops: Promise[] = []; - let idleTimer: NodeJS.Timeout | null = null; - let stopping = false; - let loopStopRequested = false; - - // Activity accounting feeding the idle detector. The worker is considered idle - // only when no HTTP request is in flight, no SSE stream is open, no job is - // processing, and nothing has happened for IDLE_DISCONNECT_MS. - let inFlightHttp = 0; - let activeSse = 0; - let inFlightJobs = 0; - let lastActivityAt = Date.now(); - const jobGate = new ConcurrencyGate(jobConcurrency); - - const markActivity = (): void => { - lastActivityAt = Date.now(); - }; - - function startIdleTimer(): void { - if (idleTimer) return; - idleTimer = setInterval(() => { - if (!session || stopping) return; - if (inFlightHttp > 0 || activeSse > 0 || inFlightJobs > 0) return; - if (Date.now() - lastActivityAt < IDLE_DISCONNECT_MS) return; - void disconnect('idle'); - }, IDLE_CHECK_INTERVAL_MS); - // Don't let the idle checker keep the process alive on its own. - idleTimer.unref?.(); - } - - async function disconnect(reason: string): Promise { - const current = session; - if (!current) return; - // Clear synchronously (before any await) so concurrent requests reconnect a - // fresh session instead of using the connection we're about to close. - session = null; - loopStopRequested = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - try { - await current.nc.close(); - } catch { - // ignore close errors - } - await Promise.allSettled(workerLoops); - workerLoops = []; - loopStopRequested = false; - app.log.info({ reason }, 'nats disconnected'); - } - - async function ensureConnected(): Promise { - if (session) return session; - if (connecting) return connecting; - connecting = (async () => { - const nc: NatsConnection = await connect(connectOpts); - const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); - const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); - await ensureJetStreamResources( - jsm, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - jobsStreamMaxBytes, - eventsStreamMaxBytes, - natsReplicas, - ); - const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { - replicas: natsReplicas, - history: 1, - ttl: COMPUTE_STATE_TTL_MS, - max_bytes: jobStatesMaxBytes, - }); - const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); - const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); - const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; - session = next; - markActivity(); - startWorkerLoops(next); - startIdleTimer(); - // Safety net: if the connection closes for any reason (network drop after - // exhausting reconnects, or our own disconnect), drop the stale session so - // the next request reconnects cleanly. - void nc.closed().then(() => { - if (session?.nc === nc) session = null; - }); - app.log.info('nats connected'); - return next; - })(); - try { - return await connecting; - } finally { - connecting = null; - } - } - - const s3 = buildS3Client(); - const s3Bucket = requireEnv('S3_BUCKET'); - const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); - - const ensureSafeKey = (key: string): string => { - const trimmed = key.trim(); - if (!trimmed.startsWith(`${s3Prefix}/`)) { - throw new Error('Object key prefix mismatch'); - } - return trimmed; - }; - - const readObjectByKey = async (key: string): Promise => { - const safeKey = ensureSafeKey(key); - const response = await s3.send(new GetObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - const bytes = await bodyToBuffer(response.Body); - return toArrayBuffer(new Uint8Array(bytes)); - }; - - const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { - const key = documentParsedKey(documentId, namespace, s3Prefix); - const body = Buffer.from(JSON.stringify(parsed)); - await s3.send(new PutObjectCommand({ - Bucket: s3Bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - })); - return key; - }; - - if (prewarmModels) { - await ensureComputeModels(); - } - - const app = Fastify({ - logger: buildLoggerConfig(), - disableRequestLogging: true, - }); - app.log.info({ - jobConcurrency, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - opStaleMs, - availableCpuCores: getAvailableCpuCores(), - onnxThreadsPerJob: getOnnxThreadsPerJob(), - natsApiTimeoutMs: NATS_API_TIMEOUT_MS, - natsReplicas, - eventsStreamMaxBytes, - pdfLayoutHardCapMs: pdfHardCapMs, - }, 'compute runtime config'); - - const whisperJobCodec = createJsonCodec>(); - const layoutJobCodec = createJsonCodec>(); - - const operationStateStore = new JetStreamOperationStateStore({ - getKv: async () => (await ensureConnected()).kv, - }); - - const operationEventStream = new JetStreamOperationEventStream({ - getJs: async () => (await ensureConnected()).js, - getJsm: async () => (await ensureConnected()).jsm, - eventsStreamName: EVENTS_STREAM_NAME, - }); - - const operationQueue = new JetStreamOperationQueue({ - getJs: async () => (await ensureConnected()).js, - whisperSubject: WHISPER_JOBS_SUBJECT, - layoutSubject: LAYOUT_JOBS_SUBJECT, - }); - - const orchestrator = new OperationOrchestrator({ - queue: operationQueue, - stateStore: operationStateStore, - eventStream: operationEventStream, - config: { - opStaleMs, - maxCasRetries: 10, - }, - }); - - const getOpState = async (opId: string): Promise => { - return await operationStateStore.getOpState(opId); - }; - - const releaseHttp = (request: FastifyRequest): void => { - const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; - if (!counted[REQUEST_COUNTED_KEY]) return; - counted[REQUEST_COUNTED_KEY] = false; - inFlightHttp = Math.max(0, inFlightHttp - 1); - markActivity(); - }; - - app.addHook('onRequest', async (request, reply) => { - const path = requestPath(request); - (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); - // Count every request as in-flight activity so the idle detector never - // disconnects mid-request. Released in onResponse, or manually after hijack - // for SSE streams (where onResponse does not fire). - (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; - inFlightHttp += 1; - markActivity(); - if (isHealthPath(path)) return; - if (!isAuthed(request, workerToken)) { - return reply.code(401).send({ error: 'Unauthorized' }); - } - return; - }); - - app.addHook('onResponse', async (request, reply) => { - releaseHttp(request); - const path = requestPath(request); - if (isHealthPath(path)) return; - if (reply.statusCode >= 500) { - const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; - const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; - app.log.error({ - reqId: request.id, - method: request.method, - path, - statusCode: reply.statusCode, - durationMs, - traceId: extractTraceId(request) ?? null, - opId: extractOpId(request, path), - }, 'http.error'); - } - }); - - app.get('/health/live', async () => ({ ok: true })); - - // Reports readiness without forcing a NATS round-trip. Probing NATS here would - // reconnect (and keep) the connection open, defeating idle sleep, so we only - // report the current connection state. The worker reconnects lazily on the next - // /ops request regardless of what this returns. - app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); - - app.post('/ops', async (request, reply) => { - const parsed = operationCreateSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - const requestOp = parsed.data as WorkerOperationRequest; - const op = await orchestrator.enqueueOrReuse(requestOp); - app.log.info({ - kind: requestOp.kind, - opId: op.opId, - jobId: op.jobId, - status: op.status, - opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), - }, 'op.accepted'); - reply.code(202); - return op; - }); - - app.get('/ops/:opId', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const state = await getOpState(params.data.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; - }); - - app.get('/ops/:opId/events', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const initial = await getOpState(params.data.opId); - if (!initial) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; - const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); - const lastEventIdHeader = request.headers['last-event-id']; - const cursorFromHeader = Number( - Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), - ); - const sinceEventId = Math.max( - 0, - Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, - Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, - ); - - reply.hijack(); - // onResponse will not fire for a hijacked reply, so release the HTTP in-flight - // count here and track the long-lived stream via activeSse instead. - releaseHttp(request); - activeSse += 1; - markActivity(); - reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); - reply.raw.setHeader('Connection', 'keep-alive'); - reply.raw.setHeader('X-Accel-Buffering', 'no'); - - let closed = false; - let unsubscribe: (() => void) | null = null; - - const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { - if (closed || reply.raw.writableEnded) return; - const frameEvent: WorkerOperationEvent = { - eventId, - snapshot, - }; - reply.raw.write(encodeSseFrame({ - id: eventId, - event: 'snapshot', - data: frameEvent, - })); - }; - - const closeStream = (): void => { - if (closed) return; - closed = true; - if (unsubscribe) { - unsubscribe(); - unsubscribe = null; - } - activeSse = Math.max(0, activeSse - 1); - markActivity(); - if (!reply.raw.writableEnded) { - reply.raw.end(); - } - }; - - request.raw.on('close', () => { - closeStream(); - }); - - try { - let current = initial; - let signature = JSON.stringify(current); - writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); - if (isTerminalStatus(current.status)) { - return reply; - } - - unsubscribe = await operationEventStream.subscribe({ - opId: params.data.opId, - sinceEventId, - onEvent: (event) => { - if (closed) return; - if (event.snapshot.opId !== params.data.opId) return; - const nextSignature = JSON.stringify(event.snapshot); - if (nextSignature !== signature) { - current = event.snapshot; - signature = nextSignature; - writeSnapshot(current, event.eventId); - } - if (isTerminalStatus(event.snapshot.status)) { - closeStream(); - } - }, - onError: (error) => { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - closeStream(); - }, - }); - - await new Promise((resolve) => { - request.raw.once('close', () => resolve()); - }); - } catch (error) { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - } finally { - closeStream(); - } - - return reply; - }); - - const runWhisper = async ( - payload: WhisperAlignJobRequest, - queueWaitMs: number, - ): Promise => { - const parsed = alignSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const audioBuffer = await withTimeout( - readObjectByKey(parsed.audioObjectKey), - whisperTimeoutMs, - 'whisper s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - const computeStartedAt = Date.now(); - const result = await withTimeout( - runWhisperAlignmentFromAudioBuffer({ - audioBuffer, - text: parsed.text, - cacheKey: parsed.cacheKey, - lang: parsed.lang, - }), - whisperTimeoutMs, - 'whisper alignment job', - ); - - const computeMs = Date.now() - computeStartedAt; - return { - ...result, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - const runLayout = async ( - payload: PdfLayoutJobRequest, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ): Promise => { - const parsed = layoutSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const pdfBytes = await withTimeout( - readObjectByKey(parsed.documentObjectKey), - Math.max(pdfTimeoutMs, 1_000), - 'pdf s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - let lastTotalPages = 0; - let lastPagesParsed = 0; - const computeStartedAt = Date.now(); - const result = await withIdleTimeoutAndHardCap({ - idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), - hardCapMs: pdfHardCapMs, - label: 'pdf layout job', - run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ - documentId: parsed.documentId, - pdfBytes, - onPageStarted: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageStart({ - pageNumber, - totalPages, - })); - }, - onPageParsed: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - lastPagesParsed = pageNumber; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageParsed({ - pageNumber, - totalPages, - })); - }, - }), - }); - - const computeMs = Date.now() - computeStartedAt; - if (hooks?.onProgress && lastTotalPages > 0) { - await hooks.onProgress({ - totalPages: lastTotalPages, - pagesParsed: lastPagesParsed, - currentPage: lastPagesParsed || undefined, - phase: 'merge', - }); - } - const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); - return { - parsedObjectKey, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - type ProcessMessageInput = { - msg: JsMsg; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }; - - type ProcessMessageContext = { - decoded: QueuedJob; - workerLabel: string; - startedAt: number; - queueWaitTiming?: { queueWaitMs: number }; - latestProgress?: PdfLayoutProgress; - }; - - function buildQueueWaitTiming(queuedAt: number, now: number): { queueWaitMs: number } | undefined { - const queueWaitMs = safeDurationMs(queuedAt, now); - return typeof queueWaitMs === 'number' ? { queueWaitMs } : undefined; - } - - function extractTiming(result: unknown): WorkerJobTiming | undefined { - if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; - return (result as { timing?: WorkerJobTiming }).timing; - } - - async function markRunning( - context: ProcessMessageContext, - updatedAt: number, - options?: { includeStartedAt?: boolean }, - ): Promise { - if (context.latestProgress) { - await orchestrator.markProgress({ - opId: context.decoded.opId, - progress: context.latestProgress, - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - return; - } - - await orchestrator.markRunning({ - opId: context.decoded.opId, - ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - } - - async function markProgress( - context: ProcessMessageContext, - progress: PdfLayoutProgress, - updatedAt: number, - ): Promise { - context.latestProgress = progress; - await markRunning(context, updatedAt); - } - - async function markTerminal(input: { - context: ProcessMessageContext; - status: 'succeeded' | 'failed'; - result?: TResult; - errorMessage?: string; - timing?: WorkerJobTiming; - updatedAt: number; - }): Promise { - if (input.status === 'succeeded') { - await orchestrator.markSucceeded({ - opId: input.context.decoded.opId, - result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - return; - } - - await orchestrator.markFailed({ - opId: input.context.decoded.opId, - error: { message: input.errorMessage ?? 'unknown worker failure' }, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - } - - async function handleRetry(input: { - context: ProcessMessageContext | null; - msg: JsMsg; - errorMessage: string; - }): Promise { - const deliveryCount = input.msg.info.deliveryCount; - const isWhisperAlign = input.context?.decoded.kind === 'whisper_align'; - const maxAttempts = isWhisperAlign ? WHISPER_MAX_DELIVER : pdfAttempts; - const hasRetriesLeft = !isWhisperAlign && deliveryCount < maxAttempts; - - if (input.context) { - const now = Date.now(); - const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); - const persistOpUpdate = hasRetriesLeft - ? (input.context.latestProgress - ? orchestrator.markProgress({ - opId: input.context.decoded.opId, - progress: input.context.latestProgress, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }) - : orchestrator.markRunning({ - opId: input.context.decoded.opId, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - })) - : markTerminal({ - context: input.context, - status: 'failed', - errorMessage: input.errorMessage, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }); - - await persistOpUpdate.catch((stateError) => { - app.log.error({ - worker: input.context?.workerLabel, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation state'); - }); - } - - if (hasRetriesLeft) { - input.msg.nak(); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'running', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retryAction: 'nack_retry', - }, 'job.terminal'); - return; - } - - input.msg.term(input.errorMessage); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'failed', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retrySuppressed: isWhisperAlign ? 'whisper_align' : undefined, - retryAction: 'term', - }, 'job.terminal'); - } - - async function processMessage(input: ProcessMessageInput): Promise { - let context: ProcessMessageContext | null = null; - let heartbeat: NodeJS.Timeout | null = null; - try { - const decoded = input.codec.decode(input.msg.data); - const startedAt = Date.now(); - context = { - decoded, - workerLabel: input.workerLabel, - startedAt, - queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), - }; - - await markRunning(context, startedAt); - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, - deliveryCount: input.msg.info.deliveryCount, - }, 'job.started'); - - heartbeat = setInterval(() => { - const now = Date.now(); - void markRunning(context!, now).catch((stateError) => { - app.log.error({ - worker: input.workerLabel, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation heartbeat state'); - }); - }, RUNNING_HEARTBEAT_MS); - - const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { - onProgress: async (progress) => { - await markProgress(context!, progress, Date.now()); - }, - }); - const resultTiming = extractTiming(result); - const now = Date.now(); - - await markTerminal({ - context, - status: 'succeeded', - result, - timing: resultTiming, - updatedAt: now, - }); - - input.msg.ack(); - const terminalDurationMs = safeDurationMs(context.startedAt, now); - const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; - if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - durationMs: terminalDurationMs ?? null, - timing: resultTiming ?? null, - }, 'job.stage'); - } - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - status: 'succeeded', - durationMs: terminalDurationMs ?? null, - resultRef: extractResultRef(decoded.kind, result), - timing: resultTiming ?? null, - }, 'job.terminal'); - } catch (error) { - await handleRetry({ - context, - msg: input.msg, - errorMessage: toErrorMessage(error), - }); - } finally { - if (heartbeat) clearInterval(heartbeat); - } - } - - async function createWorkerLoop(input: { - owner: NatsSession; - consumer: Consumer; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }): Promise { - // Exit when the loop's connection is no longer the active session (idle - // disconnect, unexpected close, or replaced by a reconnect). - const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; - while (!detached()) { - let msg: JsMsg | null = null; - try { - try { - // Bounded pull so the loop yields periodically and exits promptly when - // the session is torn down for idle (nc.close() rejects the pending pull). - msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); - } catch (error) { - if (detached()) return; - app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); - await sleep(LOOP_ERROR_BACKOFF_MS); - continue; - } - - // An empty pull is not activity; let the idle window advance. - if (!msg) continue; - markActivity(); - inFlightJobs += 1; - await jobGate.acquire(); - if (detached()) { - return; - } - await processMessage({ - msg, - codec: input.codec, - run: input.run, - workerLabel: input.workerLabel, - }); - } finally { - if (msg) { - jobGate.release(); - inFlightJobs = Math.max(0, inFlightJobs - 1); - markActivity(); - } - } - } - } - - function startWorkerLoops(active: NatsSession): void { - // Always starts a fresh set bound to the new session. Any loops from a prior - // session self-terminate once they observe session !== their owner. - workerLoops = []; - loopStopRequested = false; - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.whisperConsumer, - codec: whisperJobCodec, - run: runWhisper, - workerLabel: `whisper-${i + 1}`, - })); - } - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.layoutConsumer, - codec: layoutJobCodec, - run: runLayout, - workerLabel: `layout-${i + 1}`, - })); - } - } - - const close = async (): Promise => { - if (stopping) return; - stopping = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - await app.close(); - await Promise.allSettled(workerLoops); - const current = session; - session = null; - if (current) { - try { - await current.nc.drain(); - } catch { - try { - await current.nc.close(); - } catch { - // ignore close errors - } - } - } - }; - - process.once('SIGINT', () => { - void close().finally(() => process.exit(0)); - }); - - process.once('SIGTERM', () => { - void close().finally(() => process.exit(0)); - }); - - await app.listen({ host, port }); - app.log.info({ host, port }, 'compute worker listening'); -} - -void main().catch((error) => { +void startComputeWorkerFromEnv().catch((error) => { console.error('[compute-worker] fatal startup error', error); process.exit(1); }); diff --git a/compute/worker/src/worker-loop-policy.ts b/compute/worker/src/worker-loop-policy.ts new file mode 100644 index 0000000..7462b4c --- /dev/null +++ b/compute/worker/src/worker-loop-policy.ts @@ -0,0 +1,22 @@ +import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts'; + +export type RetryAction = 'nak_retry' | 'term_fail'; + +export function buildQueueWaitTiming(queuedAt: number, now: number): { queueWaitMs: number } | undefined { + if (!Number.isFinite(queuedAt) || !Number.isFinite(now)) return undefined; + return { queueWaitMs: Math.max(0, Math.floor(now - queuedAt)) }; +} + +export function decideRetryAction(input: { + kind: WorkerOperationKind; + deliveryCount: number; + pdfAttempts: number; + whisperMaxDeliver?: number; +}): RetryAction { + const whisperMaxDeliver = input.whisperMaxDeliver ?? 1; + if (input.kind === 'whisper_align') { + return input.deliveryCount < whisperMaxDeliver ? 'nak_retry' : 'term_fail'; + } + + return input.deliveryCount < input.pdfAttempts ? 'nak_retry' : 'term_fail'; +} diff --git a/compute/worker/tests/api/routes.test.ts b/compute/worker/tests/api/routes.test.ts new file mode 100644 index 0000000..77c4d05 --- /dev/null +++ b/compute/worker/tests/api/routes.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { createComputeWorkerApp } from '../../src/runtime'; +import { FakeControlPlane } from '../fixtures/fake-control-plane'; + +const AUTH = { authorization: 'Bearer test-token' }; + +describe('compute worker API routes', () => { + let fake: FakeControlPlane; + let runtime: Awaited>; + + beforeEach(async () => { + fake = new FakeControlPlane(); + runtime = await createComputeWorkerApp({ + workerToken: 'test-token', + disableWorkers: true, + routeDeps: fake.deps, + }); + }); + + afterEach(async () => { + await runtime.close(); + }); + + test('allows unauthenticated health checks but protects operation routes', async () => { + const live = await runtime.app.inject({ method: 'GET', url: '/health/live' }); + expect(live.statusCode).toBe(200); + + const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' }); + expect(protectedRoute.statusCode).toBe(401); + }); + + test('validates operation creation body and returns 400 for invalid payload', async () => { + const response = await runtime.app.inject({ + method: 'POST', + url: '/ops', + headers: AUTH, + payload: { + kind: 'pdf_layout', + opKey: '', + payload: { + documentId: 'd1', + namespace: null, + documentObjectKey: 's3://bucket/doc.pdf', + }, + }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json()).toMatchObject({ error: 'Invalid request body' }); + }); + + test('creates operation and fetches state by op id', async () => { + const create = await runtime.app.inject({ + method: 'POST', + url: '/ops', + headers: AUTH, + payload: { + kind: 'pdf_layout', + opKey: 'doc-1:layout', + payload: { + documentId: 'doc-1', + namespace: null, + documentObjectKey: 'openreader/doc-1.pdf', + }, + }, + }); + + expect(create.statusCode).toBe(202); + const created = create.json(); + expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' }); + + const fetch = await runtime.app.inject({ + method: 'GET', + url: `/ops/${created.opId}`, + headers: AUTH, + }); + + expect(fetch.statusCode).toBe(200); + expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' }); + }); + + test('returns not found for unknown operation and event stream lookups', async () => { + const opResponse = await runtime.app.inject({ + method: 'GET', + url: '/ops/missing', + headers: AUTH, + }); + expect(opResponse.statusCode).toBe(404); + + const eventsResponse = await runtime.app.inject({ + method: 'GET', + url: '/ops/missing/events', + headers: AUTH, + }); + expect(eventsResponse.statusCode).toBe(404); + }); + + test('streams initial SSE snapshot for terminal operation and honors cursor id', async () => { + fake.seedState({ + opId: 'op-terminal', + opKey: 'k-terminal', + kind: 'pdf_layout', + jobId: 'job-op-terminal', + status: 'succeeded', + queuedAt: 1000, + updatedAt: 2000, + result: { parsedObjectKey: 'openreader/parsed.json' }, + }); + + const stream = await runtime.app.inject({ + method: 'GET', + url: '/ops/op-terminal/events?sinceEventId=7', + headers: AUTH, + }); + + expect(stream.statusCode).toBe(200); + expect(stream.headers['content-type']).toContain('text/event-stream'); + expect(stream.body).toContain('event: snapshot'); + expect(stream.body).toContain('id: 7'); + expect(stream.body).toContain('"status":"succeeded"'); + }); +}); diff --git a/compute/worker/tests/fixtures/fake-control-plane.ts b/compute/worker/tests/fixtures/fake-control-plane.ts new file mode 100644 index 0000000..2d18f2b --- /dev/null +++ b/compute/worker/tests/fixtures/fake-control-plane.ts @@ -0,0 +1,119 @@ +import type { + PdfLayoutJobResult, + WhisperAlignJobResult, + WorkerOperationEvent, + WorkerOperationRequest, + WorkerOperationState, +} from '@openreader/compute-core/api-contracts'; +import type { ComputeWorkerRouteDeps } from '../../src/runtime'; + +type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; +type ComputeState = WorkerOperationState; +type ComputeEvent = WorkerOperationEvent; + +export class FakeControlPlane { + private readonly stateByOpId = new Map(); + private readonly opIdByOpKey = new Map(); + private readonly eventsByOpId = new Map(); + private nextOpId = 1; + + readonly deps: ComputeWorkerRouteDeps = { + orchestrator: { + enqueueOrReuse: async (request) => this.enqueueOrReuse(request), + markRunning: async (input) => this.updateState(input.opId, { + status: 'running', + startedAt: input.startedAt, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markProgress: async (input) => this.updateState(input.opId, { + status: 'running', + progress: input.progress, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markSucceeded: async (input) => this.updateState(input.opId, { + status: 'succeeded', + result: input.result as ComputeResult, + updatedAt: input.updatedAt, + timing: input.timing, + }), + markFailed: async (input) => this.updateState(input.opId, { + status: 'failed', + error: typeof input.error === 'string' ? { message: input.error } : input.error, + updatedAt: input.updatedAt, + timing: input.timing, + }), + }, + operationStateStore: { + getOpState: async (opId) => this.stateByOpId.get(opId) ?? null, + }, + operationEventStream: { + subscribe: async ({ opId, sinceEventId, onEvent }) => { + const since = Math.max(0, Math.floor(sinceEventId ?? 0)); + const replay = (this.eventsByOpId.get(opId) ?? []).filter((event) => event.eventId > since); + for (const event of replay) { + await onEvent(event); + } + return () => undefined; + }, + }, + }; + + seedState(state: ComputeState): void { + this.stateByOpId.set(state.opId, state); + this.opIdByOpKey.set(state.opKey, state.opId); + } + + seedEvent(opId: string, event: ComputeEvent): void { + const list = this.eventsByOpId.get(opId) ?? []; + list.push(event); + this.eventsByOpId.set(opId, list); + } + + private async enqueueOrReuse(request: WorkerOperationRequest): Promise { + const existingId = this.opIdByOpKey.get(request.opKey); + if (existingId) { + const existing = this.stateByOpId.get(existingId); + if (existing) return existing; + } + + const now = Date.now(); + const opId = `op-${this.nextOpId++}`; + const state: ComputeState = { + opId, + opKey: request.opKey, + kind: request.kind, + jobId: `job-${opId}`, + status: 'queued', + queuedAt: now, + updatedAt: now, + }; + + this.stateByOpId.set(opId, state); + this.opIdByOpKey.set(request.opKey, opId); + this.seedEvent(opId, { eventId: 1, snapshot: state }); + return state; + } + + private async updateState( + opId: string, + patch: Partial, + ): Promise { + const current = this.stateByOpId.get(opId); + if (!current) { + throw new Error(`Unknown opId: ${opId}`); + } + const next: ComputeState = { + ...current, + ...patch, + updatedAt: patch.updatedAt ?? Date.now(), + }; + this.stateByOpId.set(opId, next); + const currentEvents = this.eventsByOpId.get(opId) ?? []; + const nextEventId = (currentEvents.at(-1)?.eventId ?? 0) + 1; + currentEvents.push({ eventId: nextEventId, snapshot: next }); + this.eventsByOpId.set(opId, currentEvents); + return next; + } +} diff --git a/compute/worker/tests/setup-env.ts b/compute/worker/tests/setup-env.ts new file mode 100644 index 0000000..d013982 --- /dev/null +++ b/compute/worker/tests/setup-env.ts @@ -0,0 +1,8 @@ +process.env.COMPUTE_WORKER_TOKEN = process.env.COMPUTE_WORKER_TOKEN || 'test-token'; +process.env.NATS_URL = process.env.NATS_URL || 'nats://127.0.0.1:4222'; +process.env.COMPUTE_PREWARM_MODELS = 'false'; +process.env.S3_BUCKET = process.env.S3_BUCKET || 'test-bucket'; +process.env.S3_REGION = process.env.S3_REGION || 'us-east-1'; +process.env.S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID || 'test'; +process.env.S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY || 'test'; +process.env.S3_PREFIX = process.env.S3_PREFIX || 'openreader'; diff --git a/compute/worker/tests/tsconfig.json b/compute/worker/tests/tsconfig.json new file mode 100644 index 0000000..f969c4c --- /dev/null +++ b/compute/worker/tests/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["./**/*.ts"], + "exclude": [] +} diff --git a/tests/unit/compute-worker-control-plane-jetstream.spec.ts b/compute/worker/tests/unit/jetstream-adapters.test.ts similarity index 55% rename from tests/unit/compute-worker-control-plane-jetstream.spec.ts rename to compute/worker/tests/unit/jetstream-adapters.test.ts index fb3f3e3..9f30ccc 100644 --- a/tests/unit/compute-worker-control-plane-jetstream.spec.ts +++ b/compute/worker/tests/unit/jetstream-adapters.test.ts @@ -1,16 +1,17 @@ -import { expect, test } from '@playwright/test'; -import { OperationOrchestrator } from '../../compute/core/src/control-plane'; -import type { WorkerOperationRequest } from '../../compute/core/src/api-contracts'; +import { describe, expect, test } from 'vitest'; +import { OperationOrchestrator } from '@openreader/compute-core/control-plane'; +import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, JetStreamOperationStateStore, + hashOpKey, opEventsSubject, opIndexKvKey, opStateKvKey, type KvEntryLike, type KvStoreLike, -} from '../../compute/worker/src/control-plane/jetstream'; +} from '../../src/control-plane/jetstream'; class FakeKvStore implements KvStoreLike { private readonly data = new Map(); @@ -18,46 +19,31 @@ class FakeKvStore implements KvStoreLike { async get(key: string): Promise { const value = this.data.get(key); - if (!value) return null; - return { - operation: value.operation, - value: value.value.slice(), - revision: value.revision, - }; + return value + ? { + operation: value.operation, + value: value.value.slice(), + revision: value.revision, + } + : null; } async put(key: string, data: Uint8Array): Promise { this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } async create(key: string, data: Uint8Array): Promise { - if (this.data.has(key)) { - throw new Error('key exists'); - } + if (this.data.has(key)) throw new Error('key exists'); this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } async update(key: string, data: Uint8Array, version: number): Promise { const current = this.data.get(key); - if (!current || current.revision !== version) { - throw new Error('wrong last sequence'); - } + if (!current || current.revision !== version) throw new Error('wrong last sequence'); this.revision += 1; - this.data.set(key, { - operation: 'PUT', - value: data.slice(), - revision: this.revision, - }); + this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } } @@ -72,7 +58,7 @@ class FakeJetStream { async publish(subject: string, data: Uint8Array): Promise<{ seq: number; stream: string; duplicate: boolean }> { this.seq += 1; - const payload = JSON.parse(new TextDecoder().decode(data)) as unknown; + const payload = JSON.parse(new TextDecoder().decode(data)); this.published.push({ subject, payload, seq: this.seq }); return { seq: this.seq, stream: 'fake', duplicate: false }; } @@ -90,97 +76,81 @@ function buildPdfRequest(opKey: string): WorkerOperationRequest { }; } -test.describe('worker jetstream control-plane adapters', () => { - test('state store compareAndSet handles create/update semantics', async () => { +describe('jetstream adapters', () => { + test('state store compareAndSet enforces create/update semantics', async () => { const kv = new FakeKvStore(); const store = new JetStreamOperationStateStore({ getKv: async () => kv }); - const created = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-1', - expectedOpId: null, - }); - const failedCreate = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: null, - }); - const wrongExpected = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: 'op-x', - }); - const updated = await store.compareAndSetOpIndex({ - opKey: 'k1', - newOpId: 'op-2', - expectedOpId: 'op-1', - }); + const created = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-1', expectedOpId: null }); + const duplicateCreate = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: null }); + const wrongExpected = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: 'op-x' }); + const updated = await store.compareAndSetOpIndex({ opKey: 'k1', newOpId: 'op-2', expectedOpId: 'op-1' }); - expect(created).toBeTruthy(); - expect(failedCreate).toBeFalsy(); - expect(wrongExpected).toBeFalsy(); - expect(updated).toBeTruthy(); + expect(created).toBe(true); + expect(duplicateCreate).toBe(false); + expect(wrongExpected).toBe(false); + expect(updated).toBe(true); expect(await store.getOpIndex('k1')).toEqual({ opId: 'op-2' }); }); - test('queue and event adapters publish expected JetStream subjects', async () => { + test('queue routes layout jobs to the expected subject and publishes events', async () => { const js = new FakeJetStream(); const queue = new JetStreamOperationQueue({ - getJs: async () => js as any, + getJs: async () => js as never, whisperSubject: 'jobs.whisper', layoutSubject: 'jobs.layout', }); const events = new JetStreamOperationEventStream({ - getJs: async () => js as any, + getJs: async () => js as never, getJsm: async () => ({ consumers: { add: async () => ({ name: 'noop' }), delete: async () => true, }, - }) as any, + }) as never, eventsStreamName: 'compute_events', }); await queue.enqueue({ - jobId: 'j1', - opId: 'o1', - opKey: 'k1', + jobId: 'job-1', + opId: 'op-1', + opKey: 'k-1', kind: 'pdf_layout', queuedAt: 1000, payload: { documentId: 'd1', namespace: null, documentObjectKey: 'obj' }, }); - const appended = await events.append('o1', { - opId: 'o1', - opKey: 'k1', + const appended = await events.append('op-1', { + opId: 'op-1', + opKey: 'k-1', kind: 'pdf_layout', - jobId: 'j1', + jobId: 'job-1', status: 'queued', queuedAt: 1000, updatedAt: 1000, }); - expect(js.published.map((entry) => entry.subject)).toEqual(['jobs.layout', opEventsSubject('o1')]); + expect(js.published.map((entry) => entry.subject)).toEqual(['jobs.layout', opEventsSubject('op-1')]); expect(appended.eventId).toBe(2); }); - test('orchestrator integration writes index/state and reuses active op', async () => { + test('orchestrator writes expected index/state keys', async () => { const kv = new FakeKvStore(); const js = new FakeJetStream(); - const store = new JetStreamOperationStateStore({ getKv: async () => kv }); - const events = new JetStreamOperationEventStream({ - getJs: async () => js as any, + const stateStore = new JetStreamOperationStateStore({ getKv: async () => kv }); + const eventStream = new JetStreamOperationEventStream({ + getJs: async () => js as never, getJsm: async () => ({ consumers: { add: async () => ({ name: 'noop' }), delete: async () => true, }, - }) as any, + }) as never, eventsStreamName: 'compute_events', }); const queue = new JetStreamOperationQueue({ - getJs: async () => js as any, + getJs: async () => js as never, whisperSubject: 'jobs.whisper', layoutSubject: 'jobs.layout', }); @@ -189,8 +159,8 @@ test.describe('worker jetstream control-plane adapters', () => { let nextId = 1; const orchestrator = new OperationOrchestrator({ queue, - stateStore: store, - eventStream: events, + stateStore, + eventStream, config: { opStaleMs: 10_000, maxCasRetries: 3 }, clock: { now: () => now }, idFactory: { @@ -199,18 +169,18 @@ test.describe('worker jetstream control-plane adapters', () => { }, }); - const req = buildPdfRequest('k-integration'); - const first = await orchestrator.enqueueOrReuse(req); + const first = await orchestrator.enqueueOrReuse(buildPdfRequest('k-integration')); now = 2_000; - const reused = await orchestrator.enqueueOrReuse(req); + const reused = await orchestrator.enqueueOrReuse(buildPdfRequest('k-integration')); expect(first.opId).toBe('op-1'); expect(reused.opId).toBe('op-1'); const indexEntry = await kv.get(opIndexKvKey('k-integration')); const stateEntry = await kv.get(opStateKvKey('op-1')); + expect(indexEntry?.operation).toBe('PUT'); expect(stateEntry?.operation).toBe('PUT'); - expect(js.published.map((entry) => entry.subject)).toEqual(['ops.events.op-1', 'jobs.layout']); + expect(hashOpKey('k-integration')).toHaveLength(64); }); }); diff --git a/compute/worker/tests/unit/key-and-progress.test.ts b/compute/worker/tests/unit/key-and-progress.test.ts new file mode 100644 index 0000000..6887cad --- /dev/null +++ b/compute/worker/tests/unit/key-and-progress.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from 'vitest'; +import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/control-plane/jetstream'; +import { + buildInferProgressForPageParsed, + buildInferProgressForPageStart, +} from '../../src/pdf-progress'; + +describe('compute worker helpers', () => { + test('hash and kv key helpers are stable and deterministic', () => { + const hashA = hashOpKey('doc-123|layout|v1'); + const hashB = hashOpKey('doc-123|layout|v1'); + const hashC = hashOpKey('doc-123|layout|v2'); + + expect(hashA).toBe(hashB); + expect(hashA).toHaveLength(64); + expect(hashC).not.toBe(hashA); + + expect(opIndexKvKey('doc-123|layout|v1')).toBe(`op_index.${hashA}`); + expect(opStateKvKey('op-1')).toBe('op_state.op-1'); + }); + + test('progress helpers infer page counters correctly', () => { + expect(buildInferProgressForPageStart({ pageNumber: 1, totalPages: 12 })).toEqual({ + totalPages: 12, + pagesParsed: 0, + currentPage: 1, + phase: 'infer', + }); + + expect(buildInferProgressForPageParsed({ pageNumber: 5, totalPages: 12 })).toEqual({ + totalPages: 12, + pagesParsed: 5, + currentPage: 5, + phase: 'infer', + }); + }); +}); diff --git a/compute/worker/tests/unit/worker-loop-policy.test.ts b/compute/worker/tests/unit/worker-loop-policy.test.ts new file mode 100644 index 0000000..818e459 --- /dev/null +++ b/compute/worker/tests/unit/worker-loop-policy.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from 'vitest'; +import { buildQueueWaitTiming, decideRetryAction } from '../../src/worker-loop-policy'; + +describe('worker loop policy', () => { + test('returns queue wait timing with non-negative clamped duration', () => { + expect(buildQueueWaitTiming(1000, 1300)).toEqual({ queueWaitMs: 300 }); + expect(buildQueueWaitTiming(1500, 1300)).toEqual({ queueWaitMs: 0 }); + }); + + test('retry policy: layout jobs can retry until max attempts', () => { + expect(decideRetryAction({ kind: 'pdf_layout', deliveryCount: 1, pdfAttempts: 3 })).toBe('nak_retry'); + expect(decideRetryAction({ kind: 'pdf_layout', deliveryCount: 3, pdfAttempts: 3 })).toBe('term_fail'); + }); + + test('retry policy: whisper jobs terminate immediately by default', () => { + expect(decideRetryAction({ kind: 'whisper_align', deliveryCount: 1, pdfAttempts: 10 })).toBe('term_fail'); + }); +}); diff --git a/compute/worker/tsconfig.json b/compute/worker/tsconfig.json index 5912d1a..428e252 100644 --- a/compute/worker/tsconfig.json +++ b/compute/worker/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "noEmit": true + "noEmit": true, }, "include": ["src/**/*.ts"], "exclude": [] diff --git a/package.json b/package.json index eb62569..a7aa4d5 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,9 @@ "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", + "test:e2e": "playwright test", + "test:unit": "vitest run", + "test:compute": "vitest run --project compute-core --project compute-worker", "test:ci-env": "CI=true playwright test", "migrate": "node drizzle/scripts/migrate.mjs", "migrate-fs": "node scripts/migrate-fs-v2.mjs", @@ -90,6 +93,7 @@ "eslint-config-next": "^15.5.18", "postcss": "^8.5.14", "tailwindcss": "^3.4.19", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^4.1.7" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f15c27b..9d3aedd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,7 +42,7 @@ importers: version: 7.0.1 better-auth: specifier: ^1.6.11 - version: 1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + version: 1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))) better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -194,6 +194,9 @@ importers: typescript: specifier: ^5.9.3 version: 5.9.3 + vitest: + specifier: ^4.1.7 + version: 4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) compute/core: dependencies: @@ -1569,6 +1572,12 @@ packages: '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@napi-rs/wasm-runtime@1.1.4': + resolution: {integrity: sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + '@nats-io/jetstream@3.4.0': resolution: {integrity: sha512-GzHQodNJ942+R5LRb8PuZ5ugVWVWMRiufxUYLLVWkXKfwDXYN+Owo0d7L/b9O7BPyrbYD7jQWAC6+ZVuXa9Gyw==} @@ -1679,6 +1688,9 @@ packages: resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} + '@oxc-project/types@0.132.0': + resolution: {integrity: sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -1717,6 +1729,104 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1 + '@rolldown/binding-android-arm64@1.0.2': + resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.0.2': + resolution: {integrity: sha512-vdFA9+C/rekyGce7WqHs/xoT0ioZEWaOFyZLIV1mEeNFaFDUQrPIo8Vs2GvJ6eetb3rzDUtUBgzto3ExpXJB3w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.0.2': + resolution: {integrity: sha512-BewSOwTHazv77DTYiAZXSqqKZ4KP/KonFisDMVU7PImxoWfB2aepnPhd2E4SWz3zDzYgDNbs6jBmTdgNnF02GA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.0.2': + resolution: {integrity: sha512-m41o7M0YWtUdqk61Tb+jnKb2rN++iRdIASlExkUoKfIAH30DOHCB8fVLzSUpbWHHU8esmEioY62PxzexE8MBuA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-jcojB9H7W/jS29pMKWAK1N+fU99vXodHDTatS3b3y/XSOCiHo0kkA74pL3jJmkoQtYpOCxDvaKs1fo2Ij/1X5w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-1jn6qDU5iiOgFgygDzKUuKP0maTi0/f1+sBLgvij/76C77Nm3ts6ufz9Bjg5q5dduxiUIxtq86JIoBvo1xQ4Ig==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-QVLO/czFMdoMFSqlX3bcswcJNm/23r+qoa/jgtmFc/qEp6/jXmIkDjF/XIo8dPfGaiwy1xfQn8o77L79GeXFgw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + resolution: {integrity: sha512-hgO5Abm0w5UL6FEa2iFnZqo2KlK7TQ5QhV5x09hujBf7t5KzHQ1VmfPuTpqRy/rNlSxua3eWH374xxiVrP+lcA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.0.2': + resolution: {integrity: sha512-fy8rXxuYEu602abC8MUNaPjYLIFzReOaEIEMKMUa0rFEUxNpVXhs15KSSQ4qlqSaM7B6rcj9rDZgADh/IGDzLQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-0+bOkiQ779+r1WpoHOWHqncvyySci0vKph+myNDYb+im6meJAzHQXay6oEgnkHuUGouM1LKTZwqKpBow6Kj7CQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-mjSkrzZK5Qsl0a9d1JgILOiuZOSDTVdKENcSXBoqbzSrspLR/4/IRVDo5wd2GgZjNss/viBFJdeq+j7qH2nypw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.0.2': + resolution: {integrity: sha512-1v5vHasdfQAZoEHakBV72LIFAC9JjnymsiKxp+GEr/ma3+NJCPSaYK+qavInOovJkgwFrs7GccX2d6IgDA3Z5w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.0.2': + resolution: {integrity: sha512-mb1VobWn6NheziTk5/WEaR6AKVbrwT5sOi6C7zk3gy/pD1qtJfU1j4PgTo2NJnOtbL9Dl3Aeei8w9jJ7qC2jZQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@rolldown/binding-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-SqKonF56vA/L2yHwHYcEp2P34URpOZ7d1fS635cTkpDnUtEGdUbhI6NzsPdqeSWvAAeGDrxjWjNmibDIdFf9/A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-v7qRI7gXLRINcOGXt+7YmAZ6iFuyZVMIoXAxhd8oP+DR9dLfL9GfNIx7PLMxmhZdvq8waUJBQiWN9EKNy+TRBQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} @@ -1960,9 +2070,15 @@ packages: '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/debug@4.1.13': resolution: {integrity: sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==} + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -2207,6 +2323,35 @@ packages: vue-router: optional: true + '@vitest/expect@4.1.7': + resolution: {integrity: sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==} + + '@vitest/mocker@4.1.7': + resolution: {integrity: sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.1.7': + resolution: {integrity: sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==} + + '@vitest/runner@4.1.7': + resolution: {integrity: sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==} + + '@vitest/snapshot@4.1.7': + resolution: {integrity: sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==} + + '@vitest/spy@4.1.7': + resolution: {integrity: sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==} + + '@vitest/utils@4.1.7': + resolution: {integrity: sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==} + '@xmldom/xmldom@0.9.10': resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} engines: {node: '>=14.6'} @@ -2327,6 +2472,10 @@ packages: resolution: {integrity: sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==} engines: {node: '>= 0.4'} + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} @@ -2569,6 +2718,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -2635,6 +2788,9 @@ packages: resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} engines: {'0': node >= 6.0} + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + cookie@1.1.1: resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} engines: {node: '>=18'} @@ -2909,6 +3065,9 @@ packages: resolution: {integrity: sha512-HVLACW1TppGYjJ8H6/jqH/pqOtKRw6wMlrB23xfExmFWxFquAIWCmwoLsOyN96K4a5KbmOf5At9ZUO3GZbetAw==} engines: {node: '>= 0.4'} + es-module-lexer@2.1.0: + resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} @@ -3087,6 +3246,9 @@ packages: estree-util-is-identifier-name@3.0.0: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} @@ -3109,6 +3271,10 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + ext@1.7.0: resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} @@ -3624,6 +3790,80 @@ packages: light-my-request@6.6.0: resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -3658,6 +3898,9 @@ packages: resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} engines: {node: 20 || >=22} + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + make-cancellable-promise@1.3.2: resolution: {integrity: sha512-GCXh3bq/WuMbS+Ky4JBPW1hYTOU+znU+Q5m9Pu+pI8EoUqIHk9+tviOKC6/qhHh8C4/As3tzJ69IF32kdz85ww==} @@ -3953,6 +4196,9 @@ packages: resolution: {integrity: sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==} engines: {node: '>= 0.4'} + obug@2.1.1: + resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + on-exit-leak-free@2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} @@ -4037,6 +4283,9 @@ packages: resolution: {integrity: sha512-+vnG6S4dYcYxZd+CZxzXCNKdELYZSKfohrk98yajCo1PtRoDgCTrrwOvK1GT0UoAdVszagDVllQc0U1vaX4NUQ==} engines: {node: '>=6'} + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdfjs-dist@4.8.69: resolution: {integrity: sha512-IHZsA4T7YElCKNNXtiLgqScw4zPd3pG9do8UrznC757gMd7UPeHSL2qwNNMJo4r79fl8oj1Xx+1nh2YkzdMpLQ==} engines: {node: '>=18'} @@ -4177,6 +4426,10 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} + postcss@8.5.15: + resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} + engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} @@ -4411,6 +4664,11 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + rolldown@1.0.2: + resolution: {integrity: sha512-oZx5zVDtVB44AW3eaifgDml1gWRDZGvjcfdxonE4swNPG98PrrXjaO/KrnUjzlMnztCCRVlUueA1kCXhARGk6g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + rou3@0.7.12: resolution: {integrity: sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg==} @@ -4511,6 +4769,9 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -4545,6 +4806,12 @@ packages: stable-hash@0.0.5: resolution: {integrity: sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA==} + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@4.1.0: + resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + stop-iteration-iterator@1.1.0: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} @@ -4692,10 +4959,21 @@ packages: tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@1.2.3: + resolution: {integrity: sha512-g62dB+w1/OEFnPvmX0yd/HnetYITOL+1nJW7kitOycOeAvmbWC/nu0fwmmQ/kupNojqExzyC/T++pST/jRJ2mQ==} + engines: {node: '>=18'} + tinyglobby@0.2.16: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} + engines: {node: '>=14.0.0'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -4825,6 +5103,90 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite@8.0.14: + resolution: {integrity: sha512-s4BJJ+5y1pYL6Otw51FHhVJQhPnuRinKig64g/1+EUNaJsd3gCKdD31IPFvswUgW9/60QT9oFHbZHbQK5imcxw==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.1.18 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.1.7: + resolution: {integrity: sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.7 + '@vitest/browser-preview': 4.1.7 + '@vitest/browser-webdriverio': 4.1.7 + '@vitest/coverage-istanbul': 4.1.7 + '@vitest/coverage-v8': 4.1.7 + '@vitest/ui': 4.1.7 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + warning@4.0.3: resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} @@ -4849,6 +5211,11 @@ packages: engines: {node: '>= 8'} hasBin: true + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} @@ -6248,6 +6615,13 @@ snapshots: '@tybys/wasm-util': 0.10.2 optional: true + '@napi-rs/wasm-runtime@1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@tybys/wasm-util': 0.10.2 + optional: true + '@nats-io/jetstream@3.4.0': dependencies: '@nats-io/nats-core': 3.4.0 @@ -6326,6 +6700,8 @@ snapshots: '@opentelemetry/semantic-conventions@1.41.1': {} + '@oxc-project/types@0.132.0': {} + '@pinojs/redact@0.4.0': {} '@pkgjs/parseargs@0.11.0': @@ -6360,6 +6736,57 @@ snapshots: dependencies: react: 19.2.6 + '@rolldown/binding-android-arm64@1.0.2': + optional: true + + '@rolldown/binding-darwin-arm64@1.0.2': + optional: true + + '@rolldown/binding-darwin-x64@1.0.2': + optional: true + + '@rolldown/binding-freebsd-x64@1.0.2': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.0.2': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.0.2': + optional: true + + '@rolldown/binding-linux-x64-musl@1.0.2': + optional: true + + '@rolldown/binding-openharmony-arm64@1.0.2': + optional: true + + '@rolldown/binding-wasm32-wasi@1.0.2': + dependencies: + '@emnapi/core': 1.10.0 + '@emnapi/runtime': 1.10.0 + '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.0.2': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.0.2': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + '@rtsao/scc@1.1.0': {} '@rushstack/eslint-patch@1.16.1': {} @@ -6660,10 +7087,17 @@ snapshots: dependencies: '@types/node': 20.19.41 + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + '@types/debug@4.1.13': dependencies: '@types/ms': 2.1.0 + '@types/deep-eql@4.0.2': {} + '@types/estree-jsx@1.0.5': dependencies: '@types/estree': 1.0.9 @@ -6873,6 +7307,47 @@ snapshots: next: 15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 + '@vitest/expect@4.1.7': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 + chai: 6.2.2 + tinyrainbow: 3.1.0 + + '@vitest/mocker@4.1.7(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))': + dependencies: + '@vitest/spy': 4.1.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3) + + '@vitest/pretty-format@4.1.7': + dependencies: + tinyrainbow: 3.1.0 + + '@vitest/runner@4.1.7': + dependencies: + '@vitest/utils': 4.1.7 + pathe: 2.0.3 + + '@vitest/snapshot@4.1.7': + dependencies: + '@vitest/pretty-format': 4.1.7 + '@vitest/utils': 4.1.7 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.1.7': {} + + '@vitest/utils@4.1.7': + dependencies: + '@vitest/pretty-format': 4.1.7 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + '@xmldom/xmldom@0.9.10': {} abort-controller@3.0.0: @@ -7031,6 +7506,8 @@ snapshots: get-intrinsic: 1.3.0 is-array-buffer: 3.0.5 + assertion-error@2.0.1: {} + ast-types-flow@0.0.8: {} async-function@1.0.0: {} @@ -7096,7 +7573,7 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6): + better-auth@1.6.11(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0))(next@15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(pg@8.20.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3))): dependencies: '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.20.0)) @@ -7123,6 +7600,7 @@ snapshots: pg: 8.20.0 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) + vitest: 4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) transitivePeerDependencies: - '@cloudflare/workers-types' - '@opentelemetry/api' @@ -7219,6 +7697,8 @@ snapshots: ccount@2.0.1: {} + chai@6.2.2: {} + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -7287,6 +7767,8 @@ snapshots: readable-stream: 3.6.2 typedarray: 0.0.6 + convert-source-map@2.0.0: {} + cookie@1.1.1: {} core-js@3.49.0: {} @@ -7531,6 +8013,8 @@ snapshots: iterator.prototype: 1.1.5 math-intrinsics: 1.1.0 + es-module-lexer@2.1.0: {} + es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 @@ -7891,6 +8375,10 @@ snapshots: estree-util-is-identifier-name@3.0.0: {} + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + esutils@2.0.3: {} event-emitter@0.3.5: @@ -7910,6 +8398,8 @@ snapshots: expand-template@2.0.3: {} + expect-type@1.3.0: {} + ext@1.7.0: dependencies: type: 2.7.3 @@ -8485,6 +8975,55 @@ snapshots: process-warning: 4.0.1 set-cookie-parser: 2.7.2 + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} @@ -8511,6 +9050,10 @@ snapshots: lru-cache@11.3.6: {} + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + make-cancellable-promise@1.3.2: {} make-event-props@1.6.2: {} @@ -9008,6 +9551,8 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 + obug@2.1.1: {} + on-exit-leak-free@2.1.2: {} once@1.4.0: @@ -9087,6 +9632,8 @@ snapshots: path2d@0.2.2: optional: true + pathe@2.0.3: {} + pdfjs-dist@4.8.69: optionalDependencies: canvas: 3.2.3 @@ -9232,6 +9779,12 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postcss@8.5.15: + dependencies: + nanoid: 3.3.12 + picocolors: 1.1.1 + source-map-js: 1.2.1 + postgres-array@2.0.0: {} postgres-bytea@1.0.1: {} @@ -9537,6 +10090,27 @@ snapshots: rfdc@1.4.1: {} + rolldown@1.0.2: + dependencies: + '@oxc-project/types': 0.132.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.0.2 + '@rolldown/binding-darwin-arm64': 1.0.2 + '@rolldown/binding-darwin-x64': 1.0.2 + '@rolldown/binding-freebsd-x64': 1.0.2 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.2 + '@rolldown/binding-linux-arm64-gnu': 1.0.2 + '@rolldown/binding-linux-arm64-musl': 1.0.2 + '@rolldown/binding-linux-ppc64-gnu': 1.0.2 + '@rolldown/binding-linux-s390x-gnu': 1.0.2 + '@rolldown/binding-linux-x64-gnu': 1.0.2 + '@rolldown/binding-linux-x64-musl': 1.0.2 + '@rolldown/binding-openharmony-arm64': 1.0.2 + '@rolldown/binding-wasm32-wasi': 1.0.2 + '@rolldown/binding-win32-arm64-msvc': 1.0.2 + '@rolldown/binding-win32-x64-msvc': 1.0.2 + rou3@0.7.12: {} run-parallel@1.2.0: @@ -9678,6 +10252,8 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} + signal-exit@4.1.0: {} simple-concat@1.0.1: {} @@ -9707,6 +10283,10 @@ snapshots: stable-hash@0.0.5: {} + stackback@0.0.2: {} + + std-env@4.1.0: {} + stop-iteration-iterator@1.1.0: dependencies: es-errors: 1.3.0 @@ -9928,11 +10508,17 @@ snapshots: tiny-invariant@1.3.3: {} + tinybench@2.9.0: {} + + tinyexec@1.2.3: {} + tinyglobby@0.2.16: dependencies: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 + tinyrainbow@3.1.0: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -10110,6 +10696,47 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 + vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.4 + postcss: 8.5.15 + rolldown: 1.0.2 + tinyglobby: 0.2.16 + optionalDependencies: + '@types/node': 20.19.41 + esbuild: 0.28.0 + fsevents: 2.3.3 + jiti: 1.21.7 + tsx: 4.22.3 + + vitest@4.1.7(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)): + dependencies: + '@vitest/expect': 4.1.7 + '@vitest/mocker': 4.1.7(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3)) + '@vitest/pretty-format': 4.1.7 + '@vitest/runner': 4.1.7 + '@vitest/snapshot': 4.1.7 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.2.3 + tinyglobby: 0.2.16 + tinyrainbow: 3.1.0 + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.3) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.19.41 + transitivePeerDependencies: + - msw + warning@4.0.3: dependencies: loose-envify: 1.4.0 @@ -10159,6 +10786,11 @@ snapshots: dependencies: isexe: 2.0.0 + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + word-wrap@1.2.5: {} wrap-ansi@7.0.0: diff --git a/tests/unit/compute-abort-like-error.spec.ts b/tests/unit/compute-abort-like-error.spec.ts deleted file mode 100644 index e31287e..0000000 --- a/tests/unit/compute-abort-like-error.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { isAbortLikeError } from '../../src/lib/server/compute/abort-like-error'; - -test.describe('isAbortLikeError', () => { - test('matches abort-shaped errors', () => { - expect(isAbortLikeError(new DOMException('This operation was aborted', 'AbortError'))).toBe(true); - expect(isAbortLikeError(Object.assign(new Error('random'), { name: 'AbortError' }))).toBe(true); - expect(isAbortLikeError(new Error('This operation was aborted'))).toBe(true); - expect(isAbortLikeError({ name: 'AbortError' })).toBe(true); - }); - - test('does not match non-abort errors', () => { - expect(isAbortLikeError(new Error('boom'))).toBe(false); - expect(isAbortLikeError({ name: 'TypeError' })).toBe(false); - expect(isAbortLikeError(null)).toBe(false); - expect(isAbortLikeError(undefined)).toBe(false); - }); -}); diff --git a/tests/unit/compute-control-plane.spec.ts b/tests/unit/compute-control-plane.spec.ts deleted file mode 100644 index 7a9e338..0000000 --- a/tests/unit/compute-control-plane.spec.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { expect, test } from '@playwright/test'; -import type { WorkerOperationRequest, WorkerOperationState } from '../../compute/core/src/api-contracts'; -import { - encodeSseFrame, - explainReplacementReason, - InMemoryOperationEventStream, - InMemoryOperationQueue, - InMemoryOperationStateStore, - OperationOrchestrator, - parseSseEventId, - parseSsePayload, - shouldReuseExistingOperation, -} from '../../compute/core/src/control-plane'; - -function buildPdfLayoutRequest(opKey: string): WorkerOperationRequest { - return { - kind: 'pdf_layout', - opKey, - payload: { - documentId: `doc-${opKey}`, - documentObjectKey: `s3://bucket/${opKey}.pdf`, - namespace: null, - }, - }; -} - -test.describe('compute control-plane', () => { - test('in-memory queue and state store support enqueue/claim/CAS', async () => { - const queue = new InMemoryOperationQueue(); - const store = new InMemoryOperationStateStore(); - - await queue.enqueue({ - jobId: 'job-1', - opId: 'op-1', - opKey: 'k-1', - kind: 'pdf_layout', - queuedAt: 1000, - payload: { documentId: 'd1', documentObjectKey: 'obj1', namespace: null }, - }); - await queue.enqueue({ - jobId: 'job-2', - opId: 'op-2', - opKey: 'k-2', - kind: 'whisper_align', - queuedAt: 1100, - payload: { text: 'hello', audioObjectKey: 'obj2' }, - }); - - expect(queue.size()).toBe(2); - expect(queue.size('pdf_layout')).toBe(1); - expect(queue.size('whisper_align')).toBe(1); - - const claimedLayout = await queue.claimNext('pdf_layout'); - expect(claimedLayout?.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(0); - - const firstCas = await store.compareAndSetOpIndex({ - opKey: 'k-1', - newOpId: 'op-1', - expectedOpId: null, - }); - const secondCas = await store.compareAndSetOpIndex({ - opKey: 'k-1', - newOpId: 'op-2', - expectedOpId: null, - }); - - expect(firstCas).toBeTruthy(); - expect(secondCas).toBeFalsy(); - expect(await store.getOpIndex('k-1')).toEqual({ opId: 'op-1' }); - }); - - test('in-memory event stream replays from sinceEventId and streams live events', async () => { - const stream = new InMemoryOperationEventStream(); - - const queued: WorkerOperationState = { - opId: 'op-1', - opKey: 'k-1', - kind: 'pdf_layout', - jobId: 'job-1', - status: 'queued', - queuedAt: 1000, - updatedAt: 1000, - }; - const running: WorkerOperationState = { - ...queued, - status: 'running', - startedAt: 1200, - updatedAt: 1200, - }; - const succeeded: WorkerOperationState = { - ...running, - status: 'succeeded', - updatedAt: 1400, - result: { ok: true }, - }; - - await stream.append('op-1', queued); - await stream.append('op-1', running); - - const receivedEventIds: number[] = []; - const unsubscribe = await stream.subscribe({ - opId: 'op-1', - sinceEventId: 1, - onEvent: (event) => { - receivedEventIds.push(event.eventId); - }, - }); - - await stream.append('op-1', succeeded); - unsubscribe(); - - expect(receivedEventIds).toEqual([2, 3]); - }); - - test('orchestrator reuses fresh inflight operations and replaces stale ones', async () => { - let now = 1_000; - let nextId = 1; - const queue = new InMemoryOperationQueue(); - const stateStore = new InMemoryOperationStateStore(); - const eventStream = new InMemoryOperationEventStream(); - const orchestrator = new OperationOrchestrator({ - queue, - stateStore, - eventStream, - config: { opStaleMs: 2_000, maxCasRetries: 5 }, - clock: { now: () => now }, - idFactory: { - opId: () => `op-${nextId}`, - jobId: () => `job-${nextId++}`, - }, - }); - - const request = buildPdfLayoutRequest('same-op-key'); - - const first = await orchestrator.enqueueOrReuse(request); - expect(first.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(1); - - now = 2_000; - const reused = await orchestrator.enqueueOrReuse(request); - expect(reused.opId).toBe('op-1'); - expect(queue.size('pdf_layout')).toBe(1); - - await orchestrator.markRunning({ opId: first.opId, updatedAt: 2_100 }); - - now = 6_000; - const replaced = await orchestrator.enqueueOrReuse(request); - expect(replaced.opId).toBe('op-2'); - expect(queue.size('pdf_layout')).toBe(2); - expect(await stateStore.getOpIndex('same-op-key')).toEqual({ opId: 'op-2' }); - - const op1Events = await eventStream.listSince('op-1', 0); - const op2Events = await eventStream.listSince('op-2', 0); - expect(op1Events.map((event) => event.eventId)).toEqual([1, 2]); - expect(op2Events.map((event) => event.eventId)).toEqual([1]); - }); - - test('state-machine helpers return consistent reuse/replacement decisions', () => { - const current: WorkerOperationState = { - opId: 'op-1', - opKey: 'same-op-key', - kind: 'pdf_layout', - jobId: 'job-1', - status: 'running', - queuedAt: 1000, - updatedAt: 2000, - }; - - expect(shouldReuseExistingOperation({ - current, - requestKind: 'pdf_layout', - now: 2500, - opStaleMs: 1_000, - })).toBeTruthy(); - - expect(shouldReuseExistingOperation({ - current, - requestKind: 'pdf_layout', - now: 4005, - opStaleMs: 1_000, - })).toBeFalsy(); - - expect(explainReplacementReason({ - current, - requestKind: 'pdf_layout', - now: 4005, - opStaleMs: 1_000, - })).toBe('stale_running'); - }); - - test('sse helpers encode and decode frame payload and id', () => { - const frame = encodeSseFrame({ - id: 42, - event: 'snapshot', - data: { ok: true }, - }); - expect(parseSseEventId(frame)).toBe(42); - expect(parseSsePayload(frame)).toBe('{"ok":true}'); - }); -}); diff --git a/tests/unit/compute-worker-op-key.spec.ts b/tests/unit/compute-worker-op-key.spec.ts deleted file mode 100644 index 2101af6..0000000 --- a/tests/unit/compute-worker-op-key.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; - -test.describe('compute worker pdf opKey', () => { - test('keeps stable key when no force token is provided', () => { - const base = { - documentId: 'doc-123', - namespace: 'ns-1', - documentObjectKey: 'docs/ns-1/doc-123', - }; - expect(buildPdfOpKey(base)).toBe('pdf_layout|v1|doc-123|ns-1|docs/ns-1/doc-123|'); - expect(buildPdfOpKey(base)).toBe(buildPdfOpKey(base)); - }); - - test('cache-busts key when force token is provided', () => { - const base = { - documentId: 'doc-123', - namespace: 'ns-1', - documentObjectKey: 'docs/ns-1/doc-123', - }; - const opKeyA = buildPdfOpKey({ ...base, forceToken: 'force-a' }); - const opKeyB = buildPdfOpKey({ ...base, forceToken: 'force-b' }); - const normal = buildPdfOpKey(base); - - expect(opKeyA).not.toBe(opKeyB); - expect(opKeyA).not.toBe(normal); - expect(opKeyB).not.toBe(normal); - }); -}); diff --git a/tests/unit/compute-worker-pdf-progress.spec.ts b/tests/unit/compute-worker-pdf-progress.spec.ts deleted file mode 100644 index 284fd04..0000000 --- a/tests/unit/compute-worker-pdf-progress.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { - buildInferProgressForPageParsed, - buildInferProgressForPageStart, -} from '../../compute/worker/src/pdf-progress'; - -test.describe('compute worker pdf progress helpers', () => { - test('page-start progress keeps current page but does not count it as parsed yet', () => { - expect(buildInferProgressForPageStart({ pageNumber: 1, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 0, - currentPage: 1, - phase: 'infer', - }); - - expect(buildInferProgressForPageStart({ pageNumber: 5, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 4, - currentPage: 5, - phase: 'infer', - }); - }); - - test('page-parsed progress counts the current page as parsed', () => { - expect(buildInferProgressForPageParsed({ pageNumber: 5, totalPages: 12 })).toEqual({ - totalPages: 12, - pagesParsed: 5, - currentPage: 5, - phase: 'infer', - }); - }); -}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..fb32f4f --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + reporters: process.env.CI ? ['default', 'github'] : ['default'], + projects: [ + { + test: { + name: 'compute-core', + environment: 'node', + include: ['compute/core/tests/control-plane/**/*.test.ts'], + }, + }, + { + test: { + name: 'compute-worker', + environment: 'node', + include: ['compute/worker/tests/{unit,api}/**/*.test.ts'], + setupFiles: ['compute/worker/tests/setup-env.ts'], + }, + }, + ], + }, +}); From b16328abe9dd89f9c7dc948b30b27a3ade736e1d Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 11:01:44 -0600 Subject: [PATCH 05/16] fix(vitest): update test command from 'test:vitest' to 'test:unit' --- .github/workflows/vitest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 652663a..1ed52d9 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -20,4 +20,4 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run Vitest suites - run: pnpm test:vitest + run: pnpm test:unit From 7570181b8a9086d3a8e9a30debcfffa9197f78d7 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 11:11:44 -0600 Subject: [PATCH 06/16] test(unit): migrate batch1 core contracts to vitest --- tests/unit/auth-config.spec.ts | 119 ------------------ tests/unit/auth-config.vitest.spec.ts | 111 ++++++++++++++++ ...ate-limit-runtime-settings.vitest.spec.ts} | 7 +- ...s => server-error-contract.vitest.spec.ts} | 38 +++++- tests/unit/server-error-response.spec.ts | 43 ------- .../unit/server-error-response.vitest.spec.ts | 90 +++++++++++++ .../unit/server-route-error-mappings.spec.ts | 78 ------------ ...server-route-error-mappings.vitest.spec.ts | 80 ++++++++++++ ... shared-provider-selection.vitest.spec.ts} | 11 +- ...y.spec.ts => signup-policy.vitest.spec.ts} | 9 +- tests/unit/support/env.ts | 30 +++++ tests/unit/support/factories.ts | 30 +++++ tests/unit/support/server-stubs.ts | 22 ++++ vitest.config.ts | 21 ++++ 14 files changed, 433 insertions(+), 256 deletions(-) delete mode 100644 tests/unit/auth-config.spec.ts create mode 100644 tests/unit/auth-config.vitest.spec.ts rename tests/unit/{rate-limit-runtime-settings.spec.ts => rate-limit-runtime-settings.vitest.spec.ts} (76%) rename tests/unit/{server-error-contract.spec.ts => server-error-contract.vitest.spec.ts} (62%) delete mode 100644 tests/unit/server-error-response.spec.ts create mode 100644 tests/unit/server-error-response.vitest.spec.ts delete mode 100644 tests/unit/server-route-error-mappings.spec.ts create mode 100644 tests/unit/server-route-error-mappings.vitest.spec.ts rename tests/unit/{shared-provider-selection.spec.ts => shared-provider-selection.vitest.spec.ts} (85%) rename tests/unit/{signup-policy.spec.ts => signup-policy.vitest.spec.ts} (76%) create mode 100644 tests/unit/support/env.ts create mode 100644 tests/unit/support/factories.ts create mode 100644 tests/unit/support/server-stubs.ts diff --git a/tests/unit/auth-config.spec.ts b/tests/unit/auth-config.spec.ts deleted file mode 100644 index 24fb496..0000000 --- a/tests/unit/auth-config.spec.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { test, expect } from '@playwright/test'; -import { isAnonymousAuthSessionsEnabled, isGithubAuthEnabled } from '../../src/lib/server/auth/config'; - -const ORIGINAL_BASE_URL = process.env.BASE_URL; -const ORIGINAL_AUTH_SECRET = process.env.AUTH_SECRET; -const ORIGINAL_USE_ANON = process.env.USE_ANONYMOUS_AUTH_SESSIONS; -const ORIGINAL_GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID; -const ORIGINAL_GITHUB_CLIENT_SECRET = process.env.GITHUB_CLIENT_SECRET; - -function restoreEnv() { - if (ORIGINAL_BASE_URL === undefined) delete process.env.BASE_URL; - else process.env.BASE_URL = ORIGINAL_BASE_URL; - - if (ORIGINAL_AUTH_SECRET === undefined) delete process.env.AUTH_SECRET; - else process.env.AUTH_SECRET = ORIGINAL_AUTH_SECRET; - - if (ORIGINAL_USE_ANON === undefined) delete process.env.USE_ANONYMOUS_AUTH_SESSIONS; - else process.env.USE_ANONYMOUS_AUTH_SESSIONS = ORIGINAL_USE_ANON; - - if (ORIGINAL_GITHUB_CLIENT_ID === undefined) delete process.env.GITHUB_CLIENT_ID; - else process.env.GITHUB_CLIENT_ID = ORIGINAL_GITHUB_CLIENT_ID; - - if (ORIGINAL_GITHUB_CLIENT_SECRET === undefined) delete process.env.GITHUB_CLIENT_SECRET; - else process.env.GITHUB_CLIENT_SECRET = ORIGINAL_GITHUB_CLIENT_SECRET; -} - -function setAuthEnabledEnv() { - process.env.BASE_URL = 'http://localhost:3003'; - process.env.AUTH_SECRET = 'test-secret'; -} - -test.describe('auth config anonymous-session flag', () => { - test.afterEach(() => { - restoreEnv(); - }); - - test('returns false when auth is disabled', () => { - delete process.env.BASE_URL; - delete process.env.AUTH_SECRET; - process.env.USE_ANONYMOUS_AUTH_SESSIONS = 'true'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); - - test('defaults to false when env var is unset', () => { - setAuthEnabledEnv(); - delete process.env.USE_ANONYMOUS_AUTH_SESSIONS; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); - - test('returns true only when env var is true', () => { - setAuthEnabledEnv(); - process.env.USE_ANONYMOUS_AUTH_SESSIONS = 'true'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(true); - }); - - test('returns false when env var is false', () => { - setAuthEnabledEnv(); - process.env.USE_ANONYMOUS_AUTH_SESSIONS = 'false'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); - - test('falls back to false for invalid values', () => { - setAuthEnabledEnv(); - process.env.USE_ANONYMOUS_AUTH_SESSIONS = '1'; - - expect(isAnonymousAuthSessionsEnabled()).toBe(false); - }); -}); - -test.describe('auth config github-auth flag', () => { - test.afterEach(() => { - restoreEnv(); - }); - - test('returns false when auth is disabled', () => { - delete process.env.BASE_URL; - delete process.env.AUTH_SECRET; - process.env.GITHUB_CLIENT_ID = 'some-id'; - process.env.GITHUB_CLIENT_SECRET = 'some-secret'; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns false when GITHUB_CLIENT_ID is missing', () => { - setAuthEnabledEnv(); - delete process.env.GITHUB_CLIENT_ID; - process.env.GITHUB_CLIENT_SECRET = 'some-secret'; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns false when GITHUB_CLIENT_SECRET is missing', () => { - setAuthEnabledEnv(); - process.env.GITHUB_CLIENT_ID = 'some-id'; - delete process.env.GITHUB_CLIENT_SECRET; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns false when both GitHub env vars are missing', () => { - setAuthEnabledEnv(); - delete process.env.GITHUB_CLIENT_ID; - delete process.env.GITHUB_CLIENT_SECRET; - - expect(isGithubAuthEnabled()).toBe(false); - }); - - test('returns true when auth is enabled and both GitHub env vars are set', () => { - setAuthEnabledEnv(); - process.env.GITHUB_CLIENT_ID = 'some-id'; - process.env.GITHUB_CLIENT_SECRET = 'some-secret'; - - expect(isGithubAuthEnabled()).toBe(true); - }); -}); diff --git a/tests/unit/auth-config.vitest.spec.ts b/tests/unit/auth-config.vitest.spec.ts new file mode 100644 index 0000000..eac330d --- /dev/null +++ b/tests/unit/auth-config.vitest.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from 'vitest'; +import { getAuthBaseUrl, isAnonymousAuthSessionsEnabled, isAuthEnabled, isGithubAuthEnabled } from '../../src/lib/server/auth/config'; +import { withEnv } from './support/env'; + +describe('auth config contract', () => { + test('auth is enabled only when AUTH_SECRET and BASE_URL are both set', async () => { + await withEnv( + { + AUTH_SECRET: undefined, + BASE_URL: undefined, + }, + async () => { + expect(isAuthEnabled()).toBe(false); + expect(getAuthBaseUrl()).toBeNull(); + }, + ); + + await withEnv( + { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + }, + async () => { + expect(isAuthEnabled()).toBe(true); + expect(getAuthBaseUrl()).toBe('http://localhost:3003'); + }, + ); + }); + + test.each([ + { envValue: undefined, expected: false }, + { envValue: 'true', expected: true }, + { envValue: 'false', expected: false }, + { envValue: '1', expected: false }, + { envValue: 'TRUE', expected: true }, + ])( + 'anonymous sessions honor strict boolean parsing when auth is enabled (value: $envValue)', + async ({ envValue, expected }) => { + await withEnv( + { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + USE_ANONYMOUS_AUTH_SESSIONS: envValue, + }, + async () => { + expect(isAnonymousAuthSessionsEnabled()).toBe(expected); + }, + ); + }, + ); + + test('anonymous sessions are always disabled when auth is disabled', async () => { + await withEnv( + { + AUTH_SECRET: undefined, + BASE_URL: undefined, + USE_ANONYMOUS_AUTH_SESSIONS: 'true', + }, + async () => { + expect(isAnonymousAuthSessionsEnabled()).toBe(false); + }, + ); + }); + + test.each([ + { + title: 'returns false when auth is disabled', + env: { + AUTH_SECRET: undefined, + BASE_URL: undefined, + GITHUB_CLIENT_ID: 'id', + GITHUB_CLIENT_SECRET: 'secret', + }, + expected: false, + }, + { + title: 'returns false when GitHub client id is missing', + env: { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + GITHUB_CLIENT_ID: undefined, + GITHUB_CLIENT_SECRET: 'secret', + }, + expected: false, + }, + { + title: 'returns false when GitHub client secret is missing', + env: { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + GITHUB_CLIENT_ID: 'id', + GITHUB_CLIENT_SECRET: undefined, + }, + expected: false, + }, + { + title: 'returns true when auth is enabled and GitHub credentials are present', + env: { + AUTH_SECRET: 'unit-secret', + BASE_URL: 'http://localhost:3003', + GITHUB_CLIENT_ID: 'id', + GITHUB_CLIENT_SECRET: 'secret', + }, + expected: true, + }, + ])('$title', async ({ env, expected }) => { + await withEnv(env, async () => { + expect(isGithubAuthEnabled()).toBe(expected); + }); + }); +}); diff --git a/tests/unit/rate-limit-runtime-settings.spec.ts b/tests/unit/rate-limit-runtime-settings.vitest.spec.ts similarity index 76% rename from tests/unit/rate-limit-runtime-settings.spec.ts rename to tests/unit/rate-limit-runtime-settings.vitest.spec.ts index df7748d..b39f3e9 100644 --- a/tests/unit/rate-limit-runtime-settings.spec.ts +++ b/tests/unit/rate-limit-runtime-settings.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { RUNTIME_CONFIG_SCHEMA } from '../../src/lib/server/admin/settings'; -test.describe('TTS rate limit runtime config seeds', () => { +describe('TTS rate limit runtime config seeds', () => { test('defaults disable TTS daily rate limiting', () => { expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.default).toBe(true); }); @@ -12,6 +12,9 @@ test.describe('TTS rate limit runtime config seeds', () => { expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('false')).toBe(false); expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('1')).toBe(true); expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('0')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('on')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('no')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.disableTtsRateLimit.parseEnv('invalid')).toBeUndefined(); }); test('daily limit values are runtime-only (no env seed vars)', () => { diff --git a/tests/unit/server-error-contract.spec.ts b/tests/unit/server-error-contract.vitest.spec.ts similarity index 62% rename from tests/unit/server-error-contract.spec.ts rename to tests/unit/server-error-contract.vitest.spec.ts index 84307d7..e489af3 100644 --- a/tests/unit/server-error-contract.spec.ts +++ b/tests/unit/server-error-contract.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { ServerAppError, createServerAppError, @@ -7,10 +7,12 @@ import { toApiErrorBody, toHttpStatus, } from '../../src/lib/server/errors/contract'; +import { makeServerAppErrorInput } from './support/factories'; -test.describe('server error contract', () => { +describe('server error contract', () => { test('normalizes unknown throwable to fallback shape', () => { const normalized = normalizeServerError('boom'); + expect(normalized.code).toBe('UNKNOWN_SERVER_ERROR'); expect(normalized.errorClass).toBe('unknown'); expect(normalized.httpStatus).toBe(500); @@ -19,15 +21,17 @@ test.describe('server error contract', () => { }); test('preserves ServerAppError metadata', () => { - const err = createServerAppError({ + const err = createServerAppError(makeServerAppErrorInput({ code: 'USER_PROGRESS_UPDATE_FAILED', message: 'Failed to update progress', errorClass: 'db', retryable: true, httpStatus: 500, details: { operation: 'update_progress' }, - }); + })); + const normalized = normalizeServerError(err); + expect(isServerAppError(err)).toBe(true); expect(normalized.code).toBe('USER_PROGRESS_UPDATE_FAILED'); expect(normalized.errorClass).toBe('db'); @@ -36,6 +40,22 @@ test.describe('server error contract', () => { expect(normalized.details?.operation).toBe('update_progress'); }); + test('normalizes partially-shaped thrown errors with code sanitization', () => { + const thrown = Object.assign(new Error('Timed out waiting for storage read'), { + code: 'not_valid_code', + errorClass: 'timeout', + httpStatus: 604, + retryable: true, + }); + + const normalized = normalizeServerError(thrown); + + expect(normalized.code).toBe('UNKNOWN_SERVER_ERROR'); + expect(normalized.errorClass).toBe('timeout'); + expect(normalized.httpStatus).toBe(599); + expect(normalized.retryable).toBe(true); + }); + test('maps normalized errors to API body + status', () => { const normalized = normalizeServerError( new ServerAppError({ @@ -44,7 +64,9 @@ test.describe('server error contract', () => { errorClass: 'upstream', }), ); + const body = toApiErrorBody(normalized, { includeDetails: false }); + expect(body).toEqual({ error: 'Upstream failure', errorCode: 'UPSTREAM_TTS_ERROR', @@ -52,4 +74,12 @@ test.describe('server error contract', () => { }); expect(toHttpStatus(normalized)).toBe(502); }); + + test('rejects invalid server error code at creation boundary', () => { + expect(() => createServerAppError(makeServerAppErrorInput({ + code: 'bad-code', + message: 'Invalid', + errorClass: 'validation', + }))).toThrow(/Invalid ServerAppError code/i); + }); }); diff --git a/tests/unit/server-error-response.spec.ts b/tests/unit/server-error-response.spec.ts deleted file mode 100644 index a5ed779..0000000 --- a/tests/unit/server-error-response.spec.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { errorResponse } from '../../src/lib/server/errors/next-response'; -import { createServerAppError } from '../../src/lib/server/errors/contract'; - -test.describe('server error response helper', () => { - test('returns mapped 4xx response with explicit app code', async () => { - const response = errorResponse( - createServerAppError({ - code: 'AUTH_UNAUTHORIZED', - message: 'Unauthorized', - errorClass: 'auth', - httpStatus: 401, - retryable: false, - }), - { - apiErrorMessage: 'Unauthorized', - }, - ); - - expect(response.status).toBe(401); - await expect(response.json()).resolves.toEqual({ - error: 'Unauthorized', - errorCode: 'AUTH_UNAUTHORIZED', - retryable: false, - }); - }); - - test('normalizes unknown errors to safe 500 without stack leakage', async () => { - const response = errorResponse(new Error('sensitive internal details'), { - apiErrorMessage: 'Internal Server Error', - normalize: { code: 'UNKNOWN_SERVER_ERROR', errorClass: 'unknown' }, - }); - const body = await response.json(); - expect(response.status).toBe(500); - expect(body).toEqual({ - error: 'Internal Server Error', - errorCode: 'UNKNOWN_SERVER_ERROR', - retryable: false, - }); - expect(JSON.stringify(body)).not.toContain('stack'); - expect(JSON.stringify(body)).not.toContain('cause'); - }); -}); diff --git a/tests/unit/server-error-response.vitest.spec.ts b/tests/unit/server-error-response.vitest.spec.ts new file mode 100644 index 0000000..5bfba1d --- /dev/null +++ b/tests/unit/server-error-response.vitest.spec.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'vitest'; +import { createServerAppError } from '../../src/lib/server/errors/contract'; +import { asRouteErrorContext, errorResponse, withErrorBoundary } from '../../src/lib/server/errors/next-response'; +import { createServerLoggerStub } from './support/server-stubs'; +import { makeServerAppErrorInput } from './support/factories'; + +describe('server error response helper', () => { + test('returns mapped 4xx response with explicit app code', async () => { + const response = errorResponse( + createServerAppError(makeServerAppErrorInput({ + code: 'AUTH_UNAUTHORIZED', + message: 'Unauthorized', + errorClass: 'auth', + httpStatus: 401, + retryable: false, + })), + { + apiErrorMessage: 'Unauthorized', + }, + ); + + expect(response.status).toBe(401); + await expect(response.json()).resolves.toEqual({ + error: 'Unauthorized', + errorCode: 'AUTH_UNAUTHORIZED', + retryable: false, + }); + }); + + test('normalizes unknown errors to safe 500 without stack leakage', async () => { + const response = errorResponse(new Error('sensitive internal details'), { + apiErrorMessage: 'Internal Server Error', + normalize: { code: 'UNKNOWN_SERVER_ERROR', errorClass: 'unknown' }, + }); + const body = await response.json(); + expect(response.status).toBe(500); + expect(body).toEqual({ + error: 'Internal Server Error', + errorCode: 'UNKNOWN_SERVER_ERROR', + retryable: false, + }); + expect(JSON.stringify(body)).not.toContain('stack'); + expect(JSON.stringify(body)).not.toContain('cause'); + }); + + test('withErrorBoundary returns normalized error response and emits a structured log', async () => { + const logger = createServerLoggerStub(); + const handler = withErrorBoundary( + async () => { + throw new Error('storage dependency timed out'); + }, + { + route: '/api/unit-test', + logger: logger as never, + event: 'unit_test.route.failed', + msg: 'Route failed', + normalize: { + code: 'DOCUMENT_BLOB_FETCH_FAILED', + errorClass: 'storage', + }, + apiErrorMessage: 'Failed to fetch document blob', + }, + ); + + const response = await handler(); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body).toEqual({ + error: 'Failed to fetch document blob', + errorCode: 'DOCUMENT_BLOB_FETCH_FAILED', + retryable: true, + }); + expect(logger.error).toHaveBeenCalledTimes(1); + }); + + test('asRouteErrorContext derives method and request id consistently', () => { + const context = asRouteErrorContext({ + request: { method: 'PATCH' } as never, + route: '/api/runtime-config', + requestId: 'req-123', + }); + + expect(context).toEqual({ + route: '/api/runtime-config', + method: 'PATCH', + requestId: 'req-123', + }); + }); +}); diff --git a/tests/unit/server-route-error-mappings.spec.ts b/tests/unit/server-route-error-mappings.spec.ts deleted file mode 100644 index ac1a131..0000000 --- a/tests/unit/server-route-error-mappings.spec.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { errorResponse } from '../../src/lib/server/errors/next-response'; -import { AdminProviderError } from '../../src/lib/server/admin/providers'; - -test.describe('route error mapping contract', () => { - test('admin provider validation errors map to 4xx via route normalize policy', async () => { - const adminError = new AdminProviderError('slug is required', 400); - const response = errorResponse(adminError, { - apiErrorMessage: adminError.message, - normalize: { - code: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', - errorClass: 'validation', - httpStatus: adminError.status, - retryable: false, - }, - }); - - expect(response.status).toBe(400); - await expect(response.json()).resolves.toEqual({ - error: 'slug is required', - errorCode: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', - retryable: false, - }); - }); - - test('documents storage failures map to retryable 503', async () => { - const response = errorResponse(new Error('blobstore timeout'), { - apiErrorMessage: 'Failed to fetch document blob', - normalize: { - code: 'DOCUMENT_BLOB_FETCH_FAILED', - errorClass: 'storage', - }, - }); - - expect(response.status).toBe(503); - await expect(response.json()).resolves.toEqual({ - error: 'Failed to fetch document blob', - errorCode: 'DOCUMENT_BLOB_FETCH_FAILED', - retryable: true, - }); - }); - - test('audiobook upstream failures map to retryable 502', async () => { - const response = errorResponse(new Error('provider overloaded'), { - apiErrorMessage: 'Failed to process audio chapter', - normalize: { - code: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', - errorClass: 'upstream', - }, - }); - - expect(response.status).toBe(502); - await expect(response.json()).resolves.toEqual({ - error: 'Failed to process audio chapter', - errorCode: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', - retryable: true, - }); - }); - - test('user export auth initialization failure maps to 500 auth classification', async () => { - const response = errorResponse(new Error('Auth not initialized'), { - apiErrorMessage: 'Auth not initialized', - normalize: { - code: 'USER_EXPORT_AUTH_NOT_INITIALIZED', - errorClass: 'auth', - httpStatus: 500, - retryable: false, - }, - }); - - expect(response.status).toBe(500); - await expect(response.json()).resolves.toEqual({ - error: 'Auth not initialized', - errorCode: 'USER_EXPORT_AUTH_NOT_INITIALIZED', - retryable: false, - }); - }); -}); diff --git a/tests/unit/server-route-error-mappings.vitest.spec.ts b/tests/unit/server-route-error-mappings.vitest.spec.ts new file mode 100644 index 0000000..6d5bb23 --- /dev/null +++ b/tests/unit/server-route-error-mappings.vitest.spec.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from 'vitest'; +import { AdminProviderError } from '../../src/lib/server/admin/providers'; +import { errorResponse } from '../../src/lib/server/errors/next-response'; +import { makeServerErrorContext } from './support/factories'; + +describe('route error mapping contract', () => { + test('admin provider validation errors map to 4xx via route normalize policy', async () => { + const adminError = new AdminProviderError('slug is required', 400); + const response = errorResponse(adminError, { + apiErrorMessage: adminError.message, + normalize: makeServerErrorContext({ + code: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', + errorClass: 'validation', + httpStatus: adminError.status, + retryable: false, + }), + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toEqual({ + error: 'slug is required', + errorCode: 'ADMIN_PROVIDERS_CREATE_REQUEST_FAILED', + retryable: false, + }); + }); + + test.each([ + { + name: 'documents storage failures map to retryable 503', + thrown: new Error('blobstore timeout'), + apiErrorMessage: 'Failed to fetch document blob', + normalize: makeServerErrorContext({ + code: 'DOCUMENT_BLOB_FETCH_FAILED', + errorClass: 'storage', + }), + expectedStatus: 503, + expectedBody: { + error: 'Failed to fetch document blob', + errorCode: 'DOCUMENT_BLOB_FETCH_FAILED', + retryable: true, + }, + }, + { + name: 'audiobook upstream failures map to retryable 502', + thrown: new Error('provider overloaded'), + apiErrorMessage: 'Failed to process audio chapter', + normalize: makeServerErrorContext({ + code: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', + errorClass: 'upstream', + }), + expectedStatus: 502, + expectedBody: { + error: 'Failed to process audio chapter', + errorCode: 'AUDIOBOOK_CHAPTER_PROCESS_FAILED', + retryable: true, + }, + }, + { + name: 'user export auth initialization failure maps to 500 auth classification', + thrown: new Error('Auth not initialized'), + apiErrorMessage: 'Auth not initialized', + normalize: makeServerErrorContext({ + code: 'USER_EXPORT_AUTH_NOT_INITIALIZED', + errorClass: 'auth', + httpStatus: 500, + retryable: false, + }), + expectedStatus: 500, + expectedBody: { + error: 'Auth not initialized', + errorCode: 'USER_EXPORT_AUTH_NOT_INITIALIZED', + retryable: false, + }, + }, + ])('$name', async ({ thrown, apiErrorMessage, normalize, expectedStatus, expectedBody }) => { + const response = errorResponse(thrown, { apiErrorMessage, normalize }); + expect(response.status).toBe(expectedStatus); + await expect(response.json()).resolves.toEqual(expectedBody); + }); +}); diff --git a/tests/unit/shared-provider-selection.spec.ts b/tests/unit/shared-provider-selection.vitest.spec.ts similarity index 85% rename from tests/unit/shared-provider-selection.spec.ts rename to tests/unit/shared-provider-selection.vitest.spec.ts index a371989..69d9eb2 100644 --- a/tests/unit/shared-provider-selection.spec.ts +++ b/tests/unit/shared-provider-selection.vitest.spec.ts @@ -1,14 +1,11 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { resolvePreferredSharedProviderSlug } from '../../src/lib/shared/shared-provider-selection'; +import { makeSharedProviders } from './support/factories'; -const PROVIDERS = [ - { slug: 'shared-a' }, - { slug: 'default-openai' }, - { slug: 'shared-b' }, -] as const; +const PROVIDERS = makeSharedProviders(['shared-a', 'default-openai', 'shared-b']); -test.describe('resolvePreferredSharedProviderSlug', () => { +describe('resolvePreferredSharedProviderSlug', () => { test('prefers requested shared slug when present', () => { expect(resolvePreferredSharedProviderSlug({ providers: PROVIDERS, diff --git a/tests/unit/signup-policy.spec.ts b/tests/unit/signup-policy.vitest.spec.ts similarity index 76% rename from tests/unit/signup-policy.spec.ts rename to tests/unit/signup-policy.vitest.spec.ts index 190ef67..a74ba83 100644 --- a/tests/unit/signup-policy.spec.ts +++ b/tests/unit/signup-policy.vitest.spec.ts @@ -1,9 +1,9 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { RUNTIME_CONFIG_SCHEMA } from '../../src/lib/server/admin/settings'; import { assertUserSignupAllowed } from '../../src/lib/server/auth/signup-policy'; -test.describe('enableUserSignups runtime config', () => { +describe('enableUserSignups runtime config', () => { test('defaults to enabled', () => { expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.default).toBe(true); }); @@ -13,10 +13,13 @@ test.describe('enableUserSignups runtime config', () => { expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('false')).toBe(false); expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('1')).toBe(true); expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('0')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('yes')).toBe(true); + expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('off')).toBe(false); + expect(RUNTIME_CONFIG_SCHEMA.enableUserSignups.parseEnv('maybe')).toBeUndefined(); }); }); -test.describe('signup policy enforcement', () => { +describe('signup policy enforcement', () => { test('allows new non-anonymous users when signups are enabled', () => { expect(() => assertUserSignupAllowed({ enableUserSignups: true, isAnonymous: false })).not.toThrow(); }); diff --git a/tests/unit/support/env.ts b/tests/unit/support/env.ts new file mode 100644 index 0000000..4f76129 --- /dev/null +++ b/tests/unit/support/env.ts @@ -0,0 +1,30 @@ +export type EnvPatch = Record; + +export function captureEnv(keys: readonly string[]): EnvPatch { + const snapshot: EnvPatch = {}; + for (const key of keys) { + snapshot[key] = process.env[key]; + } + return snapshot; +} + +export function restoreEnv(snapshot: EnvPatch): void { + for (const [key, value] of Object.entries(snapshot)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +} + +export async function withEnv(patch: EnvPatch, run: () => T | Promise): Promise { + const keys = Object.keys(patch); + const snapshot = captureEnv(keys); + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { + return await run(); + } finally { + restoreEnv(snapshot); + } +} diff --git a/tests/unit/support/factories.ts b/tests/unit/support/factories.ts new file mode 100644 index 0000000..cb8835b --- /dev/null +++ b/tests/unit/support/factories.ts @@ -0,0 +1,30 @@ +import type { ServerErrorClass, ServerErrorContext } from '../../../src/lib/server/errors/contract'; + +export function makeServerAppErrorInput(overrides: Partial<{ + code: string; + message: string; + errorClass: ServerErrorClass; + httpStatus: number; + retryable: boolean; + details: Record; + cause: unknown; +}> = {}) { + return { + code: 'UNKNOWN_SERVER_ERROR', + message: 'Unhandled server failure', + errorClass: 'unknown' as const, + ...overrides, + }; +} + +export function makeServerErrorContext(overrides: ServerErrorContext = {}): ServerErrorContext { + return { + code: 'UNKNOWN_SERVER_ERROR', + errorClass: 'unknown', + ...overrides, + }; +} + +export function makeSharedProviders(slugs: readonly string[]): Array<{ slug: string }> { + return slugs.map((slug) => ({ slug })); +} diff --git a/tests/unit/support/server-stubs.ts b/tests/unit/support/server-stubs.ts new file mode 100644 index 0000000..c8b7e29 --- /dev/null +++ b/tests/unit/support/server-stubs.ts @@ -0,0 +1,22 @@ +import type { Mock } from 'vitest'; +import { vi } from 'vitest'; + +export interface ServerLoggerStub { + child: Mock; + debug: Mock; + error: Mock; + info: Mock; + warn: Mock; +} + +export function createServerLoggerStub(): ServerLoggerStub { + const logger = { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + child: vi.fn(), + } as ServerLoggerStub; + logger.child.mockReturnValue(logger); + return logger; +} diff --git a/vitest.config.ts b/vitest.config.ts index fb32f4f..fe7ca8b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,9 +1,30 @@ import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'node:url'; + +const srcDir = fileURLToPath(new URL('./src/', import.meta.url)); +const alias = [ + { find: /^@\//, replacement: `${srcDir}` }, + { find: '@', replacement: srcDir }, +]; export default defineConfig({ + resolve: { + alias, + }, test: { + alias, reporters: process.env.CI ? ['default', 'github'] : ['default'], projects: [ + { + resolve: { + alias, + }, + test: { + name: 'openreader', + environment: 'node', + include: ['tests/unit/**/*.vitest.spec.ts'], + }, + }, { test: { name: 'compute-core', From 22a1b5de57b5906c393316af5e4f251fed8eaa20 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 11:15:19 -0600 Subject: [PATCH 07/16] test(unit): migrate batch2 document and parser suites to vitest --- ... canonicalize-epub-segment.vitest.spec.ts} | 6 +-- ...cstore.spec.ts => docstore.vitest.spec.ts} | 15 ++++-- ....spec.ts => document-cache.vitest.spec.ts} | 41 ++++++++++---- ... => html-audiobook-adapter.vitest.spec.ts} | 8 +-- ...cks.spec.ts => html-blocks.vitest.spec.ts} | 10 ++-- ...s => pdf-audiobook-adapter.vitest.spec.ts} | 8 +-- ....ts => pdf-build-page-text.vitest.spec.ts} | 4 +- ...ec.ts => pdf-force-reparse.vitest.spec.ts} | 4 +- ...df-merge-text-with-regions.vitest.spec.ts} | 18 ++++++- ...parse-normalize-text-items.vitest.spec.ts} | 12 ++++- ...f-stitch-cross-page-blocks.vitest.spec.ts} | 19 +++---- ...pec.ts => pdf-tts-planning.vitest.spec.ts} | 4 +- tests/unit/support/document-fixtures.ts | 54 +++++++++++++++++++ vitest.config.ts | 10 ++++ 14 files changed, 161 insertions(+), 52 deletions(-) rename tests/unit/{canonicalize-epub-segment.spec.ts => canonicalize-epub-segment.vitest.spec.ts} (97%) rename tests/unit/{docstore.spec.ts => docstore.vitest.spec.ts} (79%) rename tests/unit/{document-cache.spec.ts => document-cache.vitest.spec.ts} (71%) rename tests/unit/{html-audiobook-adapter.spec.ts => html-audiobook-adapter.vitest.spec.ts} (95%) rename tests/unit/{html-blocks.spec.ts => html-blocks.vitest.spec.ts} (94%) rename tests/unit/{pdf-audiobook-adapter.spec.ts => pdf-audiobook-adapter.vitest.spec.ts} (96%) rename tests/unit/{pdf-build-page-text.spec.ts => pdf-build-page-text.vitest.spec.ts} (92%) rename tests/unit/{pdf-force-reparse.spec.ts => pdf-force-reparse.vitest.spec.ts} (89%) rename tests/unit/{pdf-merge-text-with-regions.spec.ts => pdf-merge-text-with-regions.vitest.spec.ts} (59%) rename tests/unit/{pdf-parse-normalize-text-items.spec.ts => pdf-parse-normalize-text-items.vitest.spec.ts} (69%) rename tests/unit/{pdf-stitch-cross-page-blocks.spec.ts => pdf-stitch-cross-page-blocks.vitest.spec.ts} (89%) rename tests/unit/{pdf-tts-planning.spec.ts => pdf-tts-planning.vitest.spec.ts} (97%) create mode 100644 tests/unit/support/document-fixtures.ts diff --git a/tests/unit/canonicalize-epub-segment.spec.ts b/tests/unit/canonicalize-epub-segment.vitest.spec.ts similarity index 97% rename from tests/unit/canonicalize-epub-segment.spec.ts rename to tests/unit/canonicalize-epub-segment.vitest.spec.ts index 42a02b8..164d857 100644 --- a/tests/unit/canonicalize-epub-segment.spec.ts +++ b/tests/unit/canonicalize-epub-segment.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { canonicalizeEpubSegmentAgainstSpineText, @@ -6,7 +6,7 @@ import { } from '../../src/lib/client/epub/canonicalize-epub-segment'; import { planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan'; -test.describe('canonicalizeEpubSegmentAgainstSpineText', () => { +describe('canonicalizeEpubSegmentAgainstSpineText', () => { test('maps an exact sentence to the canonical segment identity', () => { const spineText = [ 'First section sentence with enough words to stand alone.', @@ -95,7 +95,7 @@ test.describe('canonicalizeEpubSegmentAgainstSpineText', () => { }); }); -test.describe('canonicalizeEpubSegmentsAgainstSpineText', () => { +describe('canonicalizeEpubSegmentsAgainstSpineText', () => { test('maps overlap-boundary drift sentences to forward canonical segments', () => { const sourceSentences = [ 'The star was particularly bright when the station lights switched off for cycle night.', diff --git a/tests/unit/docstore.spec.ts b/tests/unit/docstore.vitest.spec.ts similarity index 79% rename from tests/unit/docstore.spec.ts rename to tests/unit/docstore.vitest.spec.ts index 7e9f144..684cc52 100644 --- a/tests/unit/docstore.spec.ts +++ b/tests/unit/docstore.vitest.spec.ts @@ -1,10 +1,10 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { getMigratedDocumentFileName } from '../../src/lib/server/storage/docstore-legacy'; -test.describe('Docstore Filename Safety', () => { +describe('docstore filename safety', () => { const id = 'a'.repeat(64); // Simulate sha256 hex ID - test('should generate standard filename for short names', () => { + test('generates stable file names for short inputs', () => { const name = 'test-file.pdf'; const result = getMigratedDocumentFileName(id, name); expect(result).toBe(`${id}__test-file.pdf`); @@ -50,4 +50,13 @@ test.describe('Docstore Filename Safety', () => { expect(result.length).toBe(240); expect(result).not.toContain('truncated-'); }); + + test('drops path traversal and null-byte fragments from migrated file names', () => { + const dirtyName = '../nested/\u0000financial-report.pdf'; + const result = getMigratedDocumentFileName(id, dirtyName); + + expect(result).toBe(`${id}__financial-report.pdf`); + expect(result).not.toContain('..'); + expect(result).not.toContain('\u0000'); + }); }); diff --git a/tests/unit/document-cache.spec.ts b/tests/unit/document-cache.vitest.spec.ts similarity index 71% rename from tests/unit/document-cache.spec.ts rename to tests/unit/document-cache.vitest.spec.ts index 061db9a..c807c39 100644 --- a/tests/unit/document-cache.spec.ts +++ b/tests/unit/document-cache.vitest.spec.ts @@ -1,16 +1,17 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import type { BaseDocument, EPUBDocument, HTMLDocument, PDFDocument } from '../../src/types/documents'; import { ensureCachedDocumentCore } from '../../src/lib/client/cache/documents'; +import { makeBaseDocument } from './support/document-fixtures'; -test.describe('document-cache-core', () => { +describe('document-cache-core', () => { test('returns cached PDF without downloading', async () => { - const meta: BaseDocument = { + const meta: BaseDocument = makeBaseDocument({ id: 'pdf1', name: 'a.pdf', size: 10, - lastModified: Date.now(), + lastModified: 1_700_000_000_001, type: 'pdf', - }; + }); let downloads = 0; const cached: PDFDocument = { ...meta, type: 'pdf', data: new ArrayBuffer(1) }; @@ -32,13 +33,13 @@ test.describe('document-cache-core', () => { }); test('downloads and stores on cache miss (EPUB)', async () => { - const meta: BaseDocument = { + const meta: BaseDocument = makeBaseDocument({ id: 'epub1', name: 'b.epub', size: 10, - lastModified: Date.now(), + lastModified: 1_700_000_000_002, type: 'epub', - }; + }); const store = new Map(); let downloads = 0; @@ -63,13 +64,13 @@ test.describe('document-cache-core', () => { }); test('downloads, decodes, and stores HTML on cache miss', async () => { - const meta: BaseDocument = { + const meta: BaseDocument = makeBaseDocument({ id: 'html1', name: 'c.txt', size: 5, - lastModified: Date.now(), + lastModified: 1_700_000_000_003, type: 'html', - }; + }); const store = new Map(); let decodedCalls = 0; @@ -92,4 +93,22 @@ test.describe('document-cache-core', () => { expect(result.type).toBe('html'); expect((result as HTMLDocument).data).toBe('hello'); }); + + test('throws deterministic cache error when download succeeds but backend misses persisted PDF', async () => { + const meta = makeBaseDocument({ + id: 'pdf-cache-miss', + type: 'pdf', + }); + + await expect( + ensureCachedDocumentCore(meta, { + get: async () => null, + putPdf: async () => { /* simulate write path without persisted row */ }, + putEpub: async () => { /* unused */ }, + putHtml: async () => { /* unused */ }, + download: async () => new Uint8Array([1, 2, 3]).buffer, + decodeText: () => '', + }), + ).rejects.toThrow('Failed to cache PDF'); + }); }); diff --git a/tests/unit/html-audiobook-adapter.spec.ts b/tests/unit/html-audiobook-adapter.vitest.spec.ts similarity index 95% rename from tests/unit/html-audiobook-adapter.spec.ts rename to tests/unit/html-audiobook-adapter.vitest.spec.ts index 9395533..7fda86c 100644 --- a/tests/unit/html-audiobook-adapter.spec.ts +++ b/tests/unit/html-audiobook-adapter.vitest.spec.ts @@ -1,11 +1,11 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { createHtmlAudiobookSourceAdapter } from '../../src/lib/client/audiobooks/adapters/html'; import { parseHtmlBlocks, type HtmlBlock } from '../../src/lib/client/html/blocks'; const blocksFromMd = (src: string): HtmlBlock[] => parseHtmlBlocks(src, false); const blocksFromTxt = (src: string): HtmlBlock[] => parseHtmlBlocks(src, true); -test.describe('createHtmlAudiobookSourceAdapter (markdown chapter splitting)', () => { +describe('createHtmlAudiobookSourceAdapter (markdown chapter splitting)', () => { test('starts a new chapter at each h1/h2 heading by default', async () => { const blocks = blocksFromMd( [ @@ -95,7 +95,7 @@ test.describe('createHtmlAudiobookSourceAdapter (markdown chapter splitting)', ( }); }); -test.describe('createHtmlAudiobookSourceAdapter (txt chapter splitting)', () => { +describe('createHtmlAudiobookSourceAdapter (txt chapter splitting)', () => { test('chunks TXT documents into "Part N" of 50 blocks by default', async () => { const blocks = blocksFromTxt( Array.from({ length: 120 }, (_, i) => `Block ${i + 1}.`).join('\n\n'), @@ -130,7 +130,7 @@ test.describe('createHtmlAudiobookSourceAdapter (txt chapter splitting)', () => }); }); -test.describe('createHtmlAudiobookSourceAdapter — prepareChapter', () => { +describe('createHtmlAudiobookSourceAdapter — prepareChapter', () => { test('returns the same chapter by index that prepareChapters lists', async () => { const blocks = blocksFromMd(['# A', '', 'one', '', '## B', '', 'two'].join('\n')); const adapter = createHtmlAudiobookSourceAdapter({ diff --git a/tests/unit/html-blocks.spec.ts b/tests/unit/html-blocks.vitest.spec.ts similarity index 94% rename from tests/unit/html-blocks.spec.ts rename to tests/unit/html-blocks.vitest.spec.ts index 533d153..02f6b0e 100644 --- a/tests/unit/html-blocks.spec.ts +++ b/tests/unit/html-blocks.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { mdToPlainText, parseHtmlBlocks, @@ -6,7 +6,7 @@ import { splitTxtBlocks, } from '../../src/lib/client/html/blocks'; -test.describe('parseHtmlBlocks (markdown)', () => { +describe('parseHtmlBlocks (markdown)', () => { test('splits headings, paragraphs, and lists into separate blocks', () => { const src = [ '# Title', @@ -55,7 +55,7 @@ test.describe('parseHtmlBlocks (markdown)', () => { }); }); -test.describe('parseHtmlBlocks (txt)', () => { +describe('parseHtmlBlocks (txt)', () => { test('splits on blank-line boundaries and preserves intra-block whitespace', () => { const src = 'First block\nmore.\n\nSecond block.\n\n\nThird block.'; const blocks = parseHtmlBlocks(src, true); @@ -73,7 +73,7 @@ test.describe('parseHtmlBlocks (txt)', () => { }); }); -test.describe('mdToPlainText badge/image stripping', () => { +describe('mdToPlainText badge/image stripping', () => { // The bug: badge alt text was being kept in plainText. Since the rendered // DOM is just an with no text node, the sentence-highlight pattern // matcher couldn't find those words and the WHOLE first-segment match @@ -124,7 +124,7 @@ test.describe('mdToPlainText badge/image stripping', () => { }); }); -test.describe('buildFullDocumentText-style integration (badge-only blocks)', () => { +describe('buildFullDocumentText-style integration (badge-only blocks)', () => { // If a paragraph is composed only of badges, its plainText is empty after // stripping. The reader filters empty plainText out of the TTS source // (`useHtmlDocument#buildFullDocumentText`), so badge blocks don't generate diff --git a/tests/unit/pdf-audiobook-adapter.spec.ts b/tests/unit/pdf-audiobook-adapter.vitest.spec.ts similarity index 96% rename from tests/unit/pdf-audiobook-adapter.spec.ts rename to tests/unit/pdf-audiobook-adapter.vitest.spec.ts index dd110b5..17cebce 100644 --- a/tests/unit/pdf-audiobook-adapter.spec.ts +++ b/tests/unit/pdf-audiobook-adapter.vitest.spec.ts @@ -1,15 +1,15 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { createPdfAudiobookSourceAdapter } from '../../src/lib/client/audiobooks/adapters/pdf'; import type { ParsedPdfDocument } from '../../src/types/parsed-pdf'; import type { DocumentSettings } from '../../src/types/document-settings'; -test.describe('pdf audiobook adapter', () => { +describe('pdf audiobook adapter', () => { test('builds chapters from paragraph titles and filters skipped kinds', async () => { const parsed: ParsedPdfDocument = { schemaVersion: 1, documentId: 'doc-1', parserVersion: 'test', - parsedAt: Date.now(), + parsedAt: 1_700_000_000_000, pages: [ { pageNumber: 1, @@ -77,7 +77,7 @@ test.describe('pdf audiobook adapter', () => { schemaVersion: 1, documentId: 'doc-2', parserVersion: 'test', - parsedAt: Date.now(), + parsedAt: 1_700_000_000_001, pages: [ { pageNumber: 1, diff --git a/tests/unit/pdf-build-page-text.spec.ts b/tests/unit/pdf-build-page-text.vitest.spec.ts similarity index 92% rename from tests/unit/pdf-build-page-text.spec.ts rename to tests/unit/pdf-build-page-text.vitest.spec.ts index 247507c..6604d2d 100644 --- a/tests/unit/pdf-build-page-text.spec.ts +++ b/tests/unit/pdf-build-page-text.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildPageTextFromBlocks } from '../../src/lib/client/pdf-block-text'; import type { ParsedPdfPage } from '../../src/types/parsed-pdf'; -test.describe('buildPageTextFromBlocks', () => { +describe('buildPageTextFromBlocks', () => { test('filters skipped kinds and preserves reading order', () => { const page: ParsedPdfPage = { pageNumber: 1, diff --git a/tests/unit/pdf-force-reparse.spec.ts b/tests/unit/pdf-force-reparse.vitest.spec.ts similarity index 89% rename from tests/unit/pdf-force-reparse.spec.ts rename to tests/unit/pdf-force-reparse.vitest.spec.ts index 95c2ce4..6db012b 100644 --- a/tests/unit/pdf-force-reparse.spec.ts +++ b/tests/unit/pdf-force-reparse.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { FORCE_REPARSE_CONFIRM_MESSAGE, FORCE_REPARSE_CONFIRM_TEXT, @@ -6,7 +6,7 @@ import { isForceReparseDisabled, } from '../../src/lib/client/pdf/force-reparse'; -test.describe('pdf force reparse controls', () => { +describe('pdf force reparse controls', () => { test('disables action while parse is pending or running', () => { expect(isForceReparseDisabled('pending')).toBeTruthy(); expect(isForceReparseDisabled('running')).toBeTruthy(); diff --git a/tests/unit/pdf-merge-text-with-regions.spec.ts b/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts similarity index 59% rename from tests/unit/pdf-merge-text-with-regions.spec.ts rename to tests/unit/pdf-merge-text-with-regions.vitest.spec.ts index 86bd435..135e9c3 100644 --- a/tests/unit/pdf-merge-text-with-regions.spec.ts +++ b/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts @@ -1,7 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { mergeTextWithRegions } from '@openreader/compute-core'; -test.describe('mergeTextWithRegions', () => { +describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { const regions = [ { bbox: [0, 0, 100, 50] as [number, number, number, number], label: 'text' as const }, @@ -20,4 +20,18 @@ test.describe('mergeTextWithRegions', () => { expect(merged[0].text).toBe('hello world'); expect(merged[1].text).toBe('Figure 1.2'); }); + + test('drops text whose centroid is outside every region', () => { + const regions = [ + { bbox: [0, 0, 50, 50] as [number, number, number, number], label: 'text' as const }, + ]; + const textItems = [ + { text: 'inside', x: 10, y: 10, width: 10, height: 8 }, + { text: 'outside', x: 80, y: 80, width: 12, height: 8 }, + ]; + + const merged = mergeTextWithRegions(regions, textItems); + expect(merged).toHaveLength(1); + expect(merged[0].text).toBe('inside'); + }); }); diff --git a/tests/unit/pdf-parse-normalize-text-items.spec.ts b/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts similarity index 69% rename from tests/unit/pdf-parse-normalize-text-items.spec.ts rename to tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts index aac976c..3ceccd0 100644 --- a/tests/unit/pdf-parse-normalize-text-items.spec.ts +++ b/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { normalizeTextItemsForLayout } from '@openreader/compute-core'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; @@ -19,7 +19,7 @@ function makeTextItem( } as unknown as TextItem; } -test.describe('normalizeTextItemsForLayout', () => { +describe('normalizeTextItemsForLayout', () => { test('keeps horizontal body text and drops rotated/skewed margin text', () => { const horizontal = makeTextItem( 'Powered by large language models', @@ -36,4 +36,12 @@ test.describe('normalizeTextItemsForLayout', () => { expect(normalized).toHaveLength(1); expect(normalized[0]?.text).toBe('Powered by large language models'); }); + + test('drops malformed/vertical-only runs so downstream layout planning sees no body text', () => { + const vertical = makeTextItem('Side label', [0, 10, -10, 0, 30, 200]); + const skewed = makeTextItem('Watermark', [10, 5, 2, 10, 200, 500]); + + const normalized = normalizeTextItemsForLayout([vertical, skewed], 800); + expect(normalized).toEqual([]); + }); }); diff --git a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts b/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts similarity index 89% rename from tests/unit/pdf-stitch-cross-page-blocks.spec.ts rename to tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts index 86e6d4a..0aa37e6 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts +++ b/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts @@ -1,6 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { stitchCrossPageBlocks } from '@openreader/compute-core'; import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf'; +import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( id: string, @@ -23,19 +24,13 @@ function makeBlock( } function makeDoc(page1Blocks: ParsedPdfBlock[], page2Blocks: ParsedPdfBlock[]): ParsedPdfDocument { - return { - schemaVersion: 1, - documentId: 'doc', - parserVersion: 'test', - parsedAt: 0, - pages: [ - { pageNumber: 1, width: 100, height: 100, blocks: page1Blocks }, - { pageNumber: 2, width: 100, height: 100, blocks: page2Blocks }, - ], - }; + return makeParsedPdfDocument([ + makeParsedPdfPage(1, page1Blocks), + makeParsedPdfPage(2, page2Blocks), + ]); } -test.describe('stitchCrossPageBlocks', () => { +describe('stitchCrossPageBlocks', () => { test('stitches paragraph continuation across footer/header noise', () => { const doc = makeDoc( [ diff --git a/tests/unit/pdf-tts-planning.spec.ts b/tests/unit/pdf-tts-planning.vitest.spec.ts similarity index 97% rename from tests/unit/pdf-tts-planning.spec.ts rename to tests/unit/pdf-tts-planning.vitest.spec.ts index 1066d7b..c9314dd 100644 --- a/tests/unit/pdf-tts-planning.spec.ts +++ b/tests/unit/pdf-tts-planning.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildPdfPageSourceUnits, buildPdfPrefetchPayload } from '../../src/lib/client/pdf-tts-planning'; import type { ParsedPdfPage } from '../../src/types/parsed-pdf'; @@ -39,7 +39,7 @@ function buildPage(pageNumber: number): ParsedPdfPage { }; } -test.describe('pdf tts planning helpers', () => { +describe('pdf tts planning helpers', () => { test('buildPdfPageSourceUnits uses parsed blocks, honors skip kinds, and pins locator page', () => { const page = buildPage(2); const units = buildPdfPageSourceUnits(page, 2, ['header']); diff --git a/tests/unit/support/document-fixtures.ts b/tests/unit/support/document-fixtures.ts new file mode 100644 index 0000000..aefe931 --- /dev/null +++ b/tests/unit/support/document-fixtures.ts @@ -0,0 +1,54 @@ +import type { BaseDocument } from '../../../src/types/documents'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../src/types/parsed-pdf'; + +export function makeBaseDocument(overrides: Partial = {}): BaseDocument { + return { + id: 'doc-1', + name: 'document.pdf', + size: 1_024, + lastModified: 1_700_000_000_000, + type: 'pdf', + ...overrides, + }; +} + +export function makeParsedPdfBlock(input: { + id: string; + kind: ParsedPdfBlockKind; + text: string; + page: number; + readingOrder: number; +}): ParsedPdfBlock { + return { + id: input.id, + kind: input.kind, + text: input.text, + fragments: [ + { + page: input.page, + bbox: [0, 0, 100, 10], + text: input.text, + readingOrder: input.readingOrder, + }, + ], + }; +} + +export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage { + return { + pageNumber, + width: 800, + height: 1_200, + blocks, + }; +} + +export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument { + return { + schemaVersion: 1, + documentId: 'doc-fixture', + parserVersion: 'test', + parsedAt: 1_700_000_000_000, + pages, + }; +} diff --git a/vitest.config.ts b/vitest.config.ts index fe7ca8b..c3e8a47 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,7 +2,17 @@ import { defineConfig } from 'vitest/config'; import { fileURLToPath } from 'node:url'; const srcDir = fileURLToPath(new URL('./src/', import.meta.url)); +const computeCoreIndex = fileURLToPath(new URL('./compute/core/src/index.ts', import.meta.url)); +const computeCoreApiContracts = fileURLToPath(new URL('./compute/core/src/api-contracts/index.ts', import.meta.url)); +const computeCoreControlPlane = fileURLToPath(new URL('./compute/core/src/control-plane/index.ts', import.meta.url)); +const computeCoreLocalRuntime = fileURLToPath(new URL('./compute/core/src/local-runtime.ts', import.meta.url)); +const computeCoreTypes = fileURLToPath(new URL('./compute/core/src/types/index.ts', import.meta.url)); const alias = [ + { find: /^@openreader\/compute-core$/, replacement: computeCoreIndex }, + { find: /^@openreader\/compute-core\/api-contracts$/, replacement: computeCoreApiContracts }, + { find: /^@openreader\/compute-core\/control-plane$/, replacement: computeCoreControlPlane }, + { find: /^@openreader\/compute-core\/local-runtime$/, replacement: computeCoreLocalRuntime }, + { find: /^@openreader\/compute-core\/types$/, replacement: computeCoreTypes }, { find: /^@\//, replacement: `${srcDir}` }, { find: '@', replacement: srcDir }, ]; From 330747754879e551e0ca717e7eb0cc465406418f Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 11:17:18 -0600 Subject: [PATCH 08/16] test(unit): complete vitest migration for remaining unit suites --- ... admin-providers-validation.vitest.spec.ts} | 4 ++-- ...s => admin-tts-instructions.vitest.spec.ts} | 4 ++-- ....spec.ts => audiobook-scope.vitest.spec.ts} | 4 ++-- ...ec.ts => audiobook-settings.vitest.spec.ts} | 4 ++-- ...iobook.spec.ts => audiobook.vitest.spec.ts} | 8 ++++---- ....ts => audiobooks-blobstore.vitest.spec.ts} | 4 ++-- ....spec.ts => changelog-check.vitest.spec.ts} | 4 ++-- ...ngelog.spec.ts => changelog.vitest.spec.ts} | 4 ++-- ...=> document-previews-render.vitest.spec.ts} | 4 ++-- ...c.ts => epub-audiobook-hook.vitest.spec.ts} | 4 ++-- ...=> epub-location-controller.vitest.spec.ts} | 4 ++-- ...s => epub-spine-coordinates.vitest.spec.ts} | 18 +++++++++--------- ...c.ts => epub-word-highlight.vitest.spec.ts} | 4 ++-- ...-grid.spec.ts => icons-grid.vitest.spec.ts} | 4 ++-- .../{kokoro.spec.ts => kokoro.vitest.spec.ts} | 4 ++-- tests/unit/{nlp.spec.ts => nlp.vitest.spec.ts} | 10 +++++----- ....spec.ts => onboarding-flow.vitest.spec.ts} | 6 +++--- ...spec.ts => onboarding-state.vitest.spec.ts} | 4 ++-- .../{sha256.spec.ts => sha256.vitest.spec.ts} | 4 ++-- ... => transfer-user-documents.vitest.spec.ts} | 4 ++-- ....ts => tts-audio-warm-cache.vitest.spec.ts} | 4 ++-- ...spec.ts => tts-epub-handoff.vitest.spec.ts} | 4 ++-- ...spec.ts => tts-epub-preload.vitest.spec.ts} | 4 ++-- ...ate.spec.ts => tts-generate.vitest.spec.ts} | 4 ++-- ...ator.spec.ts => tts-locator.vitest.spec.ts} | 12 ++++++------ ....ts => tts-provider-catalog.vitest.spec.ts} | 6 +++--- ...s => tts-segment-audio-urls.vitest.spec.ts} | 4 ++-- ...=> tts-segment-plan-leading.vitest.spec.ts} | 4 ++-- ...spec.ts => tts-segment-plan.vitest.spec.ts} | 6 +++--- ...=> tts-segment-retry-policy.vitest.spec.ts} | 4 ++-- ...=> tts-segments-audio-cache.vitest.spec.ts} | 4 ++-- ...ts => tts-segments-manifest.vitest.spec.ts} | 10 +++------- ...nts.spec.ts => tts-segments.vitest.spec.ts} | 4 ++-- ...s-settings-admin-view-model.vitest.spec.ts} | 4 ++-- ...eme.spec.ts => walker-theme.vitest.spec.ts} | 4 ++-- ...> whisper-alignment-mapping.vitest.spec.ts} | 4 ++-- ...spec.ts => whisper-spectral.vitest.spec.ts} | 4 ++-- ...=> whisper-token-timestamps.vitest.spec.ts} | 4 ++-- 38 files changed, 96 insertions(+), 100 deletions(-) rename tests/unit/{admin-providers-validation.spec.ts => admin-providers-validation.vitest.spec.ts} (97%) rename tests/unit/{admin-tts-instructions.spec.ts => admin-tts-instructions.vitest.spec.ts} (92%) rename tests/unit/{audiobook-scope.spec.ts => audiobook-scope.vitest.spec.ts} (93%) rename tests/unit/{audiobook-settings.spec.ts => audiobook-settings.vitest.spec.ts} (96%) rename tests/unit/{audiobook.spec.ts => audiobook.vitest.spec.ts} (95%) rename tests/unit/{audiobooks-blobstore.spec.ts => audiobooks-blobstore.vitest.spec.ts} (95%) rename tests/unit/{changelog-check.spec.ts => changelog-check.vitest.spec.ts} (95%) rename tests/unit/{changelog.spec.ts => changelog.vitest.spec.ts} (96%) rename tests/unit/{document-previews-render.spec.ts => document-previews-render.vitest.spec.ts} (90%) rename tests/unit/{epub-audiobook-hook.spec.ts => epub-audiobook-hook.vitest.spec.ts} (94%) rename tests/unit/{epub-location-controller.spec.ts => epub-location-controller.vitest.spec.ts} (92%) rename tests/unit/{epub-spine-coordinates.spec.ts => epub-spine-coordinates.vitest.spec.ts} (97%) rename tests/unit/{epub-word-highlight.spec.ts => epub-word-highlight.vitest.spec.ts} (95%) rename tests/unit/{icons-grid.spec.ts => icons-grid.vitest.spec.ts} (91%) rename tests/unit/{kokoro.spec.ts => kokoro.vitest.spec.ts} (89%) rename tests/unit/{nlp.spec.ts => nlp.vitest.spec.ts} (96%) rename tests/unit/{onboarding-flow.spec.ts => onboarding-flow.vitest.spec.ts} (94%) rename tests/unit/{onboarding-state.spec.ts => onboarding-state.vitest.spec.ts} (90%) rename tests/unit/{sha256.spec.ts => sha256.vitest.spec.ts} (91%) rename tests/unit/{transfer-user-documents.spec.ts => transfer-user-documents.vitest.spec.ts} (95%) rename tests/unit/{tts-audio-warm-cache.spec.ts => tts-audio-warm-cache.vitest.spec.ts} (97%) rename tests/unit/{tts-epub-handoff.spec.ts => tts-epub-handoff.vitest.spec.ts} (97%) rename tests/unit/{tts-epub-preload.spec.ts => tts-epub-preload.vitest.spec.ts} (94%) rename tests/unit/{tts-generate.spec.ts => tts-generate.vitest.spec.ts} (92%) rename tests/unit/{tts-locator.spec.ts => tts-locator.vitest.spec.ts} (96%) rename tests/unit/{tts-provider-catalog.spec.ts => tts-provider-catalog.vitest.spec.ts} (99%) rename tests/unit/{tts-segment-audio-urls.spec.ts => tts-segment-audio-urls.vitest.spec.ts} (94%) rename tests/unit/{tts-segment-plan-leading.spec.ts => tts-segment-plan-leading.vitest.spec.ts} (98%) rename tests/unit/{tts-segment-plan.spec.ts => tts-segment-plan.vitest.spec.ts} (98%) rename tests/unit/{tts-segment-retry-policy.spec.ts => tts-segment-retry-policy.vitest.spec.ts} (93%) rename tests/unit/{tts-segments-audio-cache.spec.ts => tts-segments-audio-cache.vitest.spec.ts} (91%) rename tests/unit/{tts-segments-manifest.spec.ts => tts-segments-manifest.vitest.spec.ts} (94%) rename tests/unit/{tts-segments.spec.ts => tts-segments.vitest.spec.ts} (98%) rename tests/unit/{tts-settings-admin-view-model.spec.ts => tts-settings-admin-view-model.vitest.spec.ts} (97%) rename tests/unit/{walker-theme.spec.ts => walker-theme.vitest.spec.ts} (93%) rename tests/unit/{whisper-alignment-mapping.spec.ts => whisper-alignment-mapping.vitest.spec.ts} (88%) rename tests/unit/{whisper-spectral.spec.ts => whisper-spectral.vitest.spec.ts} (91%) rename tests/unit/{whisper-token-timestamps.spec.ts => whisper-token-timestamps.vitest.spec.ts} (95%) diff --git a/tests/unit/admin-providers-validation.spec.ts b/tests/unit/admin-providers-validation.vitest.spec.ts similarity index 97% rename from tests/unit/admin-providers-validation.spec.ts rename to tests/unit/admin-providers-validation.vitest.spec.ts index ca3b68b..e977cc7 100644 --- a/tests/unit/admin-providers-validation.spec.ts +++ b/tests/unit/admin-providers-validation.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { inArray } from 'drizzle-orm'; import { db } from '../../src/db'; import { adminProviders } from '../../src/db/schema'; @@ -12,7 +12,7 @@ import { type AdminProviderRecord, } from '../../src/lib/server/admin/providers'; -test.describe('admin provider validation', () => { +describe('admin provider validation', () => { test('normalizes valid slugs to lowercase', () => { expect(validateSlug('My-Provider-1')).toBe('my-provider-1'); }); diff --git a/tests/unit/admin-tts-instructions.spec.ts b/tests/unit/admin-tts-instructions.vitest.spec.ts similarity index 92% rename from tests/unit/admin-tts-instructions.spec.ts rename to tests/unit/admin-tts-instructions.vitest.spec.ts index 0a7198e..11e06e8 100644 --- a/tests/unit/admin-tts-instructions.spec.ts +++ b/tests/unit/admin-tts-instructions.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { resolveEffectiveTtsInstructions } from '../../src/lib/server/admin/tts-instructions'; -test.describe('resolveEffectiveTtsInstructions', () => { +describe('resolveEffectiveTtsInstructions', () => { test('uses explicit request instructions when model supports them', () => { const out = resolveEffectiveTtsInstructions({ model: 'gpt-4o-mini-tts', diff --git a/tests/unit/audiobook-scope.spec.ts b/tests/unit/audiobook-scope.vitest.spec.ts similarity index 93% rename from tests/unit/audiobook-scope.spec.ts rename to tests/unit/audiobook-scope.vitest.spec.ts index b73ba40..ca9705b 100644 --- a/tests/unit/audiobook-scope.spec.ts +++ b/tests/unit/audiobook-scope.vitest.spec.ts @@ -1,7 +1,7 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '../../src/lib/server/audiobooks/user-scope'; -test.describe('audiobook scope selection', () => { +describe('audiobook scope selection', () => { test('uses only unclaimed scope when auth is disabled', () => { const result = buildAllowedAudiobookUserIds(false, null, 'unclaimed::ns'); expect(result.preferredUserId).toBe('unclaimed::ns'); diff --git a/tests/unit/audiobook-settings.spec.ts b/tests/unit/audiobook-settings.vitest.spec.ts similarity index 96% rename from tests/unit/audiobook-settings.spec.ts rename to tests/unit/audiobook-settings.vitest.spec.ts index af3c788..1cc371c 100644 --- a/tests/unit/audiobook-settings.spec.ts +++ b/tests/unit/audiobook-settings.vitest.spec.ts @@ -1,10 +1,10 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { canonicalizeAudiobookSettingsForRuntime, coerceAudiobookGenerationSettings, } from '../../src/lib/server/audiobooks/settings'; -test.describe('coerceAudiobookGenerationSettings', () => { +describe('coerceAudiobookGenerationSettings', () => { test('accepts current metadata shape without migration', () => { const result = coerceAudiobookGenerationSettings({ providerRef: 'openai', diff --git a/tests/unit/audiobook.spec.ts b/tests/unit/audiobook.vitest.spec.ts similarity index 95% rename from tests/unit/audiobook.spec.ts rename to tests/unit/audiobook.vitest.spec.ts index 53417a2..0533385 100644 --- a/tests/unit/audiobook.spec.ts +++ b/tests/unit/audiobook.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { escapeFFMetadata, encodeChapterTitleTag, @@ -7,7 +7,7 @@ import { decodeChapterFileName } from '../../src/lib/server/audiobooks/chapters'; -test.describe('escapeFFMetadata', () => { +describe('escapeFFMetadata', () => { test('escapes special characters correctly', () => { const input = 'Title with = ; # and backslash \\'; // Expected: Equal -> \=, Semicolon -> \;, Hash -> \#, Backslash -> \\ @@ -37,7 +37,7 @@ test.describe('escapeFFMetadata', () => { }); }); -test.describe('Title Tags', () => { +describe('Title Tags', () => { test('encodeChapterTitleTag formats correctly', () => { expect(encodeChapterTitleTag(0, 'Intro')).toBe('0001 - Intro'); expect(encodeChapterTitleTag(9, 'Chapter Ten')).toBe('0010 - Chapter Ten'); @@ -63,7 +63,7 @@ test.describe('Title Tags', () => { }); }); -test.describe('Chapter File Names', () => { +describe('Chapter File Names', () => { test('encodeChapterFileName formats correctly', () => { expect(encodeChapterFileName(0, 'Intro', 'mp3')).toBe('0001__Intro.mp3'); expect(encodeChapterFileName(1, 'Part 2', 'm4b')).toBe('0002__Part%202.m4b'); diff --git a/tests/unit/audiobooks-blobstore.spec.ts b/tests/unit/audiobooks-blobstore.vitest.spec.ts similarity index 95% rename from tests/unit/audiobooks-blobstore.spec.ts rename to tests/unit/audiobooks-blobstore.vitest.spec.ts index 198b2bf..5f2c917 100644 --- a/tests/unit/audiobooks-blobstore.spec.ts +++ b/tests/unit/audiobooks-blobstore.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { audiobookKey, audiobookPrefix, @@ -14,7 +14,7 @@ function configureS3Env() { process.env.S3_PREFIX = 'openreader-test'; } -test.describe('audiobooks-blobstore', () => { +describe('audiobooks-blobstore', () => { test.beforeAll(() => { configureS3Env(); }); diff --git a/tests/unit/changelog-check.spec.ts b/tests/unit/changelog-check.vitest.spec.ts similarity index 95% rename from tests/unit/changelog-check.spec.ts rename to tests/unit/changelog-check.vitest.spec.ts index f95bcbc..19d8642 100644 --- a/tests/unit/changelog-check.spec.ts +++ b/tests/unit/changelog-check.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { scheduleChangelogCheck } from '../../src/lib/client/changelog-check'; @@ -6,7 +6,7 @@ function wait(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -test.describe('changelog check scheduling', () => { +describe('changelog check scheduling', () => { test('cancels throwaway mount and runs exactly once on remount', async () => { const completedRef = { current: null as string | null }; const inFlightRef = { current: null as string | null }; diff --git a/tests/unit/changelog.spec.ts b/tests/unit/changelog.vitest.spec.ts similarity index 96% rename from tests/unit/changelog.spec.ts rename to tests/unit/changelog.vitest.spec.ts index 02117d9..3d56744 100644 --- a/tests/unit/changelog.spec.ts +++ b/tests/unit/changelog.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { findCurrentVersionIndex, @@ -10,7 +10,7 @@ import { type ChangelogManifestEntry, } from '../../src/lib/shared/changelog'; -test.describe('changelog utilities', () => { +describe('changelog utilities', () => { test('normalizes version strings with and without v prefix', () => { expect(normalizeVersion('v2.2.0')).toBe('2.2.0'); expect(normalizeVersion('2.2.0')).toBe('2.2.0'); diff --git a/tests/unit/document-previews-render.spec.ts b/tests/unit/document-previews-render.vitest.spec.ts similarity index 90% rename from tests/unit/document-previews-render.spec.ts rename to tests/unit/document-previews-render.vitest.spec.ts index 85f6a31..c82ed75 100644 --- a/tests/unit/document-previews-render.spec.ts +++ b/tests/unit/document-previews-render.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import path from 'path'; import { readFile } from 'fs/promises'; import { @@ -6,7 +6,7 @@ import { renderPdfFirstPageToJpeg, } from '../../src/lib/server/documents/previews-render'; -test.describe('document-previews-render', () => { +describe('document-previews-render', () => { test('renders first PDF page to JPEG preview', async () => { const pdfPath = path.join(process.cwd(), 'tests/files/sample.pdf'); const bytes = await readFile(pdfPath); diff --git a/tests/unit/epub-audiobook-hook.spec.ts b/tests/unit/epub-audiobook-hook.vitest.spec.ts similarity index 94% rename from tests/unit/epub-audiobook-hook.spec.ts rename to tests/unit/epub-audiobook-hook.vitest.spec.ts index 8efd21b..309eb4e 100644 --- a/tests/unit/epub-audiobook-hook.spec.ts +++ b/tests/unit/epub-audiobook-hook.vitest.spec.ts @@ -1,11 +1,11 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { filterNonEmptySpineTextEntries, resolveLoadedSpineSectionDocument, } from '../../src/hooks/epub/useEPUBAudiobook'; -test.describe('EPUB audiobook hook helpers', () => { +describe('EPUB audiobook hook helpers', () => { test('resolveLoadedSpineSectionDocument prefers ownerDocument when available', () => { const ownerDocument = { body: { textContent: 'from ownerDocument' } } as unknown as Document; const loaded = { ownerDocument } as unknown; diff --git a/tests/unit/epub-location-controller.spec.ts b/tests/unit/epub-location-controller.vitest.spec.ts similarity index 92% rename from tests/unit/epub-location-controller.spec.ts rename to tests/unit/epub-location-controller.vitest.spec.ts index ca165a7..c784eab 100644 --- a/tests/unit/epub-location-controller.spec.ts +++ b/tests/unit/epub-location-controller.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { isDirectionalEpubLocation, @@ -6,7 +6,7 @@ import { shouldPersistEpubLocation, } from '../../src/hooks/epub/useEPUBLocationController'; -test.describe('EPUB location controller helpers', () => { +describe('EPUB location controller helpers', () => { test('detects directional locations', () => { expect(isDirectionalEpubLocation('next')).toBe(true); expect(isDirectionalEpubLocation('prev')).toBe(true); diff --git a/tests/unit/epub-spine-coordinates.spec.ts b/tests/unit/epub-spine-coordinates.vitest.spec.ts similarity index 97% rename from tests/unit/epub-spine-coordinates.spec.ts rename to tests/unit/epub-spine-coordinates.vitest.spec.ts index 2d8b9a5..eca51e3 100644 --- a/tests/unit/epub-spine-coordinates.spec.ts +++ b/tests/unit/epub-spine-coordinates.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import type { Book } from 'epubjs'; import { buildEpubLocator, @@ -71,7 +71,7 @@ function makeFakeBook(items: Array<{ index: number; href: string; cfiBase: strin return { book, sections }; } -test.describe('findSegmentOffset', () => { +describe('findSegmentOffset', () => { test('finds an offset for matching text', () => { const spineText = 'Hello world. This is a test paragraph for offsets.'; const offset = findSegmentOffset(spineText, 'this is a test'); @@ -98,7 +98,7 @@ test.describe('findSegmentOffset', () => { }); }); -test.describe('resolveSpineFromCfi', () => { +describe('resolveSpineFromCfi', () => { test('returns spine identity for a CFI inside a known spine', () => { const { book } = makeFakeBook([ { index: 2, href: 'OEBPS/ch02.xhtml', cfiBase: '/6/4', text: 'chapter 2 contents' }, @@ -125,7 +125,7 @@ test.describe('resolveSpineFromCfi', () => { }); }); -test.describe('buildEpubLocator', () => { +describe('buildEpubLocator', () => { test('produces a stable locator with charOffset within the spine item', async () => { const { book } = makeFakeBook([ { @@ -175,7 +175,7 @@ test.describe('buildEpubLocator', () => { }); }); -test.describe('buildEpubLocatorFromChunk', () => { +describe('buildEpubLocatorFromChunk', () => { test('uses the chunk anchor offset as a hint to disambiguate repeated text', () => { const spineText = 'foo bar foo bar foo bar'; const anchorEarly = { @@ -230,7 +230,7 @@ test.describe('buildEpubLocatorFromChunk', () => { }); }); -test.describe('buildEpubLocator anchored-search regression', () => { +describe('buildEpubLocator anchored-search regression', () => { // Pins the contract for the server-side resolver path: when called with // a `chunkText` argument representing the current rendered page, the // returned locator's `charOffset` is **at or after** the chunk's position @@ -261,7 +261,7 @@ test.describe('buildEpubLocator anchored-search regression', () => { }); }); -test.describe('findSegmentOffset fallback contract', () => { +describe('findSegmentOffset fallback contract', () => { // These tests pin the documented behavior of `findSegmentOffset`. The // from-start fallback is correct for single-shot lookups but **wrong** in // a monotonic per-sentence walk — `resolveMonotonicSentenceOffsets` exists @@ -278,7 +278,7 @@ test.describe('findSegmentOffset fallback contract', () => { }); }); -test.describe('resolveMonotonicSentenceOffsets', () => { +describe('resolveMonotonicSentenceOffsets', () => { test('returns monotonically non-decreasing offsets for sentences in order', () => { const spineText = 'The cat sat on the mat. The dog barked loudly. Then it was quiet.'; const sentences = [ @@ -347,7 +347,7 @@ test.describe('resolveMonotonicSentenceOffsets', () => { }); }); -test.describe('getSpineItemPlainText shape handling', () => { +describe('getSpineItemPlainText shape handling', () => { // Regression: epubjs's `Section.load()` resolves to the spine item's // `` Element (NOT a Document). Reading `loaded.body` directly was // returning undefined for every spine item, so spineText came back '' diff --git a/tests/unit/epub-word-highlight.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts similarity index 95% rename from tests/unit/epub-word-highlight.spec.ts rename to tests/unit/epub-word-highlight.vitest.spec.ts index dc9720a..8899dc8 100644 --- a/tests/unit/epub-word-highlight.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildMonotonicWordToTokenMap, @@ -27,7 +27,7 @@ const alignmentWords = (words: string[]): TTSSentenceAlignment['words'] => charEnd: word.length, })); -test.describe('EPUB word highlight mapping', () => { +describe('EPUB word highlight mapping', () => { test('tokenizes canonical segment words with source offsets', () => { const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); diff --git a/tests/unit/icons-grid.spec.ts b/tests/unit/icons-grid.vitest.spec.ts similarity index 91% rename from tests/unit/icons-grid.spec.ts rename to tests/unit/icons-grid.vitest.spec.ts index 5663c47..c21f95b 100644 --- a/tests/unit/icons-grid.spec.ts +++ b/tests/unit/icons-grid.vitest.spec.ts @@ -1,7 +1,7 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { iconsGridStyle, maxColumnsForIconGrid } from '../../src/components/doclist/views/iconsGrid'; -test.describe('icons grid layout', () => { +describe('icons grid layout', () => { test('calculates max columns from width and icon size', () => { expect(maxColumnsForIconGrid('md', 136)).toBe(1); expect(maxColumnsForIconGrid('md', 300)).toBe(2); diff --git a/tests/unit/kokoro.spec.ts b/tests/unit/kokoro.vitest.spec.ts similarity index 89% rename from tests/unit/kokoro.spec.ts rename to tests/unit/kokoro.vitest.spec.ts index 3860c1f..5f1950a 100644 --- a/tests/unit/kokoro.spec.ts +++ b/tests/unit/kokoro.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { getMaxVoicesForProvider } from '../../src/lib/shared/kokoro'; -test.describe('kokoro voice limits', () => { +describe('kokoro voice limits', () => { test('keeps Replicate single-voice even for Kokoro models', () => { expect(getMaxVoicesForProvider('replicate', 'kokoro')).toBe(1); expect(getMaxVoicesForProvider('replicate', 'hexgrad/Kokoro-82M')).toBe(1); diff --git a/tests/unit/nlp.spec.ts b/tests/unit/nlp.vitest.spec.ts similarity index 96% rename from tests/unit/nlp.spec.ts rename to tests/unit/nlp.vitest.spec.ts index ce65ef0..4753b20 100644 --- a/tests/unit/nlp.spec.ts +++ b/tests/unit/nlp.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { preprocessSentenceForAudio, splitTextToTtsBlocks, @@ -18,7 +18,7 @@ const expectNormalizedBlocks = (blocks: string[], maxLen = Number.POSITIVE_INFIN } }; -test.describe('preprocessSentenceForAudio', () => { +describe('preprocessSentenceForAudio', () => { test('normalizes common extraction artifacts', () => { const cases: Array<{ input: string; expected: string }> = [ { @@ -45,7 +45,7 @@ test.describe('preprocessSentenceForAudio', () => { }); }); -test.describe('splitTextToTtsBlocks (PDF-oriented)', () => { +describe('splitTextToTtsBlocks (PDF-oriented)', () => { test('returns [] for empty input', () => { expect(splitTextToTtsBlocks('')).toEqual([]); expect(splitTextToTtsBlocks(' ')).toEqual([]); @@ -160,7 +160,7 @@ test.describe('splitTextToTtsBlocks (PDF-oriented)', () => { }); }); -test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { +describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { test('returns [] for empty input', () => { expect(splitTextToTtsBlocksEPUB('')).toEqual([]); expect(splitTextToTtsBlocksEPUB(' ')).toEqual([]); @@ -193,7 +193,7 @@ test.describe('splitTextToTtsBlocksEPUB (highlight-friendly)', () => { }); }); -test.describe('normalizeTextForTts', () => { +describe('normalizeTextForTts', () => { test('returns a single normalized string without newlines', () => { const input = 'Hello.\nWorld.\n\nNext paragraph.'; const normalized = normalizeTextForTts(input); diff --git a/tests/unit/onboarding-flow.spec.ts b/tests/unit/onboarding-flow.vitest.spec.ts similarity index 94% rename from tests/unit/onboarding-flow.spec.ts rename to tests/unit/onboarding-flow.vitest.spec.ts index 9533aa8..27eaaa6 100644 --- a/tests/unit/onboarding-flow.spec.ts +++ b/tests/unit/onboarding-flow.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { createCoalescedAsyncRunner, resolveNextOnboardingStep } from '../../src/lib/client/onboarding-flow'; -test.describe('onboarding flow resolver', () => { +describe('onboarding flow resolver', () => { test('resolves deterministic order with privacy first', () => { const step = resolveNextOnboardingStep({ privacyRequired: true, @@ -78,7 +78,7 @@ test.describe('onboarding flow resolver', () => { }); }); -test.describe('coalesced onboarding runner', () => { +describe('coalesced onboarding runner', () => { test('coalesces concurrent triggers into one extra rerun', async () => { let runs = 0; let nestedRequested = false; diff --git a/tests/unit/onboarding-state.spec.ts b/tests/unit/onboarding-state.vitest.spec.ts similarity index 90% rename from tests/unit/onboarding-state.spec.ts rename to tests/unit/onboarding-state.vitest.spec.ts index 9379c2a..c457b92 100644 --- a/tests/unit/onboarding-state.spec.ts +++ b/tests/unit/onboarding-state.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { ONBOARDING_STATE_REGISTRY } from '../../src/lib/shared/onboarding-state'; import { SYNCED_PREFERENCE_KEYS } from '../../src/types/user-state'; -test.describe('onboarding state storage scopes', () => { +describe('onboarding state storage scopes', () => { test('keeps local onboarding flags out of synced server preferences', () => { expect(ONBOARDING_STATE_REGISTRY.privacyAccepted.scope).toBe('local-dexie'); expect(ONBOARDING_STATE_REGISTRY.firstVisitSettingsOpened.scope).toBe('local-dexie'); diff --git a/tests/unit/sha256.spec.ts b/tests/unit/sha256.vitest.spec.ts similarity index 91% rename from tests/unit/sha256.spec.ts rename to tests/unit/sha256.vitest.spec.ts index 64796a3..1a44fec 100644 --- a/tests/unit/sha256.spec.ts +++ b/tests/unit/sha256.vitest.spec.ts @@ -1,10 +1,10 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { sha256HexSoftwareFromBytes, sha256HexFromString } from '../../src/lib/client/sha256'; -test.describe('SHA256 Software Implementation', () => { +describe('SHA256 Software Implementation', () => { // Known test vectors // "" -> e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 // "abc" -> ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad diff --git a/tests/unit/transfer-user-documents.spec.ts b/tests/unit/transfer-user-documents.vitest.spec.ts similarity index 95% rename from tests/unit/transfer-user-documents.spec.ts rename to tests/unit/transfer-user-documents.vitest.spec.ts index ca58e28..f1c0bed 100644 --- a/tests/unit/transfer-user-documents.spec.ts +++ b/tests/unit/transfer-user-documents.vitest.spec.ts @@ -1,11 +1,11 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; import { eq } from 'drizzle-orm'; import { documents } from '../../src/db/schema_sqlite'; import { transferUserDocuments } from '../../src/lib/server/user/claim-data'; -test.describe('transferUserDocuments', () => { +describe('transferUserDocuments', () => { test('moves document rows to new user without PK conflicts', async () => { process.env.BASE_URL = 'http://localhost:3003'; process.env.AUTH_SECRET = 'test-secret'; diff --git a/tests/unit/tts-audio-warm-cache.spec.ts b/tests/unit/tts-audio-warm-cache.vitest.spec.ts similarity index 97% rename from tests/unit/tts-audio-warm-cache.spec.ts rename to tests/unit/tts-audio-warm-cache.vitest.spec.ts index 06a5d24..4f38c73 100644 --- a/tests/unit/tts-audio-warm-cache.spec.ts +++ b/tests/unit/tts-audio-warm-cache.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { releaseWarmAudio, upsertWarmAudioEntry, @@ -23,7 +23,7 @@ function createTrackedAudio() { return { audio, calls }; } -test.describe('tts warm audio cache helpers', () => { +describe('tts warm audio cache helpers', () => { test('releaseWarmAudio clears source and invokes teardown methods', () => { const tracked = createTrackedAudio(); releaseWarmAudio(tracked.audio); diff --git a/tests/unit/tts-epub-handoff.spec.ts b/tests/unit/tts-epub-handoff.vitest.spec.ts similarity index 97% rename from tests/unit/tts-epub-handoff.spec.ts rename to tests/unit/tts-epub-handoff.vitest.spec.ts index 143e6e1..fa5fb82 100644 --- a/tests/unit/tts-epub-handoff.spec.ts +++ b/tests/unit/tts-epub-handoff.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { completedEpubBoundarySegment, @@ -23,7 +23,7 @@ const segment = ( ...overrides, }); -test.describe('EPUB boundary handoff', () => { +describe('EPUB boundary handoff', () => { test('records only completed segments that span a source boundary', () => { expect(completedEpubBoundarySegment(segment('same page'))).toBeNull(); diff --git a/tests/unit/tts-epub-preload.spec.ts b/tests/unit/tts-epub-preload.vitest.spec.ts similarity index 94% rename from tests/unit/tts-epub-preload.spec.ts rename to tests/unit/tts-epub-preload.vitest.spec.ts index d780e51..f74dd39 100644 --- a/tests/unit/tts-epub-preload.spec.ts +++ b/tests/unit/tts-epub-preload.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import type { CanonicalTtsSourceUnit } from '../../src/lib/shared/tts-segment-plan'; import { @@ -6,7 +6,7 @@ import { selectUpcomingWalkerItems, } from '../../src/lib/client/epub/tts-epub-preload'; -test.describe('EPUB walker preload helpers', () => { +describe('EPUB walker preload helpers', () => { test('selectUpcomingWalkerItems honors depth as current + (depth-1) upcoming', () => { const items = [ { cfi: 'epubcfi(/6/2[;s=a]!/4/2)' }, // same as current after normalization diff --git a/tests/unit/tts-generate.spec.ts b/tests/unit/tts-generate.vitest.spec.ts similarity index 92% rename from tests/unit/tts-generate.spec.ts rename to tests/unit/tts-generate.vitest.spec.ts index 364f6f2..a601952 100644 --- a/tests/unit/tts-generate.spec.ts +++ b/tests/unit/tts-generate.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { extractReplicateAudioUrl } from '../../src/lib/server/tts/generate'; -test.describe('replicate output URL extraction', () => { +describe('replicate output URL extraction', () => { test('returns direct URL string output', () => { expect(extractReplicateAudioUrl('https://replicate.delivery/audio.mp3')).toBe( 'https://replicate.delivery/audio.mp3' diff --git a/tests/unit/tts-locator.spec.ts b/tests/unit/tts-locator.vitest.spec.ts similarity index 96% rename from tests/unit/tts-locator.spec.ts rename to tests/unit/tts-locator.vitest.spec.ts index 19e0eee..6e8b32c 100644 --- a/tests/unit/tts-locator.spec.ts +++ b/tests/unit/tts-locator.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { compareSegmentLocators, locatorGroupKey, @@ -25,7 +25,7 @@ const epubLocator = ( ...(cfi ? { cfi } : {}), }); -test.describe('locatorGroupKey', () => { +describe('locatorGroupKey', () => { test('groups stable EPUB locators by spine index and href', () => { expect(locatorGroupKey(epubLocator('OEBPS/ch02.xhtml', 2, 0))).toBe('epub:2:OEBPS/ch02.xhtml'); // Different charOffset, same spine → same group (sidebar bucket is chapter-sized). @@ -55,7 +55,7 @@ test.describe('locatorGroupKey', () => { }); }); -test.describe('locatorIdentityKey', () => { +describe('locatorIdentityKey', () => { test('two EPUB rows in the same chapter at different charOffsets get distinct identity keys', () => { // This is the bug that caused page-N rows to swallow page-(N+1) rows in // the server-side manifest aggregator: the previous code keyed by the @@ -82,7 +82,7 @@ test.describe('locatorIdentityKey', () => { }); }); -test.describe('compareSegmentLocators', () => { +describe('compareSegmentLocators', () => { test('orders EPUB rows by spineIndex then charOffset numerically', () => { // Spine 10 must come AFTER spine 2 — a lexicographic compare would // wrongly place "10" before "2". @@ -137,7 +137,7 @@ test.describe('compareSegmentLocators', () => { }); }); -test.describe('type guards', () => { +describe('type guards', () => { test('isStableEpubLocator requires all spine fields', () => { expect(isStableEpubLocator(epubLocator('a.xhtml', 0, 0))).toBe(true); expect(isStableEpubLocator({ readerType: 'epub', spineHref: 'a.xhtml' })).toBe(false); @@ -159,7 +159,7 @@ test.describe('type guards', () => { }); }); -test.describe('normalizeEpubLocationToken', () => { +describe('normalizeEpubLocationToken', () => { test('strips whitespace and step markers but preserves identity', () => { const a = normalizeEpubLocationToken('epubcfi(/6/8!/4:0)'); const b = normalizeEpubLocationToken(' epubcfi(/6/8[;s=a]!/4:0) '); diff --git a/tests/unit/tts-provider-catalog.spec.ts b/tests/unit/tts-provider-catalog.vitest.spec.ts similarity index 99% rename from tests/unit/tts-provider-catalog.spec.ts rename to tests/unit/tts-provider-catalog.vitest.spec.ts index 3c40372..334ea16 100644 --- a/tests/unit/tts-provider-catalog.spec.ts +++ b/tests/unit/tts-provider-catalog.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { REPLICATE_KOKORO_82M_VERSIONED_MODEL, @@ -13,7 +13,7 @@ import { resolveReplicateVoiceInputKey, resolveVoices } from '../../src/lib/serv import { applyConfigUpdate, getVoicePreferenceKey } from '../../src/lib/client/config/updates'; import { buildSyncedPreferencePatch } from '../../src/lib/client/config/preferences'; -test.describe('tts provider catalog', () => { +describe('tts provider catalog', () => { test('resolves provider models with fixed Deepinfra catalog', () => { expect(resolveProviderModels('openai').map((model) => model.id)).toEqual([ 'tts-1', @@ -389,7 +389,7 @@ test.describe('tts provider catalog', () => { }); }); -test.describe('config helpers', () => { +describe('config helpers', () => { test('persists voice preferences by provider/model pair', () => { expect(getVoicePreferenceKey('openai', 'tts-1')).toBe('openai:tts-1'); diff --git a/tests/unit/tts-segment-audio-urls.spec.ts b/tests/unit/tts-segment-audio-urls.vitest.spec.ts similarity index 94% rename from tests/unit/tts-segment-audio-urls.spec.ts rename to tests/unit/tts-segment-audio-urls.vitest.spec.ts index 93b8d92..742d53a 100644 --- a/tests/unit/tts-segment-audio-urls.spec.ts +++ b/tests/unit/tts-segment-audio-urls.vitest.spec.ts @@ -1,10 +1,10 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildSegmentAudioFallbackUrl, resolveSegmentAudioUrls, } from '../../src/lib/server/tts/segment-audio-urls'; -test.describe('tts segment audio url resolution', () => { +describe('tts segment audio url resolution', () => { test('builds fallback URL with encoded params', () => { const url = buildSegmentAudioFallbackUrl('doc 1', 'segment/2'); expect(url).toBe('/api/tts/segments/audio/fallback?documentId=doc%201&segmentId=segment%2F2'); diff --git a/tests/unit/tts-segment-plan-leading.spec.ts b/tests/unit/tts-segment-plan-leading.vitest.spec.ts similarity index 98% rename from tests/unit/tts-segment-plan-leading.spec.ts rename to tests/unit/tts-segment-plan-leading.vitest.spec.ts index 689d36a..3114733 100644 --- a/tests/unit/tts-segment-plan-leading.spec.ts +++ b/tests/unit/tts-segment-plan-leading.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan'; -test.describe('planCanonicalTtsSegments – leading context handling', () => { +describe('planCanonicalTtsSegments – leading context handling', () => { test('clean boundary: splits block so current source keeps its first sentence', () => { const currentSourceKey = 'str:epubcfi(/6/10!/4/2)'; const plan = planCanonicalTtsSegments([ diff --git a/tests/unit/tts-segment-plan.spec.ts b/tests/unit/tts-segment-plan.vitest.spec.ts similarity index 98% rename from tests/unit/tts-segment-plan.spec.ts rename to tests/unit/tts-segment-plan.vitest.spec.ts index 8f03797..c9a3740 100644 --- a/tests/unit/tts-segment-plan.spec.ts +++ b/tests/unit/tts-segment-plan.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildSegmentKey, buildSegmentKeyPrefix, planCanonicalTtsSegments } from '../../src/lib/shared/tts-segment-plan'; -test.describe('planCanonicalTtsSegments', () => { +describe('planCanonicalTtsSegments', () => { test('emits a cross-boundary segment once and assigns it to the source where it starts', () => { const plan = planCanonicalTtsSegments([ { @@ -223,7 +223,7 @@ test.describe('planCanonicalTtsSegments', () => { }); }); -test.describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => { +describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => { // These keys are the bridge between persistence and the sidebar's merge of // synth rows with manifest rows. Both sides must produce identical keys for // the same `(documentId, readerType, text)` triple, or the sidebar will diff --git a/tests/unit/tts-segment-retry-policy.spec.ts b/tests/unit/tts-segment-retry-policy.vitest.spec.ts similarity index 93% rename from tests/unit/tts-segment-retry-policy.spec.ts rename to tests/unit/tts-segment-retry-policy.vitest.spec.ts index 0dd6e29..140e518 100644 --- a/tests/unit/tts-segment-retry-policy.spec.ts +++ b/tests/unit/tts-segment-retry-policy.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { isRetryableSegmentStatus, @@ -6,7 +6,7 @@ import { shouldDeferSegmentRetry, } from '../../src/lib/client/tts/segment-retry-policy'; -test.describe('tts segment retry policy', () => { +describe('tts segment retry policy', () => { test('marks pending/error statuses as retryable', () => { expect(isRetryableSegmentStatus('pending')).toBe(true); expect(isRetryableSegmentStatus('error')).toBe(true); diff --git a/tests/unit/tts-segments-audio-cache.spec.ts b/tests/unit/tts-segments-audio-cache.vitest.spec.ts similarity index 91% rename from tests/unit/tts-segments-audio-cache.spec.ts rename to tests/unit/tts-segments-audio-cache.vitest.spec.ts index ce09173..a1dd8d9 100644 --- a/tests/unit/tts-segments-audio-cache.spec.ts +++ b/tests/unit/tts-segments-audio-cache.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildSegmentAudioCacheHeaders, normalizeAudioByteRangeHeader, @@ -6,7 +6,7 @@ import { TTS_SEGMENT_FALLBACK_CACHE_CONTROL, } from '../../src/lib/server/tts/segments-audio'; -test.describe('tts segment audio cache headers', () => { +describe('tts segment audio cache headers', () => { test('builds fallback cache headers', () => { expect(buildSegmentAudioCacheHeaders()).toEqual({ 'Cache-Control': TTS_SEGMENT_FALLBACK_CACHE_CONTROL, diff --git a/tests/unit/tts-segments-manifest.spec.ts b/tests/unit/tts-segments-manifest.vitest.spec.ts similarity index 94% rename from tests/unit/tts-segments-manifest.spec.ts rename to tests/unit/tts-segments-manifest.vitest.spec.ts index 513ca5c..ad02218 100644 --- a/tests/unit/tts-segments-manifest.spec.ts +++ b/tests/unit/tts-segments-manifest.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { compareManifestSegments, decodeManifestCursor, @@ -6,8 +6,9 @@ import { encodeManifestCursor, parseManifestPageSize, } from '../../src/lib/server/tts/segments-manifest'; +import { locatorGroupKey, locatorIdentityKey } from '../../src/lib/shared/tts-locator'; -test.describe('tts segments manifest helpers', () => { +describe('tts segments manifest helpers', () => { test('dedupe prefers completed variant over newer pending for same settings key', () => { const variants = dedupeManifestVariants([ { @@ -125,8 +126,6 @@ test.describe('tts segments manifest helpers', () => { const sorted = rows.sort(compareManifestSegments); // Recompute groupKey using the production helper to assert the actual // grouping behavior rather than the test's hand-labeled groupKey. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { locatorGroupKey } = require('../../src/lib/shared/tts-locator'); const groupKeys = sorted.map((row) => locatorGroupKey(row.locator)); expect(groupKeys[0]).toBe(groupKeys[1]); // both ch02 expect(groupKeys[2]).not.toBe(groupKeys[0]); // ch03 is its own group @@ -141,9 +140,6 @@ test.describe('tts segments manifest helpers', () => { // rows visually "disappear" when later pages were generated. // // The fix uses `locatorIdentityKey` which includes charOffset for EPUB. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { locatorIdentityKey, locatorGroupKey } = require('../../src/lib/shared/tts-locator'); - const page1Row0 = { readerType: 'epub' as const, spineHref: 'OEBPS/ch02.xhtml', spineIndex: 2, charOffset: 0 }; const page2Row0 = { readerType: 'epub' as const, spineHref: 'OEBPS/ch02.xhtml', spineIndex: 2, charOffset: 2048 }; diff --git a/tests/unit/tts-segments.spec.ts b/tests/unit/tts-segments.vitest.spec.ts similarity index 98% rename from tests/unit/tts-segments.spec.ts rename to tests/unit/tts-segments.vitest.spec.ts index ddd2125..def8e9f 100644 --- a/tests/unit/tts-segments.spec.ts +++ b/tests/unit/tts-segments.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildProportionalAlignment, buildTtsSegmentEntryId, @@ -11,7 +11,7 @@ import { projectSegmentLocator, } from '../../src/lib/server/tts/segments'; -test.describe('tts segment helpers', () => { +describe('tts segment helpers', () => { test('builds stable settings hash', () => { const a = buildTtsSegmentSettingsHash({ providerRef: 'openai', diff --git a/tests/unit/tts-settings-admin-view-model.spec.ts b/tests/unit/tts-settings-admin-view-model.vitest.spec.ts similarity index 97% rename from tests/unit/tts-settings-admin-view-model.spec.ts rename to tests/unit/tts-settings-admin-view-model.vitest.spec.ts index c207ce6..8b1f32f 100644 --- a/tests/unit/tts-settings-admin-view-model.spec.ts +++ b/tests/unit/tts-settings-admin-view-model.vitest.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { resolveTtsSettingsViewModel } from '../../src/lib/client/settings/tts-settings'; import type { SharedProviderEntry } from '../../src/hooks/useSharedProviders'; @@ -20,7 +20,7 @@ const SHARED: SharedProviderEntry[] = [ }, ]; -test.describe('resolveTtsSettingsViewModel (admin/shared modes)', () => { +describe('resolveTtsSettingsViewModel (admin/shared modes)', () => { test('keeps default-openai selection when that shared provider exists', () => { const vm = resolveTtsSettingsViewModel({ providerRef: 'default-openai', diff --git a/tests/unit/walker-theme.spec.ts b/tests/unit/walker-theme.vitest.spec.ts similarity index 93% rename from tests/unit/walker-theme.spec.ts rename to tests/unit/walker-theme.vitest.spec.ts index c5f5000..777b963 100644 --- a/tests/unit/walker-theme.spec.ts +++ b/tests/unit/walker-theme.vitest.spec.ts @@ -1,8 +1,8 @@ -import { expect, test } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildWalkerThemeRules } from '../../src/lib/client/epub/walker-theme'; -test.describe('walker theme rules', () => { +describe('walker theme rules', () => { test('always includes foreground/background colors', () => { const rules = buildWalkerThemeRules({ foreground: 'rgb(1, 2, 3)', diff --git a/tests/unit/whisper-alignment-mapping.spec.ts b/tests/unit/whisper-alignment-mapping.vitest.spec.ts similarity index 88% rename from tests/unit/whisper-alignment-mapping.spec.ts rename to tests/unit/whisper-alignment-mapping.vitest.spec.ts index 5f91973..b53a4f6 100644 --- a/tests/unit/whisper-alignment-mapping.spec.ts +++ b/tests/unit/whisper-alignment-mapping.vitest.spec.ts @@ -1,9 +1,9 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, } from '@openreader/compute-core'; -test.describe('whisper alignment mapping', () => { +describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { const aligned = mapWordsToSentenceOffsets('Hello, world again.', [ { word: 'Hello', start: 0, end: 0.25 }, diff --git a/tests/unit/whisper-spectral.spec.ts b/tests/unit/whisper-spectral.vitest.spec.ts similarity index 91% rename from tests/unit/whisper-spectral.spec.ts rename to tests/unit/whisper-spectral.vitest.spec.ts index 90857a2..2b87a04 100644 --- a/tests/unit/whisper-spectral.spec.ts +++ b/tests/unit/whisper-spectral.vitest.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core'; function dftPower(samples: Float32Array, k: number): number { @@ -13,7 +13,7 @@ function dftPower(samples: Float32Array, k: number): number { return (re * re) + (im * im); } -test.describe('whisper spectral helpers', () => { +describe('whisper spectral helpers', () => { test('goertzel power matches direct DFT for non-power-of-two frame size', () => { const frameSize = 400; const bins = 201; diff --git a/tests/unit/whisper-token-timestamps.spec.ts b/tests/unit/whisper-token-timestamps.vitest.spec.ts similarity index 95% rename from tests/unit/whisper-token-timestamps.spec.ts rename to tests/unit/whisper-token-timestamps.vitest.spec.ts index a584d6f..c2aded0 100644 --- a/tests/unit/whisper-token-timestamps.spec.ts +++ b/tests/unit/whisper-token-timestamps.vitest.spec.ts @@ -1,11 +1,11 @@ -import { test, expect } from '@playwright/test'; +import { describe, expect, test } from 'vitest'; import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, } from '@openreader/compute-core'; -test.describe('whisper token timestamp alignment', () => { +describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { const seqLen = 6; const frames = 10; From fc3d05d65b2fed65b30691b9ff49b9192402d16b Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 14:17:49 -0600 Subject: [PATCH 09/16] feat(rate-limit): add per-user PDF parsing rate limiting and job event ledger implement a generic user_job_events table for tracking compute job creation enforce configurable burst and sustained limits for PDF layout parsing add admin panel controls for compute rate limiting and max upload size update API routes to apply and record rate checks for PDF parse jobs document new environment variables and admin settings for compute limits improve IP extraction logic for rate limiting accuracy add tests for request IP extraction and test namespace gating This change introduces a robust mechanism to throttle expensive compute operations, such as PDF parsing, on a per-user basis. It provides both burst and sustained rate controls, with admin-tunable parameters and clear user feedback on throttling. The job event ledger enables accurate concurrency and rate enforcement, while new documentation and tests ensure maintainability and clarity. --- .env.example | 10 + docs-site/docs/configure/admin-panel.md | 8 + docs-site/docs/configure/tts-rate-limiting.md | 1 + .../docs/reference/environment-variables.md | 84 +- drizzle/postgres/0008_user_job_events.sql | 9 + drizzle/postgres/meta/0008_snapshot.json | 1906 +++++++++++++++++ drizzle/postgres/meta/_journal.json | 7 + drizzle/sqlite/0008_user_job_events.sql | 9 + drizzle/sqlite/meta/0008_snapshot.json | 1751 +++++++++++++++ drizzle/sqlite/meta/_journal.json | 7 + playwright.config.ts | 6 +- src/app/api/documents/[id]/parsed/route.ts | 16 + .../documents/blob/upload/fallback/route.ts | 21 + .../documents/blob/upload/presign/route.ts | 18 +- .../api/documents/docx-to-pdf/upload/route.ts | 5 + src/app/api/documents/route.ts | 13 + src/components/admin/AdminFeaturesPanel.tsx | 87 + src/db/schema.ts | 1 + src/db/schema_postgres.ts | 16 + src/db/schema_sqlite.ts | 16 + src/lib/server/admin/settings.ts | 11 + src/lib/server/documents/blobstore.ts | 11 + src/lib/server/rate-limit/job-rate-limiter.ts | 162 ++ src/lib/server/rate-limit/problem-response.ts | 26 + src/lib/server/rate-limit/request-ip.ts | 55 +- src/lib/server/testing/test-namespace.ts | 17 + tests/unit/request-ip.vitest.spec.ts | 53 + tests/unit/test-namespace-gate.vitest.spec.ts | 27 + 28 files changed, 4319 insertions(+), 34 deletions(-) create mode 100644 drizzle/postgres/0008_user_job_events.sql create mode 100644 drizzle/postgres/meta/0008_snapshot.json create mode 100644 drizzle/sqlite/0008_user_job_events.sql create mode 100644 drizzle/sqlite/meta/0008_snapshot.json create mode 100644 src/lib/server/rate-limit/job-rate-limiter.ts create mode 100644 tests/unit/request-ip.vitest.spec.ts create mode 100644 tests/unit/test-namespace-gate.vitest.spec.ts diff --git a/.env.example b/.env.example index d3c5c47..0aebf94 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,12 @@ AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional tr # (Optional) Disable Better Auth built-in rate limiting (useful for testing; default: `false`) # DISABLE_AUTH_RATE_LIMIT=false +# (Optional) Honor the x-openreader-test-namespace header on a production build. +# This is test/CI scaffolding and MUST stay unset on real deployments; it is set +# automatically by the Playwright web server. Non-production builds honor it +# without this flag. +# ENABLE_TEST_NAMESPACE=false + # (Optional) Comma-separated list of emails that are auto-promoted to admin. # Admins see the "Admin" tab in Settings (TTS shared providers + site features). # Demotion is automatic: removing an email here demotes the user on next login. @@ -118,6 +124,10 @@ S3_BUCKET= # RUNTIME_SEED_CHANGELOG_FEED_URL=https://docs.openreader.richardr.dev/changelog/manifest.json # RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT=true # RUNTIME_SEED_DISABLE_TTS_LIMIT=true +# Seeds the PDF parsing rate limit on first boot (default: enabled). All other +# compute-limit values (burst/sustained limits, windows) and the max upload size +# are configured in Settings → Admin, not via env. +# RUNTIME_SEED_DISABLE_COMPUTE_LIMIT=false # Migrations configuration # (Optional) Skip automatic startup migrations when set to `false` (default: `true`) diff --git a/docs-site/docs/configure/admin-panel.md b/docs-site/docs/configure/admin-panel.md index 3d36571..bdb64ce 100644 --- a/docs-site/docs/configure/admin-panel.md +++ b/docs-site/docs/configure/admin-panel.md @@ -77,6 +77,14 @@ Runtime-editable settings, one row per key: | `enableAudiobookExport` | Show the audiobook export entry points on PDF/EPUB pages. | | `enableDocxConversion` | Accept .docx uploads (converted to PDF server-side). | | `enableDestructiveDeleteActions` | Show "Delete all data" buttons in the Documents tab (auth-disabled mode). | +| `disableTtsRateLimit` | Disable the per-user/IP daily TTS character limits. When `false`, the daily-limit fields below it apply. | +| `disableComputeRateLimit` | Disable per-user PDF parsing rate limiting. When `false`, the burst/sustained limit fields below it apply. | +| `maxUploadMb` | Maximum size (MB) accepted for a single document upload. Enforced server-side and signed into the presigned S3 PUT. | + +The **Disable TTS daily rate limiting** and **Disable PDF parsing rate limiting** toggles each reveal a collapsible group of numeric inputs when set to `false`: + +- TTS: anonymous/authenticated per-user daily limits and anonymous/authenticated IP daily backstops. +- PDF parsing: burst limit + window (seconds) and sustained limit + window (seconds). The sustained window doubles as a concurrency cap. Word-by-word highlighting and PDF layout parsing capability are controlled by compute-worker server env configuration, not an admin runtime flag. diff --git a/docs-site/docs/configure/tts-rate-limiting.md b/docs-site/docs/configure/tts-rate-limiting.md index b9cd801..fedf7f0 100644 --- a/docs-site/docs/configure/tts-rate-limiting.md +++ b/docs-site/docs/configure/tts-rate-limiting.md @@ -37,5 +37,6 @@ If a request exceeds the active limit, the TTS API returns `429` with reset meta ## Related docs - TTS/rate-limit environment variables: [Environment Variables](../reference/environment-variables#tts-provider-and-request-behavior) +- PDF parsing rate limiting (separate, compute-side throttle): [Admin Panel → Site features](./admin-panel#site-features) and [Environment Variables](../reference/environment-variables#compute-pdf-parsing-rate-limiting-runtime-settings) - Auth configuration: [Auth](./auth) - Provider setup: [TTS Providers](./tts-providers) diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index 4db5083..cb9fa0d 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -31,6 +31,7 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `GITHUB_CLIENT_ID` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_SECRET` to enable GitHub sign-in | | `GITHUB_CLIENT_SECRET` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_ID` to enable GitHub sign-in | | `DISABLE_AUTH_RATE_LIMIT` | Rate limiting | `false` | Set `true` to disable auth-layer rate limiting | +| `ENABLE_TEST_NAMESPACE` | Testing/CI | unset | Honor the `x-openreader-test-namespace` header on a production build; leave unset on real deployments | | `ADMIN_EMAILS` | Auth/Admin | empty | Comma-separated emails auto-promoted to admin (requires auth enabled) | | `POSTGRES_URL` | Database | unset (SQLite mode) | Set to switch metadata/auth DB to Postgres | | `USE_EMBEDDED_WEED_MINI` | Storage | `true` when unset | Set `false` to use external S3-compatible storage only | @@ -43,8 +44,6 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `S3_ENDPOINT` | Storage | derived in embedded mode | Set for S3-compatible providers (MinIO/SeaweedFS/R2/etc.) | | `S3_FORCE_PATH_STYLE` | Storage | `true` in embedded mode | Set per provider requirement | | `S3_PREFIX` | Storage | `openreader` | Customize object key prefix | -| `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations | -| `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass | | `IMPORT_LIBRARY_DIR` | Library import | `docstore/library` fallback | Set a single server library root | | `IMPORT_LIBRARY_DIRS` | Library import | unset | Set multiple roots (comma/colon/semicolon separated) | | `COMPUTE_WORKER_URL` | Heavy compute backend | unset | Set only for standalone external compute worker; leave unset for embedded worker startup | @@ -72,6 +71,9 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `RUNTIME_SEED_DEFAULT_TTS_PROVIDER` | Legacy bootstrap seed | `custom-openai` | Optional first-boot seed for default TTS provider slug | | `RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT` | Legacy bootstrap seed | `true` | Optional first-boot seed to enable audiobook export UI | | `RUNTIME_SEED_DISABLE_TTS_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps TTS daily rate limiting disabled | +| `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT` | Legacy bootstrap seed | `false` | Optional first-boot seed that disables PDF parsing rate limiting (other compute-limit values + max upload size are admin-only) | +| `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations | +| `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass | @@ -228,6 +230,15 @@ Controls Better Auth rate limiting. - This does not affect TTS character rate limiting - Related docs: [Auth](../configure/auth) +### ENABLE_TEST_NAMESPACE + +Honors the `x-openreader-test-namespace` request header, which scopes documents/storage into an isolated namespace for end-to-end tests. + +- Default: unset (header ignored on production builds) +- Non-production builds (`NODE_ENV !== 'production'`) honor the header without this flag. +- Production builds (`pnpm build && pnpm start`) honor it only when `ENABLE_TEST_NAMESPACE=true`. The Playwright web server sets this automatically. +- **Leave unset on real deployments.** It is test/CI scaffolding only. + ### ADMIN_EMAILS Comma-separated list of email addresses that are auto-promoted to admin. @@ -330,24 +341,13 @@ Prefix prepended to stored object keys. - Default: `openreader` - Related docs: [Object / Blob Storage](../configure/object-blob-storage) -## Migration Controls +### Maximum Upload Size (Runtime Settings) -### RUN_DRIZZLE_MIGRATIONS +The maximum size accepted for a single document upload is an admin runtime setting (no env var). -Controls startup migration execution in shared entrypoint. - -- Default: `true` -- Set `false` to skip automatic startup Drizzle schema migrations -- Related docs: [Migrations](../configure/migrations), [Database](../configure/database) - -### RUN_FS_MIGRATIONS - -Controls startup filesystem-to-object-store migration execution in shared entrypoint. - -- Default: `true` -- Runs `scripts/migrate-fs-v2.mjs` at startup after DB migrations -- Set `false` to skip automatic storage migration pass -- Related docs: [Migrations](../configure/migrations), [Database](../configure/database), [Object / Blob Storage](../configure/object-blob-storage) +- Runtime key: `maxUploadMb` (default: `200`) +- Configure in **Settings → Admin → Site features → Max upload size (MB)**. +- Enforced up front (`413`) and signed into the presigned S3 PUT so a client cannot stream more than it declared. ## Library Import @@ -492,6 +492,19 @@ Absolute path or executable name for the ffmpeg binary used by audiobook/process - Resolution order: `FFMPEG_BIN` -> `ffmpeg-static` - Example: `/var/task/node_modules/ffmpeg-static/ffmpeg` +### Compute (PDF Parsing) Rate Limiting (Runtime Settings) + +Per-user throttling of expensive PDF layout parsing is managed from **Settings → Admin → Site features**, not env vars. Enforcement applies only when auth is enabled. + +- `disableComputeRateLimit` default: `false` (rate limiting enabled) +- When enabled, the following admin-tunable sub-limits apply: + - `computeParseBurstMax` (default `8`) over `computeParseBurstWindowSec` (default `60`) + - `computeParseSustainedMax` (default `24`) over `computeParseSustainedWindowSec` (default `600`) +- The sustained window also acts as a concurrency cap: because the worker bounds each job's duration, the count of parses started in that window is an upper bound on those still running. +- Exceeding a limit returns `429` (`application/problem+json`) with `Retry-After`. + +Optional first-boot seed: `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT`. All other values are DB-backed admin runtime settings. + ## Legacy First-Boot Runtime Seeds (optional) These variables exist only as **first-boot seeds** for the admin-managed runtime config. Prefer changing site features from **Settings → Admin → Site features**. Keep these only when you need bootstrap defaults before the first admin login. See [Admin Panel](../configure/admin-panel) for migration behavior. @@ -564,3 +577,38 @@ Controls whether audiobook export UI/actions are shown in the client. - Default: `true` (enabled) - Affects export entry points in PDF/EPUB pages and document settings UI - Runtime key: `enableAudiobookExport` + +### RUNTIME_SEED_DISABLE_TTS_LIMIT + +Seeds the TTS daily character rate-limit on/off state on first boot. + +- Default: `true` (TTS rate limiting disabled) +- Runtime key: `disableTtsRateLimit` +- Per-user/IP daily limit values are admin-only runtime settings (see [TTS Daily Rate Limiting](#tts-daily-rate-limiting-runtime-settings)). + +### RUNTIME_SEED_DISABLE_COMPUTE_LIMIT + +Seeds the PDF parsing rate-limit on/off state on first boot. + +- Default: `false` (PDF parsing rate limiting enabled) +- Runtime key: `disableComputeRateLimit` +- The burst/sustained limits, their windows, and the max upload size are admin-only runtime settings (see [Compute (PDF Parsing) Rate Limiting](#compute-pdf-parsing-rate-limiting-runtime-settings)). + +## Migration Controls + +### RUN_DRIZZLE_MIGRATIONS + +Controls startup migration execution in shared entrypoint. + +- Default: `true` +- Set `false` to skip automatic startup Drizzle schema migrations +- Related docs: [Migrations](../configure/migrations), [Database](../configure/database) + +### RUN_FS_MIGRATIONS + +Controls startup filesystem-to-object-store migration execution in shared entrypoint. + +- Default: `true` +- Runs `scripts/migrate-fs-v2.mjs` at startup after DB migrations +- Set `false` to skip automatic storage migration pass +- Related docs: [Migrations](../configure/migrations), [Database](../configure/database), [Object / Blob Storage](../configure/object-blob-storage) diff --git a/drizzle/postgres/0008_user_job_events.sql b/drizzle/postgres/0008_user_job_events.sql new file mode 100644 index 0000000..be33cc7 --- /dev/null +++ b/drizzle/postgres/0008_user_job_events.sql @@ -0,0 +1,9 @@ +CREATE TABLE "user_job_events" ( + "user_id" text NOT NULL, + "action" text NOT NULL, + "op_id" text NOT NULL, + "created_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint NOT NULL, + CONSTRAINT "user_job_events_user_id_action_op_id_pk" PRIMARY KEY("user_id","action","op_id") +); +--> statement-breakpoint +CREATE INDEX "idx_user_job_events_user_action_created" ON "user_job_events" USING btree ("user_id","action","created_at"); \ No newline at end of file diff --git a/drizzle/postgres/meta/0008_snapshot.json b/drizzle/postgres/meta/0008_snapshot.json new file mode 100644 index 0000000..03a4af9 --- /dev/null +++ b/drizzle/postgres/meta/0008_snapshot.json @@ -0,0 +1,1906 @@ +{ + "id": "0381962f-77d4-451e-b4ec-08b9a0dde70d", + "prevId": "68baf959-cd1d-418b-a04e-11397aa05c39", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admin_providers": { + "name": "admin_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_settings": { + "name": "admin_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value_json": { + "name": "value_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobook_chapters": { + "name": "audiobook_chapters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "name": "audiobook_chapters_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobooks": { + "name": "audiobooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "name": "audiobooks_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_previews": { + "name": "document_previews", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_until_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "name": "document_previews_document_id_namespace_variant_pk", + "columns": [ + "document_id", + "namespace", + "variant" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_settings": { + "name": "document_settings", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "name": "document_settings_document_id_user_id_pk", + "columns": [ + "document_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_modified": { + "name": "last_modified", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parse_state": { + "name": "parse_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parsed_json_key": { + "name": "parsed_json_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_modified", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "name": "documents_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_entries": { + "name": "tts_segment_entries", + "schema": "", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_version": { + "name": "document_version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_reader_rank", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_char_offset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_href", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_location", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "name": "tts_segment_entries_segment_entry_id_user_id_pk", + "columns": [ + "segment_entry_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_variants": { + "name": "tts_segment_variants", + "schema": "", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_json": { + "name": "settings_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "settings_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "name": "tts_segment_variants_segment_id_user_id_pk", + "columns": [ + "segment_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_document_progress": { + "name": "user_document_progress", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "name": "user_document_progress_user_id_document_id_pk", + "columns": [ + "user_id", + "document_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_job_events": { + "name": "user_job_events", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "name": "user_job_events_user_id_action_op_id_pk", + "columns": [ + "user_id", + "action", + "op_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preferences": { + "name": "user_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_tts_chars": { + "name": "user_tts_chars", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "name": "user_tts_chars_user_id_date_pk", + "columns": [ + "user_id", + "date" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index eccd0d6..6760c50 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1779378127846, "tag": "0007_pdf_parse_progress", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1780162695652, + "tag": "0008_user_job_events", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/sqlite/0008_user_job_events.sql b/drizzle/sqlite/0008_user_job_events.sql new file mode 100644 index 0000000..f3d8b8b --- /dev/null +++ b/drizzle/sqlite/0008_user_job_events.sql @@ -0,0 +1,9 @@ +CREATE TABLE `user_job_events` ( + `user_id` text NOT NULL, + `action` text NOT NULL, + `op_id` text NOT NULL, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + PRIMARY KEY(`user_id`, `action`, `op_id`) +); +--> statement-breakpoint +CREATE INDEX `idx_user_job_events_user_action_created` ON `user_job_events` (`user_id`,`action`,`created_at`); \ No newline at end of file diff --git a/drizzle/sqlite/meta/0008_snapshot.json b/drizzle/sqlite/meta/0008_snapshot.json new file mode 100644 index 0000000..b920031 --- /dev/null +++ b/drizzle/sqlite/meta/0008_snapshot.json @@ -0,0 +1,1751 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5e21c9cb-0d9b-48fe-92ec-9e98ac43c198", + "prevId": "031a62ae-47a7-4476-a577-7b14ff787eba", + "tables": { + "admin_providers": { + "name": "admin_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "admin_settings": { + "name": "admin_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobook_chapters": { + "name": "audiobook_chapters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobook_chapters_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobooks": { + "name": "audiobooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobooks_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_previews": { + "name": "document_previews", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + "status", + "lease_until_ms" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "columns": [ + "document_id", + "namespace", + "variant" + ], + "name": "document_previews_document_id_namespace_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_settings": { + "name": "document_settings", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "columns": [ + "document_id", + "user_id" + ], + "name": "document_settings_document_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parse_state": { + "name": "parse_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parsed_json_key": { + "name": "parsed_json_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + "user_id", + "last_modified" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "documents_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_entries": { + "name": "tts_segment_entries", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_version": { + "name": "document_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + "user_id", + "document_id", + "document_version", + "locator_reader_rank", + "locator_spine_index", + "locator_char_offset", + "locator_spine_href", + "locator_page", + "locator_location", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + "user_id", + "document_id", + "document_version", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + "user_id", + "document_id", + "document_version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "columns": [ + "segment_entry_id", + "user_id" + ], + "name": "tts_segment_entries_segment_entry_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_variants": { + "name": "tts_segment_variants", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + "user_id", + "segment_entry_id", + "updated_at" + ], + "isUnique": false + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + "user_id", + "status" + ], + "isUnique": false + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + "user_id", + "segment_entry_id", + "settings_hash" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "columns": [ + "segment_id", + "user_id" + ], + "name": "tts_segment_variants_segment_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_document_progress": { + "name": "user_document_progress", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + "user_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "columns": [ + "user_id", + "document_id" + ], + "name": "user_document_progress_user_id_document_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_job_events": { + "name": "user_job_events", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + "user_id", + "action", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "columns": [ + "user_id", + "action", + "op_id" + ], + "name": "user_job_events_user_id_action_op_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_preferences": { + "name": "user_preferences", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_tts_chars": { + "name": "user_tts_chars", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + "date" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "columns": [ + "user_id", + "date" + ], + "name": "user_tts_chars_user_id_date_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index 8db8d88..5db4a53 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1779378125905, "tag": "0007_pdf_parse_progress", "breakpoints": true + }, + { + "idx": 8, + "version": "6", + "when": 1780162695101, + "tag": "0008_user_job_events", + "breakpoints": true } ] } \ No newline at end of file diff --git a/playwright.config.ts b/playwright.config.ts index a1f9f25..6cd95e5 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -39,8 +39,10 @@ export default defineConfig({ /* Run your local dev server before starting the tests */ webServer: { - // Disable auth rate limiting for tests to support parallel workers creating sessions - command: `pnpm build && DISABLE_AUTH_RATE_LIMIT=true pnpm start`, + // Disable auth rate limiting for tests to support parallel workers creating sessions. + // ENABLE_TEST_NAMESPACE opts the production build into honoring the + // x-openreader-test-namespace header (ignored on real prod deployments). + command: `pnpm build && DISABLE_AUTH_RATE_LIMIT=true ENABLE_TEST_NAMESPACE=true pnpm start`, url: 'http://localhost:3003', reuseExistingServer: !process.env.CI, timeout: 120 * 1000, diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 51ef630..3ca677f 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -22,6 +22,9 @@ import { } from '@/lib/server/documents/parse-state'; import { healStaleDocumentParseState } from '@/lib/server/documents/parse-state-healing'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; +import { checkJobRate, recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { buildComputeRateLimitedResponse } from '@/lib/server/rate-limit/problem-response'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; import { createRequestLogger, hashForLog, type ServerLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; @@ -263,11 +266,17 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string } if (effectiveStatus === 'failed' && retryFailed) { + const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig); + if (!rateDecision.allowed) { + return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); + } const created = await createOrReusePdfWorkerOperation({ documentId: id, namespace: testNamespace, documentObjectKey: documentKey(id, testNamespace), }); + await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); const snapshot = snapshotFromWorkerState(created); return NextResponse.json({ parseStatus: snapshot.parseStatus, @@ -385,12 +394,19 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string } } + const rateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + const rateDecision = await checkJobRate(authCtxOrRes.userId, 'pdf_layout', rateConfig); + if (!rateDecision.allowed) { + return buildComputeRateLimitedResponse({ decision: rateDecision, pathname: req.nextUrl.pathname }); + } + const created = await createOrReusePdfWorkerOperation({ documentId: id, namespace: testNamespace, documentObjectKey: documentKey(id, testNamespace), forceToken: randomUUID(), }); + await recordJobEvent(authCtxOrRes.userId, 'pdf_layout', created.opId, rateConfig); const snapshot = snapshotFromWorkerState(created); diff --git a/src/app/api/documents/blob/upload/fallback/route.ts b/src/app/api/documents/blob/upload/fallback/route.ts index bc188df..74f67df 100644 --- a/src/app/api/documents/blob/upload/fallback/route.ts +++ b/src/app/api/documents/blob/upload/fallback/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents/blobstore'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { isS3Configured } from '@/lib/server/storage/s3'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -33,7 +34,27 @@ export async function PUT(req: NextRequest) { } const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream'; + + const { maxUploadMb } = await getResolvedRuntimeConfig(); + const maxUploadBytes = maxUploadMb * 1024 * 1024; + // Reject before buffering when the declared length is already over the cap. + const declaredLength = Number(req.headers.get('content-length') || ''); + if (Number.isFinite(declaredLength) && declaredLength > maxUploadBytes) { + return NextResponse.json( + { error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, maxBytes: maxUploadBytes }, + { status: 413 }, + ); + } + const body = Buffer.from(await req.arrayBuffer()); + // Backstop for chunked/omitted Content-Length: enforce on the actual bytes. + if (body.byteLength > maxUploadBytes) { + return NextResponse.json( + { error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, maxBytes: maxUploadBytes }, + { status: 413 }, + ); + } + const namespace = getOpenReaderTestNamespace(req.headers); try { diff --git a/src/app/api/documents/blob/upload/presign/route.ts b/src/app/api/documents/blob/upload/presign/route.ts index 07756aa..6d91ee4 100644 --- a/src/app/api/documents/blob/upload/presign/route.ts +++ b/src/app/api/documents/blob/upload/presign/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId, presignPut } from '@/lib/server/documents/blobstore'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -53,10 +54,25 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 }); } + const { maxUploadMb } = await getResolvedRuntimeConfig(); + const maxUploadBytes = maxUploadMb * 1024 * 1024; + const oversized = uploads.find((upload) => upload.size > maxUploadBytes); + if (oversized) { + return NextResponse.json( + { + error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, + maxBytes: maxUploadBytes, + }, + { status: 413 }, + ); + } + const namespace = getOpenReaderTestNamespace(req.headers); const signed = await Promise.all( uploads.map(async (upload) => { - const res = await presignPut(upload.id, upload.contentType, namespace); + const res = await presignPut(upload.id, upload.contentType, namespace, { + contentLength: upload.size, + }); return { id: upload.id, url: res.url, diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 91866af..4d2b992 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -11,6 +11,8 @@ import { documents } from '@/db/schema'; import { safeDocumentName } from '@/lib/server/documents/utils'; import { enqueueDocumentPreview } from '@/lib/server/documents/previews'; import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; +import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; @@ -173,6 +175,9 @@ export async function POST(req: NextRequest) { }, 'Failed to enqueue preview for converted DOCX'); }); + // Record upload-driven parse load (see register route for rationale). + const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); + await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig); enqueueParsePdfJob({ documentId: id, userId: storageUserId, diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index c56891c..af9c38a 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { and, count, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; @@ -12,6 +13,8 @@ import { enqueueDocumentPreview, } from '@/lib/server/documents/previews'; import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job'; +import { recordJobEvent, getPdfLayoutRateConfig } from '@/lib/server/rate-limit/job-rate-limiter'; +import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; import { normalizeParseStatus, @@ -90,6 +93,11 @@ export async function POST(req: NextRequest) { const stored: BaseDocument[] = []; + // Resolve the parse rate-limit config once (only when a PDF is present). + const pdfRateConfig = documentsData.some((doc) => doc.type === 'pdf') + ? getPdfLayoutRateConfig(await getResolvedRuntimeConfig()) + : null; + for (const doc of documentsData) { let headSize = doc.size; @@ -184,6 +192,11 @@ export async function POST(req: NextRequest) { }); if (doc.type === 'pdf') { + // Account for upload-driven parse load in the same ledger the explicit + // re-parse limiter reads. We record (not reject) here so a legitimate + // bulk upload always parses; the recorded load still throttles + // subsequent loopable re-parse spam via /parsed. + await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false }); enqueueParsePdfJob({ documentId: doc.id, userId: storageUserId, diff --git a/src/components/admin/AdminFeaturesPanel.tsx b/src/components/admin/AdminFeaturesPanel.tsx index 575c875..a450520 100644 --- a/src/components/admin/AdminFeaturesPanel.tsx +++ b/src/components/admin/AdminFeaturesPanel.tsx @@ -160,6 +160,7 @@ export function AdminFeaturesPanel() { : fallbackShared; const selectedProviderOption = effectiveSelectedProvider; const shouldRenderRateLimitInputs = draft.disableTtsRateLimit === false; + const shouldRenderComputeRateLimitInputs = draft.disableComputeRateLimit === false; const handleProviderChange = (opt: ProviderOption) => { updateDraft('defaultTtsProvider', opt.id); @@ -347,6 +348,92 @@ export function AdminFeaturesPanel() { ) : null} + + updateDraft('disableComputeRateLimit', checked)} + right={renderSource('disableComputeRateLimit')} + variant="flat" + /> + {shouldRenderComputeRateLimitInputs ? ( +
+
+
+ + {renderSource('computeParseBurstMax')} +
+ updatePositiveIntDraft('computeParseBurstMax', event.target.value)} + /> +
+
+
+ + {renderSource('computeParseBurstWindowSec')} +
+ updatePositiveIntDraft('computeParseBurstWindowSec', event.target.value)} + /> +
+
+
+ + {renderSource('computeParseSustainedMax')} +
+ updatePositiveIntDraft('computeParseSustainedMax', event.target.value)} + /> +
+
+
+ + {renderSource('computeParseSustainedWindowSec')} +
+ updatePositiveIntDraft('computeParseSustainedWindowSec', event.target.value)} + /> +
+
+ ) : null} + +
+
+
+ + {renderSource('maxUploadMb')} +
+ updatePositiveIntDraft('maxUploadMb', event.target.value)} + /> +
+
[ + primaryKey({ columns: [table.userId, table.action, table.opId] }), + index('idx_user_job_events_user_action_created').on(table.userId, table.action, table.createdAt), +]); + export const userPreferences = pgTable('user_preferences', { userId: text('user_id').primaryKey().references(() => user.id, { onDelete: 'cascade' }), dataJson: jsonb('data_json').notNull().default({}), diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index 2781e7d..83d3420 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -66,6 +66,22 @@ export const userTtsChars = sqliteTable("user_tts_chars", { index('idx_user_tts_chars_date').on(table.date), ]); +// Generic per-user job-creation ledger for rate/concurrency limiting of +// expensive compute operations (e.g. PDF layout parsing). One row per created +// worker op. A trailing-window COUNT over (user_id, action) enforces both a +// short-window burst cap and a wider sustained/concurrency cap; because the +// worker bounds each op by a hard cap, "ops created in the last hard-cap +// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically. +export const userJobEvents = sqliteTable('user_job_events', { + userId: text('user_id').notNull(), + action: text('action').notNull(), + opId: text('op_id').notNull(), + createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS), +}, (table) => [ + primaryKey({ columns: [table.userId, table.action, table.opId] }), + index('idx_user_job_events_user_action_created').on(table.userId, table.action, table.createdAt), +]); + export const userPreferences = sqliteTable('user_preferences', { userId: text('user_id').primaryKey().references(() => user.id, { onDelete: 'cascade' }), dataJson: text('data_json').notNull().default('{}'), diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index c92b718..aacb273 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -111,6 +111,17 @@ export const RUNTIME_CONFIG_SCHEMA = { ttsDailyLimitAuthenticated: positiveIntValue(500_000), ttsIpDailyLimitAnonymous: positiveIntValue(100_000), ttsIpDailyLimitAuthenticated: positiveIntValue(1_000_000), + // Per-user throttle for expensive PDF-layout parsing. Enabled by default. + // When enabled, the sub-limits below apply (admin-tunable, no env seed): + // a short "burst" window plus a wider "sustained" window that also bounds + // concurrency (the worker caps each job's duration). + disableComputeRateLimit: booleanFlag(false, 'RUNTIME_SEED_DISABLE_COMPUTE_LIMIT'), + computeParseBurstMax: positiveIntValue(8), + computeParseBurstWindowSec: positiveIntValue(60), + computeParseSustainedMax: positiveIntValue(24), + computeParseSustainedWindowSec: positiveIntValue(600), + // Maximum size (MB) accepted for a single document upload. + maxUploadMb: positiveIntValue(200), } as const satisfies Record>; export type RuntimeConfigKey = keyof typeof RUNTIME_CONFIG_SCHEMA; diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index 5ecae17..fb84f00 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -104,17 +104,28 @@ export async function presignPut( id: string, contentType: string, namespace: string | null, + options?: { contentLength?: number }, ): Promise<{ url: string; headers: Record }> { const cfg = getS3Config(); const client = getS3Client(); const key = documentKey(id, namespace); const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; + // When the client declares an exact size, sign Content-Length so S3 rejects a + // PUT whose body does not match (the browser always sends an accurate + // Content-Length for a known body). Skipped when size is unknown/zero so the + // upload still works against stores that enforce the signed header. + const contentLength = + typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0 + ? Math.floor(options.contentLength) + : undefined; + const command = new PutObjectCommand({ Bucket: cfg.bucket, Key: key, ContentType: normalizedType, ServerSideEncryption: 'AES256', + ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), }); const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); diff --git a/src/lib/server/rate-limit/job-rate-limiter.ts b/src/lib/server/rate-limit/job-rate-limiter.ts new file mode 100644 index 0000000..d65a183 --- /dev/null +++ b/src/lib/server/rate-limit/job-rate-limiter.ts @@ -0,0 +1,162 @@ +import { and, eq, gte, lt, sql } from 'drizzle-orm'; +import { db } from '@/db'; +import { userJobEvents } from '@/db/schema'; +import { isAuthEnabled } from '@/lib/server/auth/config'; +import { nowTimestampMs } from '@/lib/shared/timestamps'; +import type { RuntimeConfig } from '@/lib/server/admin/settings'; + +/** + * Per-user rate / concurrency limiting for expensive compute operations. + * + * Backed by the `user_job_events` ledger: one row is recorded per created + * worker op, and limits are enforced by COUNTing rows for a (userId, action) + * within trailing time windows. Two windows are configured per action: + * - a short "burst" window (stops tight retry/replace loops), and + * - a wider "sustained" window. Because the worker bounds each op by a hard + * cap, "ops created in the sustained window" is an upper bound on the + * number that can still be running — i.e. an effective concurrency cap. + * + * Limits are configured by site admins via runtime config (not env), mirroring + * the TTS rate limiter. The design is generic: add a new `JobRateAction` plus a + * config mapping to throttle a different job type. Limits are a backstop — a + * small overshoot under highly concurrent requests is acceptable (the check is + * read-then-record, not a single atomic transaction). + */ +export type JobRateAction = 'pdf_layout'; + +export interface JobRateWindow { + windowMs: number; + max: number; +} + +export interface JobRateConfig { + enabled: boolean; + windows: JobRateWindow[]; +} + +export interface JobRateDecision { + allowed: boolean; + /** Milliseconds until the binding window frees a slot (0 when allowed). */ + retryAfterMs: number; + counts: Array<{ windowMs: number; max: number; count: number }>; +} + +/** Builds the PDF-layout parse limiter config from resolved runtime config. */ +export function getPdfLayoutRateConfig(runtime: Pick< + RuntimeConfig, + | 'disableComputeRateLimit' + | 'computeParseBurstMax' + | 'computeParseBurstWindowSec' + | 'computeParseSustainedMax' + | 'computeParseSustainedWindowSec' +>): JobRateConfig { + return { + enabled: !runtime.disableComputeRateLimit, + windows: [ + { windowMs: runtime.computeParseBurstWindowSec * 1000, max: runtime.computeParseBurstMax }, + { windowMs: runtime.computeParseSustainedWindowSec * 1000, max: runtime.computeParseSustainedMax }, + ], + }; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const safeDb = () => db as any; + +function isActive(config: Pick, userId: string | null | undefined): userId is string { + return isAuthEnabled() && config.enabled && Boolean(userId); +} + +async function countSince(userId: string, action: JobRateAction, since: number): Promise { + const rows = await safeDb() + .select({ c: sql`count(*)` }) + .from(userJobEvents) + .where(and( + eq(userJobEvents.userId, userId), + eq(userJobEvents.action, action), + gte(userJobEvents.createdAt, since), + )); + return Number(rows[0]?.c ?? 0); +} + +async function oldestSince(userId: string, action: JobRateAction, since: number): Promise { + const rows = await safeDb() + .select({ oldest: sql`min(${userJobEvents.createdAt})` }) + .from(userJobEvents) + .where(and( + eq(userJobEvents.userId, userId), + eq(userJobEvents.action, action), + gte(userJobEvents.createdAt, since), + )); + const value = rows[0]?.oldest; + return value == null ? null : Number(value); +} + +/** + * Returns whether a new op may be created for this user/action. Read-only — call + * `recordJobEvent` after the op is actually created. + */ +export async function checkJobRate( + userId: string | null | undefined, + action: JobRateAction, + config: JobRateConfig, +): Promise { + if (!isActive(config, userId)) { + return { allowed: true, retryAfterMs: 0, counts: [] }; + } + + const now = nowTimestampMs(); + const counts: JobRateDecision['counts'] = []; + let retryAfterMs = 0; + let allowed = true; + + for (const window of config.windows) { + if (!Number.isFinite(window.windowMs) || window.windowMs <= 0 || window.max <= 0) continue; + const since = now - window.windowMs; + const count = await countSince(userId, action, since); + counts.push({ windowMs: window.windowMs, max: window.max, count }); + if (count >= window.max) { + allowed = false; + const oldest = await oldestSince(userId, action, since); + // When the oldest in-window event ages out, a slot frees up. + const freesAt = (oldest ?? now) + window.windowMs; + retryAfterMs = Math.max(retryAfterMs, Math.max(0, freesAt - now)); + } + } + + return { allowed, retryAfterMs, counts }; +} + +/** Records a created op so it counts toward future window checks. */ +export async function recordJobEvent( + userId: string | null | undefined, + action: JobRateAction, + opId: string, + config: Pick, +): Promise { + if (!isActive(config, userId) || !opId) return; + + const now = nowTimestampMs(); + try { + await safeDb() + .insert(userJobEvents) + .values({ userId, action, opId, createdAt: now }) + .onConflictDoNothing({ target: [userJobEvents.userId, userJobEvents.action, userJobEvents.opId] }); + } catch { + // Recording is best-effort; never block op creation on ledger writes. + } + + // Opportunistic prune of rows older than a generous retention window so the + // ledger stays small without a separate cron. + if (Math.random() < 0.05) { + try { + await safeDb() + .delete(userJobEvents) + .where(and( + eq(userJobEvents.action, action), + lt(userJobEvents.createdAt, now - 24 * 60 * 60 * 1000), + )); + } catch { + // ignore prune failures + } + } +} diff --git a/src/lib/server/rate-limit/problem-response.ts b/src/lib/server/rate-limit/problem-response.ts index 69b08ab..5fa9d3a 100644 --- a/src/lib/server/rate-limit/problem-response.ts +++ b/src/lib/server/rate-limit/problem-response.ts @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server'; import { type RateLimitResult } from '@/lib/server/rate-limit/rate-limiter'; +import { type JobRateDecision } from '@/lib/server/rate-limit/job-rate-limiter'; function formatLimitForHint(limit: number): string { if (!Number.isFinite(limit) || limit <= 0) return String(limit); @@ -45,3 +46,28 @@ export function buildDailyQuotaExceededResponse(input: { }, }); } + +/** + * 429 response for the compute job rate / concurrency limiter (e.g. PDF parse). + */ +export function buildComputeRateLimitedResponse(input: { + decision: JobRateDecision; + pathname: string; +}): NextResponse { + const retryAfterSeconds = Math.max(1, Math.ceil(input.decision.retryAfterMs / 1000)); + return new NextResponse(JSON.stringify({ + type: 'https://openreader.app/problems/compute-rate-limited', + title: 'Too many compute requests', + status: 429, + detail: 'You have started too many processing operations recently. Please wait and try again.', + code: 'COMPUTE_RATE_LIMITED', + retryAfterMs: input.decision.retryAfterMs, + instance: input.pathname, + }), { + status: 429, + headers: { + 'Content-Type': 'application/problem+json', + 'Retry-After': String(retryAfterSeconds), + }, + }); +} diff --git a/src/lib/server/rate-limit/request-ip.ts b/src/lib/server/rate-limit/request-ip.ts index 16d852e..2037790 100644 --- a/src/lib/server/rate-limit/request-ip.ts +++ b/src/lib/server/rate-limit/request-ip.ts @@ -1,26 +1,55 @@ import type { NextRequest } from 'next/server'; /** - * Best-effort client IP extraction that works on Vercel and typical reverse proxies. + * Best-effort client IP extraction for rate-limit backstops. * - * Note: IP-based limits are a backstop only; they are not perfectly reliable. + * Security note: `X-Forwarded-For` is a list the client can *prepend* to, so the + * left-most entry is attacker-controlled and must never be trusted as the client + * IP — doing so lets an attacker land every request in a fresh bucket and defeat + * the IP backstop entirely. + * + * Precedence (most trustworthy first): + * 1. x-vercel-forwarded-for — set by Vercel; inbound client copies are + * stripped, so it cannot be spoofed. Only consulted when running on Vercel. + * 2. x-real-ip / cf-connecting-ip — single values set by the reverse proxy + * (Vercel / Cloudflare overwrite any client-supplied copy). + * 3. x-forwarded-for — right-most hop only (the address seen by the closest + * trusted proxy), as a best-effort fallback for single-proxy self-hosts. + * 4. NextRequest.ip — runtime-provided connecting address. + * + * IP-based limits remain a backstop only; the per-user bucket is the primary + * control for authenticated abuse. */ +function firstIp(value: string | null): string | null { + if (!value) return null; + const first = value.split(',')[0]?.trim(); + return first || null; +} + export function getClientIp(req: NextRequest): string | null { - // Standard proxy header. Vercel also sets this. - const forwardedFor = req.headers.get('x-forwarded-for'); - if (forwardedFor) { - const first = forwardedFor.split(',')[0]?.trim(); - if (first) return first; + // 1. Vercel-internal header (clients cannot set x-vercel-* — Vercel strips them). + if (process.env.VERCEL) { + const vercelIp = firstIp(req.headers.get('x-vercel-forwarded-for')); + if (vercelIp) return vercelIp; } - const realIp = req.headers.get('x-real-ip'); - if (realIp) return realIp.trim(); + // 2. Single-value proxy headers set (and overwritten) by the edge. + const realIp = req.headers.get('x-real-ip')?.trim(); + if (realIp) return realIp; - // Some proxies use this. - const cfConnectingIp = req.headers.get('cf-connecting-ip'); - if (cfConnectingIp) return cfConnectingIp.trim(); + const cfConnectingIp = req.headers.get('cf-connecting-ip')?.trim(); + if (cfConnectingIp) return cfConnectingIp; - // NextRequest may expose ip depending on runtime. + // 3. Fall back to the RIGHT-most X-Forwarded-For hop (closest trusted proxy's + // view). Never the left-most, which the client controls. + const forwardedFor = req.headers.get('x-forwarded-for'); + if (forwardedFor) { + const parts = forwardedFor.split(',').map((part) => part.trim()).filter(Boolean); + const rightmost = parts[parts.length - 1]; + if (rightmost) return rightmost; + } + + // 4. Runtime-provided connecting address, when available. // eslint-disable-next-line @typescript-eslint/no-explicit-any const reqAny = req as any; const ip = typeof reqAny.ip === 'string' ? (reqAny.ip as string) : null; diff --git a/src/lib/server/testing/test-namespace.ts b/src/lib/server/testing/test-namespace.ts index eb9278a..044e46f 100644 --- a/src/lib/server/testing/test-namespace.ts +++ b/src/lib/server/testing/test-namespace.ts @@ -5,7 +5,24 @@ import { ensureSystemUserExists } from '@/db'; const TEST_NAMESPACE_HEADER = 'x-openreader-test-namespace'; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +/** + * The test-namespace header is test/CI scaffolding and must never be honored on + * a real production deployment. It is enabled when `ENABLE_TEST_NAMESPACE=true` + * (set explicitly by the Playwright web server / CI env) or, for local dev + * convenience, whenever the build is not a production build. + * + * Note: the E2E suite runs against a production build (`pnpm build && pnpm + * start`), so a `NODE_ENV` check alone is insufficient — CI relies on the + * explicit flag. + */ +function isTestNamespaceEnabled(): boolean { + if (process.env.ENABLE_TEST_NAMESPACE?.trim().toLowerCase() === 'true') return true; + return process.env.NODE_ENV !== 'production'; +} + export function getOpenReaderTestNamespace(headers: Headers): string | null { + if (!isTestNamespaceEnabled()) return null; + const raw = headers.get(TEST_NAMESPACE_HEADER)?.trim(); if (!raw) return null; diff --git a/tests/unit/request-ip.vitest.spec.ts b/tests/unit/request-ip.vitest.spec.ts new file mode 100644 index 0000000..ee51281 --- /dev/null +++ b/tests/unit/request-ip.vitest.spec.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from 'vitest'; +import type { NextRequest } from 'next/server'; +import { getClientIp } from '../../src/lib/server/rate-limit/request-ip'; +import { withEnv } from './support/env'; + +function makeReq(headers: Record, ip?: string): NextRequest { + const req = { headers: new Headers(headers) } as Partial & { ip?: string }; + if (ip) req.ip = ip; + return req as NextRequest; +} + +describe('getClientIp', () => { + test('never trusts the left-most (client-prependable) X-Forwarded-For entry', async () => { + await withEnv({ VERCEL: undefined }, () => { + // Attacker prepends a spoofed value; the real connecting IP is the right-most hop. + const req = makeReq({ 'x-forwarded-for': '1.2.3.4, 9.9.9.9' }); + expect(getClientIp(req)).toBe('9.9.9.9'); + }); + }); + + test('prefers x-real-ip over X-Forwarded-For', async () => { + await withEnv({ VERCEL: undefined }, () => { + const req = makeReq({ 'x-real-ip': '5.5.5.5', 'x-forwarded-for': '1.2.3.4, 9.9.9.9' }); + expect(getClientIp(req)).toBe('5.5.5.5'); + }); + }); + + test('prefers x-vercel-forwarded-for when running on Vercel', async () => { + await withEnv({ VERCEL: '1' }, () => { + const req = makeReq({ + 'x-vercel-forwarded-for': '8.8.8.8', + 'x-real-ip': '5.5.5.5', + 'x-forwarded-for': '1.2.3.4', + }); + expect(getClientIp(req)).toBe('8.8.8.8'); + }); + }); + + test('ignores x-vercel-forwarded-for when not on Vercel', async () => { + await withEnv({ VERCEL: undefined }, () => { + // A self-hosted deployment must not trust a client-supplied x-vercel-* header. + const req = makeReq({ 'x-vercel-forwarded-for': '8.8.8.8', 'x-real-ip': '5.5.5.5' }); + expect(getClientIp(req)).toBe('5.5.5.5'); + }); + }); + + test('falls back to the connecting address when no proxy headers are present', async () => { + await withEnv({ VERCEL: undefined }, () => { + const req = makeReq({}, '10.0.0.1'); + expect(getClientIp(req)).toBe('10.0.0.1'); + }); + }); +}); diff --git a/tests/unit/test-namespace-gate.vitest.spec.ts b/tests/unit/test-namespace-gate.vitest.spec.ts new file mode 100644 index 0000000..b2e4eb4 --- /dev/null +++ b/tests/unit/test-namespace-gate.vitest.spec.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from 'vitest'; +import { getOpenReaderTestNamespace } from '../../src/lib/server/testing/test-namespace'; +import { withEnv } from './support/env'; + +function headers(value: string): Headers { + return new Headers({ 'x-openreader-test-namespace': value }); +} + +describe('getOpenReaderTestNamespace gate', () => { + test('honored in non-production builds', async () => { + await withEnv({ NODE_ENV: 'development', ENABLE_TEST_NAMESPACE: undefined }, () => { + expect(getOpenReaderTestNamespace(headers('chromium'))).toBe('chromium'); + }); + }); + + test('ignored on production builds without the explicit flag', async () => { + await withEnv({ NODE_ENV: 'production', ENABLE_TEST_NAMESPACE: undefined }, () => { + expect(getOpenReaderTestNamespace(headers('attacker'))).toBeNull(); + }); + }); + + test('honored on production builds when ENABLE_TEST_NAMESPACE=true (CI parity)', async () => { + await withEnv({ NODE_ENV: 'production', ENABLE_TEST_NAMESPACE: 'true' }, () => { + expect(getOpenReaderTestNamespace(headers('webkit'))).toBe('webkit'); + }); + }); +}); From 3de33e2f1ea87714e0f9461d0a3642c9dd103bee Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 14:39:04 -0600 Subject: [PATCH 10/16] docs(config): update compute rate limit defaults to disabled for first boot change RUNTIME_SEED_DISABLE_COMPUTE_LIMIT default to true in env example, documentation, and runtime config schema to align with TTS rate limit behavior. clarify admin panel as the source of truth for enabling PDF parsing rate limits. --- .env.example | 5 +---- docs-site/docs/reference/environment-variables.md | 6 +++--- src/lib/server/admin/settings.ts | 5 +++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 0aebf94..6331ec1 100644 --- a/.env.example +++ b/.env.example @@ -124,10 +124,7 @@ S3_BUCKET= # RUNTIME_SEED_CHANGELOG_FEED_URL=https://docs.openreader.richardr.dev/changelog/manifest.json # RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT=true # RUNTIME_SEED_DISABLE_TTS_LIMIT=true -# Seeds the PDF parsing rate limit on first boot (default: enabled). All other -# compute-limit values (burst/sustained limits, windows) and the max upload size -# are configured in Settings → Admin, not via env. -# RUNTIME_SEED_DISABLE_COMPUTE_LIMIT=false +# RUNTIME_SEED_DISABLE_COMPUTE_LIMIT=true # Migrations configuration # (Optional) Skip automatic startup migrations when set to `false` (default: `true`) diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index cb9fa0d..2fc71f9 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -71,7 +71,7 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `RUNTIME_SEED_DEFAULT_TTS_PROVIDER` | Legacy bootstrap seed | `custom-openai` | Optional first-boot seed for default TTS provider slug | | `RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT` | Legacy bootstrap seed | `true` | Optional first-boot seed to enable audiobook export UI | | `RUNTIME_SEED_DISABLE_TTS_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps TTS daily rate limiting disabled | -| `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT` | Legacy bootstrap seed | `false` | Optional first-boot seed that disables PDF parsing rate limiting (other compute-limit values + max upload size are admin-only) | +| `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps PDF parsing rate limiting disabled (other compute-limit values + max upload size are admin-only) | | `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations | | `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass | @@ -496,7 +496,7 @@ Absolute path or executable name for the ffmpeg binary used by audiobook/process Per-user throttling of expensive PDF layout parsing is managed from **Settings → Admin → Site features**, not env vars. Enforcement applies only when auth is enabled. -- `disableComputeRateLimit` default: `false` (rate limiting enabled) +- `disableComputeRateLimit` default: `true` (rate limiting disabled, like the TTS limit; enable it in the admin panel) - When enabled, the following admin-tunable sub-limits apply: - `computeParseBurstMax` (default `8`) over `computeParseBurstWindowSec` (default `60`) - `computeParseSustainedMax` (default `24`) over `computeParseSustainedWindowSec` (default `600`) @@ -590,7 +590,7 @@ Seeds the TTS daily character rate-limit on/off state on first boot. Seeds the PDF parsing rate-limit on/off state on first boot. -- Default: `false` (PDF parsing rate limiting enabled) +- Default: `true` (PDF parsing rate limiting disabled, matching `RUNTIME_SEED_DISABLE_TTS_LIMIT`) - Runtime key: `disableComputeRateLimit` - The burst/sustained limits, their windows, and the max upload size are admin-only runtime settings (see [Compute (PDF Parsing) Rate Limiting](#compute-pdf-parsing-rate-limiting-runtime-settings)). diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index aacb273..e3b3f4b 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -111,11 +111,12 @@ export const RUNTIME_CONFIG_SCHEMA = { ttsDailyLimitAuthenticated: positiveIntValue(500_000), ttsIpDailyLimitAnonymous: positiveIntValue(100_000), ttsIpDailyLimitAuthenticated: positiveIntValue(1_000_000), - // Per-user throttle for expensive PDF-layout parsing. Enabled by default. + // Per-user throttle for expensive PDF-layout parsing. Disabled by default + // (admins enable it in Settings → Admin), mirroring disableTtsRateLimit. // When enabled, the sub-limits below apply (admin-tunable, no env seed): // a short "burst" window plus a wider "sustained" window that also bounds // concurrency (the worker caps each job's duration). - disableComputeRateLimit: booleanFlag(false, 'RUNTIME_SEED_DISABLE_COMPUTE_LIMIT'), + disableComputeRateLimit: booleanFlag(true, 'RUNTIME_SEED_DISABLE_COMPUTE_LIMIT'), computeParseBurstMax: positiveIntValue(8), computeParseBurstWindowSec: positiveIntValue(60), computeParseSustainedMax: positiveIntValue(24), From 8b45a4f6cdea467c36d6f8ac18f404bce7eeaf82 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 14:59:45 -0600 Subject: [PATCH 11/16] refactor(admin): group rate limit and upload settings in dedicated section move TTS and PDF parsing rate limit toggles and max upload size input into a new "Rate limiting" section in the admin panel for improved organization. update documentation to reflect the new grouping and clarify the relationship between these settings. --- docs-site/docs/configure/admin-panel.md | 23 +++++++----- src/components/admin/AdminFeaturesPanel.tsx | 39 ++++++++++++++------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/docs-site/docs/configure/admin-panel.md b/docs-site/docs/configure/admin-panel.md index bdb64ce..f8d32df 100644 --- a/docs-site/docs/configure/admin-panel.md +++ b/docs-site/docs/configure/admin-panel.md @@ -77,14 +77,6 @@ Runtime-editable settings, one row per key: | `enableAudiobookExport` | Show the audiobook export entry points on PDF/EPUB pages. | | `enableDocxConversion` | Accept .docx uploads (converted to PDF server-side). | | `enableDestructiveDeleteActions` | Show "Delete all data" buttons in the Documents tab (auth-disabled mode). | -| `disableTtsRateLimit` | Disable the per-user/IP daily TTS character limits. When `false`, the daily-limit fields below it apply. | -| `disableComputeRateLimit` | Disable per-user PDF parsing rate limiting. When `false`, the burst/sustained limit fields below it apply. | -| `maxUploadMb` | Maximum size (MB) accepted for a single document upload. Enforced server-side and signed into the presigned S3 PUT. | - -The **Disable TTS daily rate limiting** and **Disable PDF parsing rate limiting** toggles each reveal a collapsible group of numeric inputs when set to `false`: - -- TTS: anonymous/authenticated per-user daily limits and anonymous/authenticated IP daily backstops. -- PDF parsing: burst limit + window (seconds) and sustained limit + window (seconds). The sustained window doubles as a concurrency cap. Word-by-word highlighting and PDF layout parsing capability are controlled by compute-worker server env configuration, not an admin runtime flag. @@ -98,6 +90,21 @@ Each row shows a source badge: Turning `restrictUserApiKeys` off allows user-supplied API keys to flow through this server. Use this only for trusted/self-hosted deployments where that tradeoff is acceptable. ::: +## Rate limiting + +A dedicated **Rate limiting** group (within the same admin panel) collects the daily quotas, the PDF parsing throttle, and the upload size cap: + +| Key | What it controls | +| --- | --- | +| `disableTtsRateLimit` | Disable the per-user/IP daily TTS character limits. When `false`, the daily-limit fields below it apply. | +| `disableComputeRateLimit` | Disable per-user PDF parsing rate limiting. When `false`, the burst/sustained limit fields below it apply. | +| `maxUploadMb` | Maximum size (MB) accepted for a single document upload. Enforced server-side and signed into the presigned S3 PUT. | + +The **Disable TTS daily rate limiting** and **Disable PDF parsing rate limiting** toggles each reveal a collapsible group of numeric inputs when set to `false`: + +- TTS: anonymous/authenticated per-user daily limits and anonymous/authenticated IP daily backstops. +- PDF parsing: burst limit + window (seconds) and sustained limit + window (seconds). The sustained window doubles as a concurrency cap. + ## Migrating off env vars The future-direction goal is to remove `RUNTIME_SEED_*` / `API_KEY` / `API_BASE` from your `.env` entirely. To do that safely: diff --git a/src/components/admin/AdminFeaturesPanel.tsx b/src/components/admin/AdminFeaturesPanel.tsx index a450520..2facf72 100644 --- a/src/components/admin/AdminFeaturesPanel.tsx +++ b/src/components/admin/AdminFeaturesPanel.tsx @@ -280,6 +280,13 @@ export function AdminFeaturesPanel() { right={renderSource('showAllProviderModels')} variant="flat" /> +
+ +
Limits} + > ) : null} -
-
-
- - {renderSource('maxUploadMb')} +
+
+
+ Max upload size + Largest single document upload accepted. +
+
{renderSource('maxUploadMb')}
+
+ updatePositiveIntDraft('maxUploadMb', event.target.value)} + /> + MB
- updatePositiveIntDraft('maxUploadMb', event.target.value)} - />
From 60c1540744583e47ae757ae9dc856280c868ce4d Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 15:13:15 -0600 Subject: [PATCH 12/16] refactor(ui): decouple settings modal trigger from modal logic extract SettingsTrigger component to separate modal open control from SettingsModal. update HomeContent to manage modal state and pass open/onOpenChange props. remove redundant trigger button from SettingsModal for improved composability. removes embedded trigger logic from SettingsModal, enabling external state management and better integration with various UI layouts. also cleans up playwright config by centralizing testIgnore for unit tests. --- playwright.config.ts | 3 +-- src/components/HomeContent.tsx | 29 ++++++++++++--------- src/components/SettingsModal.tsx | 44 ++++++++++++++++++++------------ 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index 6cd95e5..07b27bb 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -16,6 +16,7 @@ if (process.env.CI === 'true') { */ export default defineConfig({ testDir: './tests', + testIgnore: '**/unit/**', tsconfig: './tsconfig.json', timeout: 30 * 1000, outputDir: './tests/results', @@ -60,7 +61,6 @@ export default defineConfig({ { name: 'firefox', - testIgnore: '**/unit/**', use: { ...devices['Desktop Firefox'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'firefox' }, @@ -69,7 +69,6 @@ export default defineConfig({ { name: 'webkit', - testIgnore: '**/unit/**', use: { ...devices['Desktop Safari'], extraHTTPHeaders: { 'x-openreader-test-namespace': 'webkit' }, diff --git a/src/components/HomeContent.tsx b/src/components/HomeContent.tsx index 8e9240f..e548230 100644 --- a/src/components/HomeContent.tsx +++ b/src/components/HomeContent.tsx @@ -1,7 +1,8 @@ 'use client'; +import { useState } from 'react'; import { DocumentList } from '@/components/doclist/DocumentList'; -import { SettingsModal } from '@/components/SettingsModal'; +import { SettingsModal, SettingsTrigger } from '@/components/SettingsModal'; import { UserMenu } from '@/components/auth/UserMenu'; const Brand = () => ( @@ -14,20 +15,24 @@ const Brand = () => ( ); -const AppActions = () => ( -
- - -
-); - export function HomeContent() { + const [settingsOpen, setSettingsOpen] = useState(false); + + const appActions = ( +
+ setSettingsOpen(true)} + /> + +
+ ); + return (
- } appActions={} /> + } appActions={appActions} /> +
); } diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index 50fee42..330f452 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -130,19 +130,42 @@ const SIDEBAR_SECTIONS: SidebarSection[] = [ type AdminSubTab = 'providers' | 'features'; -export function SettingsModal({ +export function SettingsTrigger({ className = '', triggerLabel, + onOpen, }: { className?: string; triggerLabel?: string; + onOpen: () => void; +}) { + return ( + + ); +} + +export function SettingsModal({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; }) { const runtimeConfig = useRuntimeConfig(); const enableDestructiveDelete = runtimeConfig.enableDestructiveDeleteActions; const showAllProviderModels = runtimeConfig.showAllProviderModels; const enableTTSProvidersTab = runtimeConfig.enableTtsProvidersTab; const restrictUserApiKeys = runtimeConfig.restrictUserApiKeys; - const [isOpen, setIsOpen] = useState(false); + const isOpen = open; + const setIsOpen = onOpenChange; const [isChangelogOpen, setIsChangelogOpen] = useState(false); const [activeSection, setActiveSection] = useState(enableTTSProvidersTab ? 'api' : 'theme'); @@ -215,7 +238,7 @@ export function SettingsModal({ if (changelogOpenSignal <= 0) return; setIsOpen(true); setIsChangelogOpen(true); - }, [changelogOpenSignal]); + }, [changelogOpenSignal, setIsOpen]); useEffect(() => { setLocalApiKey(apiKey); @@ -409,7 +432,7 @@ export function SettingsModal({ } else { setCustomModelInput(''); } - }, [apiKey, baseUrl, providerRef, providerType, ttsModel, ttsInstructions, ttsModels]); + }, [apiKey, baseUrl, providerRef, providerType, ttsModel, ttsInstructions, ttsModels, setIsOpen]); const [systemIsDark, setSystemIsDark] = useState( typeof window !== 'undefined' && window.matchMedia('(prefers-color-scheme: dark)').matches @@ -489,19 +512,6 @@ export function SettingsModal({ return ( <> - - Date: Sat, 30 May 2026 15:35:02 -0600 Subject: [PATCH 13/16] style(ui): improve bottom padding for safe area compatibility update bottom padding on banners, status bars, and player controls to use env(safe-area-inset-bottom) for better support on devices with display cutouts or home indicators. also add viewportFit: cover to layout for full-viewport rendering on mobile browsers. --- src/app/(app)/epub/[id]/page.tsx | 2 +- src/app/(app)/html/[id]/page.tsx | 2 +- src/app/(app)/pdf/[id]/page.tsx | 2 +- src/app/layout.tsx | 6 +++++- src/components/CookieConsentBanner.tsx | 2 +- src/components/doclist/window/FinderSidebar.tsx | 5 ++++- src/components/doclist/window/FinderStatusBar.tsx | 2 +- src/components/player/TTSPlayer.tsx | 2 +- 8 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index 57f4c0b..19e1806 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -243,7 +243,7 @@ export default function EPUBPage() { )} {isAtLimit ? (
-
+
diff --git a/src/app/(app)/html/[id]/page.tsx b/src/app/(app)/html/[id]/page.tsx index 21ad5c6..90fa49a 100644 --- a/src/app/(app)/html/[id]/page.tsx +++ b/src/app/(app)/html/[id]/page.tsx @@ -234,7 +234,7 @@ export default function HTMLPage() { )} {isAtLimit ? (
-
+
diff --git a/src/app/(app)/pdf/[id]/page.tsx b/src/app/(app)/pdf/[id]/page.tsx index 7695b4b..7635833 100644 --- a/src/app/(app)/pdf/[id]/page.tsx +++ b/src/app/(app)/pdf/[id]/page.tsx @@ -441,7 +441,7 @@ export default function PDFViewerPage() { )} {isAtLimit ? (
-
+
diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 1315dbe..e2dcdad 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -1,6 +1,6 @@ import "./globals.css"; import { ReactNode } from "react"; -import { Metadata } from "next"; +import { Metadata, Viewport } from "next"; import { Figtree } from "next/font/google"; import { ConsentAwareAnalytics } from "@/components/ConsentAwareAnalytics"; import { CookieConsentBanner } from "@/components/CookieConsentBanner"; @@ -30,6 +30,10 @@ const themeInitScript = ` })(); `; +export const viewport: Viewport = { + viewportFit: "cover", +}; + export const metadata: Metadata = { title: { default: "OpenReader", diff --git a/src/components/CookieConsentBanner.tsx b/src/components/CookieConsentBanner.tsx index c2c4608..97e5126 100644 --- a/src/components/CookieConsentBanner.tsx +++ b/src/components/CookieConsentBanner.tsx @@ -41,7 +41,7 @@ export function CookieConsentBanner() { leave="transition ease-in duration-200 transform" leaveFrom="translate-y-0 opacity-100" leaveTo="translate-y-full opacity-0" - className="fixed bottom-0 left-0 right-0 z-[60] p-4 md:p-6" + className="fixed bottom-0 left-0 right-0 z-[60] px-4 pt-4 pb-[max(1rem,env(safe-area-inset-bottom))] md:px-6 md:pt-6 md:pb-[max(1.5rem,env(safe-area-inset-bottom))]" >
diff --git a/src/components/doclist/window/FinderSidebar.tsx b/src/components/doclist/window/FinderSidebar.tsx index d6216cb..0ecf12c 100644 --- a/src/components/doclist/window/FinderSidebar.tsx +++ b/src/components/doclist/window/FinderSidebar.tsx @@ -354,7 +354,10 @@ export function FinderSidebar({
{bottomSlot && ( -
e.stopPropagation()}> +
e.stopPropagation()} + > {bottomSlot}
)} diff --git a/src/components/doclist/window/FinderStatusBar.tsx b/src/components/doclist/window/FinderStatusBar.tsx index e8c74de..acbd573 100644 --- a/src/components/doclist/window/FinderStatusBar.tsx +++ b/src/components/doclist/window/FinderStatusBar.tsx @@ -19,7 +19,7 @@ export function FinderStatusBar({
{summary} diff --git a/src/components/player/TTSPlayer.tsx b/src/components/player/TTSPlayer.tsx index c38dc23..2689747 100644 --- a/src/components/player/TTSPlayer.tsx +++ b/src/components/player/TTSPlayer.tsx @@ -32,7 +32,7 @@ export default function TTSPlayer({ currentPage, numPages }: { return (
-
+
{/* Speed control */} Date: Sat, 30 May 2026 15:40:34 -0600 Subject: [PATCH 14/16] fix(config): update vitest reporter for CI to github-actions --- vitest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vitest.config.ts b/vitest.config.ts index c3e8a47..2bf3656 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ }, test: { alias, - reporters: process.env.CI ? ['default', 'github'] : ['default'], + reporters: process.env.CI ? ['default', 'github-actions'] : ['default'], projects: [ { resolve: { From 36fe5f66a3e758551189c18b0cabf44f0aa52e8c Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 15:45:42 -0600 Subject: [PATCH 15/16] ci(workflow): add migration step to vitest CI pipeline include drizzle migration before running Vitest to ensure database schema is up to date during test execution --- .github/workflows/vitest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 1ed52d9..f69ef42 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -19,5 +19,7 @@ jobs: - uses: pnpm/action-setup@v6 - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Apply drizzle migrations (SQLite) + run: pnpm migrate - name: Run Vitest suites run: pnpm test:unit From aa7700e8447e041f0feb0af6da8b6f76c8993088 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 30 May 2026 16:41:17 -0600 Subject: [PATCH 16/16] refactor(core): improve streaming, validation, and rate limit logic - Stream request body in blob upload fallback route to enforce size limits without buffering entire payload - Enhance admin provider panel error handling for multi-status responses - Adjust positive integer validation to require value >= 1 - Use full rate limit config for job event recording and prune by largest window - Fix PDF layout job event userId usage in docx-to-pdf upload route - Add missing windows array fallback in documents register route - Minor CI workflow and env example corrections - Update audiobooks blobstore test to use beforeAll directly These changes improve efficiency, correctness, and maintainability across API, admin, rate limiting, and test modules. --- .env.example | 2 +- .github/workflows/vitest.yml | 2 ++ .../documents/blob/upload/fallback/route.ts | 34 +++++++++++++++++-- .../api/documents/docx-to-pdf/upload/route.ts | 2 +- src/app/api/documents/route.ts | 2 +- src/components/admin/AdminProvidersPanel.tsx | 16 ++++++++- src/lib/server/admin/settings.ts | 4 +-- src/lib/server/rate-limit/job-rate-limiter.ts | 13 ++++--- .../unit/audiobooks-blobstore.vitest.spec.ts | 4 +-- 9 files changed, 64 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 6331ec1..69c8ed0 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,7 @@ API_BASE=http://localhost:8880/v1 API_KEY=api_key_optional -# (Optional) TTS request/cache tuning (deaults shown below; leave unset to use defaults) +# (Optional) TTS request/cache tuning (defaults shown below; leave unset to use defaults) # TTS_CACHE_MAX_SIZE_BYTES=268435456 # 256MB # TTS_CACHE_TTL_MS=1800000 # 30 minutes # TTS_MAX_RETRIES=2 diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index f69ef42..27c2102 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + persist-credentials: false - uses: actions/setup-node@v5 with: node-version: lts/* diff --git a/src/app/api/documents/blob/upload/fallback/route.ts b/src/app/api/documents/blob/upload/fallback/route.ts index 74f67df..7fc8208 100644 --- a/src/app/api/documents/blob/upload/fallback/route.ts +++ b/src/app/api/documents/blob/upload/fallback/route.ts @@ -46,14 +46,42 @@ export async function PUT(req: NextRequest) { ); } - const body = Buffer.from(await req.arrayBuffer()); - // Backstop for chunked/omitted Content-Length: enforce on the actual bytes. - if (body.byteLength > maxUploadBytes) { + // Backstop for chunked/omitted Content-Length: stream the body so we can + // bail out as soon as the running total crosses the cap instead of + // buffering the entire payload first. + const stream = req.body; + if (!stream) { + return NextResponse.json({ error: 'Missing request body' }, { status: 400 }); + } + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + let overLimit = false; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + totalBytes += value.byteLength; + if (totalBytes > maxUploadBytes) { + overLimit = true; + break; + } + chunks.push(value); + } + } finally { + if (overLimit) { + await reader.cancel().catch(() => {}); + } + reader.releaseLock(); + } + if (overLimit) { return NextResponse.json( { error: `Upload exceeds the maximum allowed size of ${maxUploadBytes} bytes`, maxBytes: maxUploadBytes }, { status: 413 }, ); } + const body = Buffer.concat(chunks, totalBytes); const namespace = getOpenReaderTestNamespace(req.headers); diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 4d2b992..c849adf 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -177,7 +177,7 @@ export async function POST(req: NextRequest) { // Record upload-driven parse load (see register route for rationale). const pdfRateConfig = getPdfLayoutRateConfig(await getResolvedRuntimeConfig()); - await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig); + await recordJobEvent(storageUserId, 'pdf_layout', `docx:${randomUUID()}`, pdfRateConfig); enqueueParsePdfJob({ documentId: id, userId: storageUserId, diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index af9c38a..880b5c2 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -196,7 +196,7 @@ export async function POST(req: NextRequest) { // re-parse limiter reads. We record (not reject) here so a legitimate // bulk upload always parses; the recorded load still throttles // subsequent loopable re-parse spam via /parsed. - await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false }); + await recordJobEvent(ctxOrRes.userId, 'pdf_layout', `register:${randomUUID()}`, pdfRateConfig ?? { enabled: false, windows: [] }); enqueueParsePdfJob({ documentId: doc.id, userId: storageUserId, diff --git a/src/components/admin/AdminProvidersPanel.tsx b/src/components/admin/AdminProvidersPanel.tsx index b53b15f..0c64e97 100644 --- a/src/components/admin/AdminProvidersPanel.tsx +++ b/src/components/admin/AdminProvidersPanel.tsx @@ -72,7 +72,21 @@ async function patchDefaultProviderSlug(slug: string): Promise { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ updates: { defaultTtsProvider: slug } }), }); - if (!res.ok && res.status !== 207) throw new Error(`HTTP ${res.status}`); + if (res.status === 200 || res.status === 204) return; + if (res.status === 207) { + // Multi-status: the request succeeded for some keys but failed for others. + // Only treat it as success when the defaultTtsProvider field itself was + // accepted (no matching entry in the errors array). + const payload = (await res.json().catch(() => ({}))) as { + errors?: Array<{ key?: string; message?: string }>; + }; + const failure = payload.errors?.find((entry) => entry?.key === 'defaultTtsProvider'); + if (failure) { + throw new Error(failure.message || 'Failed to update default provider'); + } + return; + } + throw new Error(`HTTP ${res.status}`); } async function patchProviderEnabled(input: { id: string; enabled: boolean }): Promise { diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index e3b3f4b..1abbd34 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -84,11 +84,11 @@ function positiveIntValue(defaultValue: number, envVar?: string): RuntimeConfigK const trimmed = raw.trim(); if (!trimmed) return undefined; const parsed = Number(trimmed); - if (!Number.isFinite(parsed) || parsed <= 0) return undefined; + if (!Number.isFinite(parsed) || parsed < 1) return undefined; return Math.floor(parsed); }, validate(value) { - if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 1) return undefined; return Math.floor(value); }, }; diff --git a/src/lib/server/rate-limit/job-rate-limiter.ts b/src/lib/server/rate-limit/job-rate-limiter.ts index d65a183..f5a41e5 100644 --- a/src/lib/server/rate-limit/job-rate-limiter.ts +++ b/src/lib/server/rate-limit/job-rate-limiter.ts @@ -131,7 +131,7 @@ export async function recordJobEvent( userId: string | null | undefined, action: JobRateAction, opId: string, - config: Pick, + config: JobRateConfig, ): Promise { if (!isActive(config, userId) || !opId) return; @@ -145,15 +145,20 @@ export async function recordJobEvent( // Recording is best-effort; never block op creation on ledger writes. } - // Opportunistic prune of rows older than a generous retention window so the - // ledger stays small without a separate cron. + // Opportunistic prune of rows older than the largest configured window so + // the ledger stays small without a separate cron, but never deletes events + // that could still affect an in-window count. if (Math.random() < 0.05) { + const largestWindowMs = config.windows.reduce( + (max, w) => (Number.isFinite(w.windowMs) && w.windowMs > max ? w.windowMs : max), + 24 * 60 * 60 * 1000, + ); try { await safeDb() .delete(userJobEvents) .where(and( eq(userJobEvents.action, action), - lt(userJobEvents.createdAt, now - 24 * 60 * 60 * 1000), + lt(userJobEvents.createdAt, now - largestWindowMs), )); } catch { // ignore prune failures diff --git a/tests/unit/audiobooks-blobstore.vitest.spec.ts b/tests/unit/audiobooks-blobstore.vitest.spec.ts index 5f2c917..cb1ee96 100644 --- a/tests/unit/audiobooks-blobstore.vitest.spec.ts +++ b/tests/unit/audiobooks-blobstore.vitest.spec.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from 'vitest'; +import { beforeAll, describe, expect, test } from 'vitest'; import { audiobookKey, audiobookPrefix, @@ -15,7 +15,7 @@ function configureS3Env() { } describe('audiobooks-blobstore', () => { - test.beforeAll(() => { + beforeAll(() => { configureS3Env(); });