From e670c3852308cb14417e5918f2ed87f79c9870f6 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sun, 7 Jun 2026 13:33:20 -0600 Subject: [PATCH] Refactor user data cleanup and TTS storage flows (#105) * refactor(user): overhaul user data cleanup and export for cascading deletes and TTS segment support Revise user data cleanup logic to ensure proper cascading deletion of user-related database rows and S3 objects, including shared document and preview artifacts. Introduce explicit checks for last ownership before removing shared resources. Add TTS segment cache and audio cleanup to document and user deletion flows. Expand user data export to include TTS segment entries and audio, job events, document settings, and linked auth sessions. Update schema with ON DELETE CASCADE for userTtsChars and userJobEvents. Add new migration scripts and comprehensive unit tests for cleanup and export scenarios. * test(user): expand cleanup and export coverage for shared docs and TTS segment files Add tests for user data cleanup with shared document ownership and for TTS segment variant export scenarios, including cases with duplicate audio keys and storage disabled. Refactor cleanup logic to use document row types and transactional ownership removal with last-owner checks. Update TTS segment cache clearing to report deleted segment count by unique entry. Adjust export logic to always include all TTS segment variants and conditionally export file buckets based on storage availability. Update migration to use NOT VALID/VALIDATE for new cascading constraints. * refactor(documents): centralize owned document deletion and add mutation locking Introduce `deleteOwnedDocument` utility to encapsulate all logic for removing a user's ownership of a document, including TTS segment cache cleanup, preview artifact removal, and S3 blob deletion for last-owner cases. Add `withDocumentMutationLock` to serialize concurrent mutations on the same document, using advisory locks in Postgres and a local queue fallback. Refactor all API routes and user data flows to use these utilities, ensuring transactional safety and preventing race conditions during document deletion or transfer. Update tests for new flows and add coverage for document cleanup sequencing and locking behavior. * feat(data): enhance anonymous claim to support document settings and TTS segment copy Expand the anonymous data claim process to include document settings transfer and TTS segment S3 prefix copying. Remove foreign key constraints from user_tts_chars to allow non-user buckets. Update claim modal and onboarding flow to display claimed document settings. Refactor TTS char count transfer to merge all dates and fix upsert logic. Add S3 copy utility for TTS segments and corresponding tests. Update migrations and schema to reflect relaxed constraints. * refactor(data): improve error handling and rollback for document and TTS segment operations Enhance robustness of document deletion and TTS segment transfer by improving error logging, partial rollback, and degraded state reporting. Add best-effort cleanup and recovery mechanisms to prevent orphaned data and ensure diagnostic information is captured for unexpected failures. Update user data cleanup to log and swallow restoration errors without masking primary failures. Refine claim logic to handle unmapped TTS audio keys safely. * refactor(data): unify document mutation locking and transaction handling Replace mutation-lock with document-lock to centralize document mutation serialization and database transaction management. Introduce runInDbTransaction utility to abstract SQLite/Postgres transaction differences. Update document deletion, user data cleanup, and rate limiter logic to delegate transaction handling and locking to shared helpers. Remove dialect-specific branching and inline transaction logic for improved maintainability and testability. BREAKING CHANGE: withDocumentMutationLock is removed in favor of withDocumentLock and runInDbTransaction * fix(documents): improve rollback error handling in deleteOwnedDocument Enhance error handling during document deletion by ensuring that failures in restoring document ownership do not obscure the original error. Log degraded events when ownership restoration fails after a deletion error, providing additional context for debugging and monitoring. This change improves reliability and traceability of document deletion operations. * feat(tasks): introduce scheduled task engine and admin UI for background jobs Add a general-purpose scheduled task system with a persistent registry and status tracking, supporting background maintenance jobs such as orphaned blob reaping, expired upload cleanup, job event pruning, and TTS usage retention. Implement a new `scheduled_tasks` table, task engine, and handlers for each maintenance operation. Integrate an admin UI panel for monitoring, manual runs, and configuration of tasks. Update document and user data cleanup flows to delegate shared blob and preview deletion to the scheduled reaper. Add Vercel cron integration for serverless environments. BREAKING CHANGE: Document and user storage cleanup now relies on background scheduled tasks for shared blob and preview deletion; immediate inline deletion is no longer performed. * refactor(user): streamline document and TTS segment transfer logic Simplify user document transfer by consolidating storage and metadata handling. Remove inline deletion of shared blobs; only metadata is moved, and TTS segment transfer is controlled via options. Add `skipStorage` option for test scenarios to bypass storage operations. Update tests to reflect new transfer behavior. * chore(instrumentation): delegate node-specific setup to separate module Move Node.js-specific instrumentation logic to a dedicated file. Update registration to dynamically import the node module only when running in a Node.js environment. This separation clarifies environment-specific behavior and improves maintainability. * ci(config): update scheduled task cron to run daily at midnight Change cron schedule for /api/admin/tasks/tick from hourly to once daily at midnight to reduce task frequency and align with updated operational requirements. * refactor(tasks): enforce positive interval and improve scheduled task updates Add database-level check constraints to ensure scheduled task intervals are always positive for both Postgres and SQLite. Update admin API and UI to support sub-minute intervals and stricter validation. Refactor scheduled task update logic to upsert rows, ensuring tasks can be updated even if not yet present in the database. Improve temporary upload cleanup to delete in paginated batches. Enhance tests to cover new constraints and update behaviors. * chore(docker): remove ffmpeg-static from runtime dependencies in image build Eliminate ffmpeg-static from the production node_modules during Docker image assembly to streamline the deployment artifact and avoid bundling unused binaries. * refactor(admin): redesign task panel UI with Card layout and running indicator Replace the bordered div layout in AdminTasksPanel with the Card component for improved visual hierarchy. Add a dynamic running indicator using a custom RunningDot component to clearly show active tasks. Update control alignment and button states for better usability. Remove Badge-based status display in favor of a more streamlined appearance. * feat(tasks): add document blob lease for safe orphan reaping and improve scheduled task robustness Introduce a document blob lease mechanism to prevent race conditions between document registration and orphaned blob cleanup. The new `document_blob_leases` table ensures that only one process can claim a document blob for mutation or deletion at a time. Update the orphan reaper to acquire a lease before deleting blobs and to re-check for ownership after acquiring the lease, avoiding accidental deletion of in-flight uploads. Enhance scheduled task infrastructure with per-task fencing tokens to prevent stale runners from overwriting newer results, enforce runtime limits with abort signals, and expose scheduler mode and minimum interval to the admin panel and API. Adjust task handlers to accept a context with abort support, and update documentation and environment variable references for the new cron secret and scheduling behavior. BREAKING CHANGE: Scheduled tasks now require a `document_blob_leases` table and updated handler signatures. Vercel deployments must set `CRON_SECRET` for scheduled maintenance. * refactor(admin): add loading skeleton to tasks panel for improved UX Introduce a TasksSkeleton component to display animated placeholders while scheduled tasks are loading. Replace direct rendering of empty task rows with the skeleton when data is pending, enhancing perceived responsiveness and user experience in the admin tasks panel. * test: clean up playwright anonymous users * test: make blob lease expiry deterministic * fix(documents): improve blob lease release error handling and test stale lease scenario Handle errors during blob lease release by logging warnings instead of allowing them to mask original results. Update unit tests to verify that releasing a stale lease does not affect a replacement lease. * refactor(documents): implement exponential backoff with jitter for blob lease retries Replace fixed retry delay with capped exponential backoff and jitter to reduce contention and thundering herd effect when acquiring document blob leases. This improves lease acquisition fairness and efficiency under high load. --- .env.example | 4 + Dockerfile | 1 + .../worker/src/pdf-artifact-persistence.ts | 18 + compute/worker/src/runtime.ts | 52 +- .../unit/pdf-artifact-persistence.test.ts | 33 + docs-site/docs/configure/admin-panel.md | 11 + docs-site/docs/configure/tts-rate-limiting.md | 2 +- docs-site/docs/deploy/local-development.md | 4 + docs-site/docs/deploy/vercel-deployment.md | 18 +- docs-site/docs/docker-quick-start.md | 1 + .../docs/reference/environment-variables.md | 10 + .../0010_user-data-cleanup-cascades.sql | 5 + drizzle/postgres/0011_scheduled-tasks.sql | 22 + drizzle/postgres/meta/0010_snapshot.json | 1908 +++++++++++++++ drizzle/postgres/meta/0011_snapshot.json | 2039 +++++++++++++++++ drizzle/postgres/meta/_journal.json | 14 + .../0010_user-data-cleanup-cascades.sql | 18 + drizzle/sqlite/0011_scheduled-tasks.sql | 22 + drizzle/sqlite/meta/0010_snapshot.json | 1751 ++++++++++++++ drizzle/sqlite/meta/0011_snapshot.json | 1892 +++++++++++++++ drizzle/sqlite/meta/_journal.json | 14 + playwright.config.ts | 3 + src/app/api/account/delete/route.ts | 19 +- src/app/api/admin/tasks/[key]/route.ts | 62 + src/app/api/admin/tasks/[key]/run/route.ts | 23 + src/app/api/admin/tasks/route.ts | 18 + src/app/api/admin/tasks/tick/route.ts | 24 + .../api/documents/blob/get/fallback/route.ts | 9 +- .../documents/blob/preview/fallback/route.ts | 9 +- .../documents/blob/upload/finalize/route.ts | 82 +- .../documents/blob/upload/presign/route.ts | 5 +- src/app/api/documents/route.ts | 71 +- src/app/api/tts/segments/ensure/route.ts | 36 +- src/app/api/user/claim/route.ts | 8 +- src/app/api/user/export/route.ts | 111 +- src/components/SettingsModal.tsx | 7 +- src/components/admin/AdminTasksPanel.tsx | 286 +++ src/components/auth/ClaimDataModal.tsx | 8 +- src/contexts/OnboardingFlowContext.tsx | 8 +- src/db/run-in-transaction.ts | 23 + src/db/schema.ts | 2 + src/db/schema_postgres.ts | 29 +- src/db/schema_sqlite.ts | 29 +- src/instrumentation.node.ts | 5 + src/instrumentation.ts | 10 + src/lib/server/admin/settings.ts | 26 +- src/lib/server/audiobooks/blobstore.ts | 6 +- src/lib/server/auth/auth.ts | 145 +- src/lib/server/documents/blob-lease.ts | 95 + src/lib/server/documents/blobstore.ts | 136 +- src/lib/server/documents/delete-owned.ts | 41 + src/lib/server/documents/previews.ts | 5 - src/lib/server/documents/register-upload.ts | 19 + src/lib/server/rate-limit/job-rate-limiter.ts | 23 +- src/lib/server/rate-limit/rate-limiter.ts | 64 +- src/lib/server/tasks/engine.ts | 242 ++ .../tasks/handlers/cleanup-temp-uploads.ts | 17 + .../server/tasks/handlers/prune-job-events.ts | 25 + .../server/tasks/handlers/prune-tts-usage.ts | 10 + .../tasks/handlers/reap-orphaned-blobs.ts | 88 + src/lib/server/tasks/registry.ts | 37 + src/lib/server/tasks/scheduler.ts | 55 + src/lib/server/tasks/types.ts | 32 + src/lib/server/tts/segments-blobstore.ts | 56 +- src/lib/server/tts/segments-cache.ts | 44 +- src/lib/server/tts/segments.ts | 11 + src/lib/server/user/claim-data.ts | 266 ++- src/lib/server/user/data-cleanup.ts | 167 +- src/lib/server/user/data-export.ts | 91 +- tests/global-teardown.ts | 89 +- .../blob-upload-finalize-docx.vitest.spec.ts | 4 + tests/unit/data-export.vitest.spec.ts | 154 ++ tests/unit/document-blob-lease.vitest.spec.ts | 59 + .../document-delete-cleanup.vitest.spec.ts | 80 + tests/unit/reap-orphaned-blobs.vitest.spec.ts | 109 + .../scheduled-tasks-engine.vitest.spec.ts | 243 ++ .../scheduled-tasks-routes.vitest.spec.ts | 98 + .../storage-prefix-cleanup.vitest.spec.ts | 132 ++ .../transfer-user-documents.vitest.spec.ts | 179 +- .../tts-segments-blobstore.vitest.spec.ts | 76 + tests/unit/tts-segments-cache.vitest.spec.ts | 78 + tests/unit/tts-segments.vitest.spec.ts | 10 + tests/unit/user-data-cleanup.vitest.spec.ts | 134 ++ vercel.json | 8 + 84 files changed, 11357 insertions(+), 523 deletions(-) create mode 100644 compute/worker/src/pdf-artifact-persistence.ts create mode 100644 compute/worker/tests/unit/pdf-artifact-persistence.test.ts create mode 100644 drizzle/postgres/0010_user-data-cleanup-cascades.sql create mode 100644 drizzle/postgres/0011_scheduled-tasks.sql create mode 100644 drizzle/postgres/meta/0010_snapshot.json create mode 100644 drizzle/postgres/meta/0011_snapshot.json create mode 100644 drizzle/sqlite/0010_user-data-cleanup-cascades.sql create mode 100644 drizzle/sqlite/0011_scheduled-tasks.sql create mode 100644 drizzle/sqlite/meta/0010_snapshot.json create mode 100644 drizzle/sqlite/meta/0011_snapshot.json create mode 100644 src/app/api/admin/tasks/[key]/route.ts create mode 100644 src/app/api/admin/tasks/[key]/run/route.ts create mode 100644 src/app/api/admin/tasks/route.ts create mode 100644 src/app/api/admin/tasks/tick/route.ts create mode 100644 src/components/admin/AdminTasksPanel.tsx create mode 100644 src/db/run-in-transaction.ts create mode 100644 src/instrumentation.node.ts create mode 100644 src/instrumentation.ts create mode 100644 src/lib/server/documents/blob-lease.ts create mode 100644 src/lib/server/documents/delete-owned.ts create mode 100644 src/lib/server/tasks/engine.ts create mode 100644 src/lib/server/tasks/handlers/cleanup-temp-uploads.ts create mode 100644 src/lib/server/tasks/handlers/prune-job-events.ts create mode 100644 src/lib/server/tasks/handlers/prune-tts-usage.ts create mode 100644 src/lib/server/tasks/handlers/reap-orphaned-blobs.ts create mode 100644 src/lib/server/tasks/registry.ts create mode 100644 src/lib/server/tasks/scheduler.ts create mode 100644 src/lib/server/tasks/types.ts create mode 100644 tests/unit/data-export.vitest.spec.ts create mode 100644 tests/unit/document-blob-lease.vitest.spec.ts create mode 100644 tests/unit/document-delete-cleanup.vitest.spec.ts create mode 100644 tests/unit/reap-orphaned-blobs.vitest.spec.ts create mode 100644 tests/unit/scheduled-tasks-engine.vitest.spec.ts create mode 100644 tests/unit/scheduled-tasks-routes.vitest.spec.ts create mode 100644 tests/unit/storage-prefix-cleanup.vitest.spec.ts create mode 100644 tests/unit/tts-segments-blobstore.vitest.spec.ts create mode 100644 tests/unit/tts-segments-cache.vitest.spec.ts create mode 100644 tests/unit/user-data-cleanup.vitest.spec.ts create mode 100644 vercel.json diff --git a/.env.example b/.env.example index 173fc65..416bbd7 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,10 @@ AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional tr # (Optional) Comma-separated list of emails that are auto-promoted to admin. ADMIN_EMAILS= +# (Required for the Vercel scheduled-task cron route; Vercel sends it as a bearer token.) +# Self-hosted deployments run scheduled tasks in-process and do not require this. +# CRON_SECRET= + # (Optional) Backend DB used for server-side metadata (documents/audiobooks) and auth tables. # Defaults to SQLite at docstore/sqlite3.db when not set. # POSTGRES_URL= diff --git a/Dockerfile b/Dockerfile index bbc222b..c6b5ac6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ COPY --from=app-builder /opt/entrypoint-migration-tools/node_modules /tmp/runtim RUN mkdir -p /app/node_modules && \ rm -rf /tmp/runtime-tools-node_modules/@aws-sdk \ /tmp/runtime-tools-node_modules/better-sqlite3 \ + /tmp/runtime-tools-node_modules/ffmpeg-static \ /tmp/runtime-tools-node_modules/pg && \ cp -an /tmp/runtime-tools-node_modules/. /app/node_modules/ && \ rm -rf /tmp/runtime-tools-node_modules diff --git a/compute/worker/src/pdf-artifact-persistence.ts b/compute/worker/src/pdf-artifact-persistence.ts new file mode 100644 index 0000000..fff9f00 --- /dev/null +++ b/compute/worker/src/pdf-artifact-persistence.ts @@ -0,0 +1,18 @@ +export async function persistParsedPdfWhileSourceExists(input: { + sourceObjectKey: string; + sourceExists: (key: string) => Promise; + putParsedObject: () => Promise; + deleteParsedObject: (key: string) => Promise; +}): Promise { + if (!await input.sourceExists(input.sourceObjectKey)) { + throw new Error('PDF source object was deleted before parsed output could be persisted'); + } + + const parsedObjectKey = await input.putParsedObject(); + if (!await input.sourceExists(input.sourceObjectKey)) { + await input.deleteParsedObject(parsedObjectKey); + throw new Error('PDF source object was deleted while parsed output was being persisted'); + } + + return parsedObjectKey; +} diff --git a/compute/worker/src/runtime.ts b/compute/worker/src/runtime.ts index a00121d..442df53 100644 --- a/compute/worker/src/runtime.ts +++ b/compute/worker/src/runtime.ts @@ -49,7 +49,13 @@ import type { WorkerOperationState, PdfLayoutProgress, } from '@openreader/compute-core/api-contracts'; -import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { JetStreamOperationEventStream, OP_EVENTS_SUBJECT_WILDCARD, @@ -64,6 +70,7 @@ import { } from './orphan-recovery'; import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; +import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; const JOBS_STREAM_NAME = 'compute_jobs'; const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; @@ -701,6 +708,42 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti return toArrayBuffer(new Uint8Array(bytes)); }; + const objectExists = async (key: string): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + const safeKey = ensureSafeKey(key); + try { + await s3.send(new HeadObjectCommand({ + Bucket: s3Bucket, + Key: safeKey, + })); + return true; + } catch (error) { + const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; + if ( + maybe.$metadata?.httpStatusCode === 404 + || maybe.name === 'NotFound' + || maybe.name === 'NoSuchKey' + || maybe.Code === 'NotFound' + || maybe.Code === 'NoSuchKey' + ) { + return false; + } + throw error; + } + }; + + const deleteObjectByKey = async (key: string): Promise => { + if (!s3) { + throw new Error('S3 access is disabled for this worker app instance'); + } + await s3.send(new DeleteObjectCommand({ + Bucket: s3Bucket, + Key: ensureSafeKey(key), + })); + }; + const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { if (!s3) { throw new Error('S3 access is disabled for this worker app instance'); @@ -1175,7 +1218,12 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti phase: 'merge', }); } - const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); + const parsedObjectKey = await persistParsedPdfWhileSourceExists({ + sourceObjectKey: parsed.documentObjectKey, + sourceExists: objectExists, + putParsedObject: () => putParsedObject(parsed.documentId, parsed.namespace, result.parsed), + deleteParsedObject: deleteObjectByKey, + }); return { parsedObjectKey, timing: { diff --git a/compute/worker/tests/unit/pdf-artifact-persistence.test.ts b/compute/worker/tests/unit/pdf-artifact-persistence.test.ts new file mode 100644 index 0000000..9378435 --- /dev/null +++ b/compute/worker/tests/unit/pdf-artifact-persistence.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test, vi } from 'vitest'; +import { persistParsedPdfWhileSourceExists } from '../../src/pdf-artifact-persistence'; + +describe('PDF artifact persistence', () => { + test('does not write parsed output after the source was deleted', async () => { + const putParsedObject = vi.fn(async () => 'parsed.json'); + + await expect(persistParsedPdfWhileSourceExists({ + sourceObjectKey: 'document.pdf', + sourceExists: async () => false, + putParsedObject, + deleteParsedObject: async () => undefined, + })).rejects.toThrow('before parsed output'); + + expect(putParsedObject).not.toHaveBeenCalled(); + }); + + test('removes parsed output when the source disappears during the write', async () => { + const sourceExists = vi.fn() + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + const deleteParsedObject = vi.fn(async () => undefined); + + await expect(persistParsedPdfWhileSourceExists({ + sourceObjectKey: 'document.pdf', + sourceExists, + putParsedObject: async () => 'parsed.json', + deleteParsedObject, + })).rejects.toThrow('while parsed output'); + + expect(deleteParsedObject).toHaveBeenCalledWith('parsed.json'); + }); +}); diff --git a/docs-site/docs/configure/admin-panel.md b/docs-site/docs/configure/admin-panel.md index 82a70c4..f239d64 100644 --- a/docs-site/docs/configure/admin-panel.md +++ b/docs-site/docs/configure/admin-panel.md @@ -117,6 +117,17 @@ At the end of the **Site features** tab, a dedicated **TTS upstream** group cont In v4 these settings are admin-only and are no longer configurable through environment variables. +## Scheduled tasks + +The **Scheduled tasks** section controls background maintenance jobs such as expired-upload cleanup, orphaned-blob reaping, and rate-limit ledger pruning. + +- Enable or disable each task, adjust its interval, or run it immediately. +- Runs use database-backed leases so multiple app instances do not normally execute the same task concurrently. +- A task that exceeds four minutes is aborted and recorded as failed. A crashed run can be reclaimed after its stale lease expires. +- Failures and the latest successful summary appear on the task card and in server logs. + +Self-hosted Node.js deployments tick the scheduler in-process once per minute. Vercel uses the authenticated `/api/admin/tasks/tick` cron route; the checked-in Vercel Hobby schedule runs once daily, so intervals shorter than one day are unavailable there. See [Vercel Deployment](../deploy/vercel-deployment#5-scheduled-maintenance-tasks). + ## Migrating off env vars In v4, runtime site features are managed by admin settings and optional JSON seed. To minimize env surface area: diff --git a/docs-site/docs/configure/tts-rate-limiting.md b/docs-site/docs/configure/tts-rate-limiting.md index 967124c..0ceac10 100644 --- a/docs-site/docs/configure/tts-rate-limiting.md +++ b/docs-site/docs/configure/tts-rate-limiting.md @@ -37,6 +37,6 @@ If a request exceeds the active limit, the TTS API returns `429` with reset meta ## Related docs - TTS/rate-limit environment variables: [Environment Variables](../reference/environment-variables#tts-provider-and-request-behavior) -- PDF parsing rate limiting (separate, compute-side throttle): [Admin Panel → Site features](./admin-panel#site-features) and [Environment Variables](../reference/environment-variables#compute-pdf-parsing-rate-limiting-runtime-settings) +- PDF parsing rate limiting (separate, compute-side throttle): [Admin Panel → Rate limiting](./admin-panel#rate-limiting) - Auth configuration: [Auth](./auth) - Provider setup: [TTS Providers](./tts-providers) diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index 3f5f0b1..dd38dcd 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -325,6 +325,10 @@ Storage configuration details are in [Object / Blob Storage](../configure/object Refer to [Database](../configure/database) for database modes. Learn about migration behavior and commands in [Migrations](../configure/migrations). +:::info Scheduled maintenance tasks +Local and self-hosted Node.js deployments start the scheduled-task loop in-process and check for due work once per minute. No `CRON_SECRET` is required unless you intentionally invoke the cron HTTP route yourself. Manage task intervals and inspect failures from **Settings → Admin → Scheduled tasks**. +::: + 4. Start the app. diff --git a/docs-site/docs/deploy/vercel-deployment.md b/docs-site/docs/deploy/vercel-deployment.md index eeedbcd..8158c10 100644 --- a/docs-site/docs/deploy/vercel-deployment.md +++ b/docs-site/docs/deploy/vercel-deployment.md @@ -36,6 +36,7 @@ S3_PREFIX=openreader BASE_URL=https://your-app.vercel.app AUTH_SECRET=... ADMIN_EMAILS=you@example.com # comma-separated; admins manage TTS + features in-app +CRON_SECRET=... # generate with: openssl rand -hex 32 # Heavy compute (required on Vercel in current releases) COMPUTE_WORKER_URL=https:// @@ -110,7 +111,15 @@ Vercel deployments do not run `scripts/openreader-entrypoint.mjs`, so automatic - Run `pnpm migrate` in a controlled environment to apply Drizzle schema migrations to your Postgres DB. - Run `pnpm migrate-fs` only when migrating legacy local filesystem data (`docstore/documents_v1`, `docstore/audiobooks_v1`) into object storage + DB rows. Fresh Vercel deployments usually do not need this. -## 5. FFmpeg packaging in Vercel functions +## 5. Scheduled maintenance tasks + +The repository configures `/api/admin/tasks/tick` as a Vercel Cron route. Set `CRON_SECRET`; requests without the matching bearer token are rejected. + +The checked-in Hobby-compatible schedule invokes the route once daily. The admin task panel therefore prevents selecting intervals shorter than one day on Vercel, even though self-hosted deployments can run tasks more frequently. + +Each due task is claimed with a database-backed lease, due tasks start independently, and individual runs are aborted and marked failed after four minutes. Review failures and run tasks manually from **Settings → Admin → Scheduled tasks**. + +## 6. FFmpeg packaging in Vercel functions `ffmpeg-static` binaries must be included in function traces. This repo already does that in `next.config.ts` via `outputFileTracingIncludes` for: @@ -124,7 +133,7 @@ Vercel deployments do not run `scripts/openreader-entrypoint.mjs`, so automatic If you change route paths or split handlers, update `outputFileTracingIncludes` accordingly. -## 6. Function memory sizing +## 7. Function memory sizing FFmpeg workloads benefit from more memory/CPU. This repo includes: @@ -140,14 +149,15 @@ FFmpeg workloads benefit from more memory/CPU. This repo includes: Adjust memory per route if your files are larger or your plan differs. -## 7. Runtime expectations and caveats +## 8. Runtime expectations and caveats - Audiobook APIs require S3 configuration; otherwise they return `503`. - For production Vercel deploys, use `POSTGRES_URL` instead of SQLite. -## 8. Smoke test after deploy +## 9. Smoke test after deploy 1. Upload and read a PDF/EPUB document. 2. Confirm sync/blob fetch works across refreshes/devices. 3. Generate at least one audiobook chapter and play/download it. 4. Verify worker-backed word highlighting and PDF parsing. +5. Open **Settings → Admin → Scheduled tasks**, run one task manually, and confirm the next daily cron invocation succeeds. diff --git a/docs-site/docs/docker-quick-start.md b/docs-site/docs/docker-quick-start.md index e9e1cc0..3495443 100644 --- a/docs-site/docs/docker-quick-start.md +++ b/docs-site/docs/docker-quick-start.md @@ -122,6 +122,7 @@ What this command enables: - `restrictUserApiKeys` controls shared-provider-only mode. For per-user BYOK, toggle it off in **Settings → Admin → Site features** or seed `runtimeConfig.restrictUserApiKeys=false` via runtime seed JSON. - Use a `/app/docstore` mount if you want data to survive container/image replacement. - Startup automatically runs DB/storage migrations via the shared entrypoint. +- Scheduled maintenance tasks run in-process and can be managed from **Settings → Admin → Scheduled tasks**; Docker/self-hosted deployments do not need `CRON_SECRET`. ::: :::warning Port `8333` Exposure diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index dfe985c..a280903 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -26,6 +26,7 @@ Runtime site features are seeded with `RUNTIME_SEED_JSON` / `RUNTIME_SEED_JSON_P | `GITHUB_CLIENT_ID` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_SECRET` to enable GitHub sign-in | | `GITHUB_CLIENT_SECRET` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_ID` to enable GitHub sign-in | | `ADMIN_EMAILS` | Admin | empty | Comma-separated emails auto-promoted to admin | +| `CRON_SECRET` | Scheduled tasks | unset | Required for Vercel cron invocations | | `POSTGRES_URL` | Database | unset (SQLite mode) | Set to switch metadata/auth DB to Postgres | | `USE_EMBEDDED_WEED_MINI` | Storage | `true` when unset | Set `false` to use external S3-compatible storage only | | `WEED_MINI_DIR` | Storage | `docstore/seaweedfs` | Override embedded SeaweedFS data directory | @@ -144,6 +145,15 @@ Comma-separated list of email addresses auto-promoted to admin. - Requires auth to be enabled - Admins can manage shared providers and runtime site features in-app +### CRON_SECRET + +Bearer-token secret for `GET /api/admin/tasks/tick`. + +- Required on Vercel so scheduled maintenance tasks can run from the configured Vercel Cron. +- Vercel automatically sends `Authorization: Bearer ` on cron invocations. +- Generate a strong random value, for example with `openssl rand -hex 32`. +- Self-hosted Node.js deployments run the scheduler in-process and do not require this variable. + ## Database and Object Blob Storage ### POSTGRES_URL diff --git a/drizzle/postgres/0010_user-data-cleanup-cascades.sql b/drizzle/postgres/0010_user-data-cleanup-cascades.sql new file mode 100644 index 0000000..463adbd --- /dev/null +++ b/drizzle/postgres/0010_user-data-cleanup-cascades.sql @@ -0,0 +1,5 @@ +ALTER TABLE "user_job_events" ADD CONSTRAINT "user_job_events_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action NOT VALID;--> statement-breakpoint +DELETE FROM "user_job_events" WHERE NOT EXISTS ( + SELECT 1 FROM "user" WHERE "user"."id" = "user_job_events"."user_id" +);--> statement-breakpoint +ALTER TABLE "user_job_events" VALIDATE CONSTRAINT "user_job_events_user_id_user_id_fk"; \ No newline at end of file diff --git a/drizzle/postgres/0011_scheduled-tasks.sql b/drizzle/postgres/0011_scheduled-tasks.sql new file mode 100644 index 0000000..3714be8 --- /dev/null +++ b/drizzle/postgres/0011_scheduled-tasks.sql @@ -0,0 +1,22 @@ +CREATE TABLE "document_blob_leases" ( + "document_id" text PRIMARY KEY NOT NULL, + "lease_owner" text NOT NULL, + "lease_until_ms" bigint NOT NULL +); +--> statement-breakpoint +CREATE TABLE "scheduled_tasks" ( + "key" text PRIMARY KEY NOT NULL, + "enabled" boolean DEFAULT true NOT NULL, + "interval_ms" bigint NOT NULL, + "last_status" text DEFAULT 'idle' NOT NULL, + "lease_owner" text, + "last_run_at" bigint, + "last_duration_ms" bigint, + "last_error" text, + "last_result_json" text, + "next_run_at" bigint, + "run_requested" boolean DEFAULT false NOT NULL, + "running_since" bigint, + "updated_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint NOT NULL, + CONSTRAINT "scheduled_tasks_interval_ms_positive" CHECK ("scheduled_tasks"."interval_ms" > 0) +); diff --git a/drizzle/postgres/meta/0010_snapshot.json b/drizzle/postgres/meta/0010_snapshot.json new file mode 100644 index 0000000..a85fbaa --- /dev/null +++ b/drizzle/postgres/meta/0010_snapshot.json @@ -0,0 +1,1908 @@ +{ + "id": "1099cbe6-0c2e-4ea8-ae3c-6883c931169d", + "prevId": "23b90a4b-fb63-4f9a-9230-09ca218d0b78", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admin_providers": { + "name": "admin_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_settings": { + "name": "admin_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value_json": { + "name": "value_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobook_chapters": { + "name": "audiobook_chapters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "name": "audiobook_chapters_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobooks": { + "name": "audiobooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "name": "audiobooks_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_previews": { + "name": "document_previews", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_until_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "name": "document_previews_document_id_namespace_variant_pk", + "columns": [ + "document_id", + "namespace", + "variant" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_settings": { + "name": "document_settings", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "name": "document_settings_document_id_user_id_pk", + "columns": [ + "document_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_modified": { + "name": "last_modified", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_modified", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "name": "documents_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_entries": { + "name": "tts_segment_entries", + "schema": "", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_version": { + "name": "document_version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_reader_rank", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_char_offset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_href", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_location", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "name": "tts_segment_entries_segment_entry_id_user_id_pk", + "columns": [ + "segment_entry_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_variants": { + "name": "tts_segment_variants", + "schema": "", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_json": { + "name": "settings_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "settings_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "name": "tts_segment_variants_segment_id_user_id_pk", + "columns": [ + "segment_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_document_progress": { + "name": "user_document_progress", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "name": "user_document_progress_user_id_document_id_pk", + "columns": [ + "user_id", + "document_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_job_events": { + "name": "user_job_events", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_job_events_user_id_user_id_fk": { + "name": "user_job_events_user_id_user_id_fk", + "tableFrom": "user_job_events", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "name": "user_job_events_user_id_action_op_id_pk", + "columns": [ + "user_id", + "action", + "op_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preferences": { + "name": "user_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_tts_chars": { + "name": "user_tts_chars", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "name": "user_tts_chars_user_id_date_pk", + "columns": [ + "user_id", + "date" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/postgres/meta/0011_snapshot.json b/drizzle/postgres/meta/0011_snapshot.json new file mode 100644 index 0000000..ebf8389 --- /dev/null +++ b/drizzle/postgres/meta/0011_snapshot.json @@ -0,0 +1,2039 @@ +{ + "id": "6fa9be62-3d2b-43d2-8c49-11d9c6411a33", + "prevId": "1099cbe6-0c2e-4ea8-ae3c-6883c931169d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admin_providers": { + "name": "admin_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_settings": { + "name": "admin_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value_json": { + "name": "value_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobook_chapters": { + "name": "audiobook_chapters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "name": "audiobook_chapters_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobooks": { + "name": "audiobooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "name": "audiobooks_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_blob_leases": { + "name": "document_blob_leases", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_previews": { + "name": "document_previews", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_until_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "name": "document_previews_document_id_namespace_variant_pk", + "columns": [ + "document_id", + "namespace", + "variant" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_settings": { + "name": "document_settings", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "name": "document_settings_document_id_user_id_pk", + "columns": [ + "document_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_modified": { + "name": "last_modified", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_modified", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "name": "documents_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scheduled_tasks": { + "name": "scheduled_tasks", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "interval_ms": { + "name": "interval_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'idle'" + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_duration_ms": { + "name": "last_duration_ms", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_result_json": { + "name": "last_result_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "run_requested": { + "name": "run_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "running_since": { + "name": "running_since", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "scheduled_tasks_interval_ms_positive": { + "name": "scheduled_tasks_interval_ms_positive", + "value": "\"scheduled_tasks\".\"interval_ms\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.tts_segment_entries": { + "name": "tts_segment_entries", + "schema": "", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_version": { + "name": "document_version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_reader_rank", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_char_offset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_href", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_location", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "name": "tts_segment_entries_segment_entry_id_user_id_pk", + "columns": [ + "segment_entry_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_variants": { + "name": "tts_segment_variants", + "schema": "", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_json": { + "name": "settings_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "settings_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "name": "tts_segment_variants_segment_id_user_id_pk", + "columns": [ + "segment_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_document_progress": { + "name": "user_document_progress", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "name": "user_document_progress_user_id_document_id_pk", + "columns": [ + "user_id", + "document_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_job_events": { + "name": "user_job_events", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_job_events_user_id_user_id_fk": { + "name": "user_job_events_user_id_user_id_fk", + "tableFrom": "user_job_events", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "name": "user_job_events_user_id_action_op_id_pk", + "columns": [ + "user_id", + "action", + "op_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preferences": { + "name": "user_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_tts_chars": { + "name": "user_tts_chars", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "name": "user_tts_chars_user_id_date_pk", + "columns": [ + "user_id", + "date" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index a2a3318..cfe7d84 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -71,6 +71,20 @@ "when": 1780625663880, "tag": "0009_drop_pdf_parse", "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1780794520924, + "tag": "0010_user-data-cleanup-cascades", + "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1780851107819, + "tag": "0011_scheduled-tasks", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/sqlite/0010_user-data-cleanup-cascades.sql b/drizzle/sqlite/0010_user-data-cleanup-cascades.sql new file mode 100644 index 0000000..be0e21c --- /dev/null +++ b/drizzle/sqlite/0010_user-data-cleanup-cascades.sql @@ -0,0 +1,18 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +DELETE FROM `user_job_events` WHERE NOT EXISTS ( + SELECT 1 FROM `user` WHERE `user`.`id` = `user_job_events`.`user_id` +);--> statement-breakpoint +CREATE TABLE `__new_user_job_events` ( + `user_id` text NOT NULL, + `action` text NOT NULL, + `op_id` text NOT NULL, + `created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + PRIMARY KEY(`user_id`, `action`, `op_id`), + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +INSERT INTO `__new_user_job_events`("user_id", "action", "op_id", "created_at") SELECT "user_id", "action", "op_id", "created_at" FROM `user_job_events`;--> statement-breakpoint +DROP TABLE `user_job_events`;--> statement-breakpoint +ALTER TABLE `__new_user_job_events` RENAME TO `user_job_events`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `idx_user_job_events_user_action_created` ON `user_job_events` (`user_id`,`action`,`created_at`); \ No newline at end of file diff --git a/drizzle/sqlite/0011_scheduled-tasks.sql b/drizzle/sqlite/0011_scheduled-tasks.sql new file mode 100644 index 0000000..de29632 --- /dev/null +++ b/drizzle/sqlite/0011_scheduled-tasks.sql @@ -0,0 +1,22 @@ +CREATE TABLE `document_blob_leases` ( + `document_id` text PRIMARY KEY NOT NULL, + `lease_owner` text NOT NULL, + `lease_until_ms` integer NOT NULL +); +--> statement-breakpoint +CREATE TABLE `scheduled_tasks` ( + `key` text PRIMARY KEY NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `interval_ms` integer NOT NULL, + `last_status` text DEFAULT 'idle' NOT NULL, + `lease_owner` text, + `last_run_at` integer, + `last_duration_ms` integer, + `last_error` text, + `last_result_json` text, + `next_run_at` integer, + `run_requested` integer DEFAULT false NOT NULL, + `running_since` integer, + `updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + CONSTRAINT "scheduled_tasks_interval_ms_positive" CHECK("scheduled_tasks"."interval_ms" > 0) +); diff --git a/drizzle/sqlite/meta/0010_snapshot.json b/drizzle/sqlite/meta/0010_snapshot.json new file mode 100644 index 0000000..ed9acdb --- /dev/null +++ b/drizzle/sqlite/meta/0010_snapshot.json @@ -0,0 +1,1751 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "a32ae7d4-c074-44f0-9e90-2473949376ef", + "prevId": "2c23f770-269c-4cae-a71e-4ef88355681f", + "tables": { + "admin_providers": { + "name": "admin_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "admin_settings": { + "name": "admin_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobook_chapters": { + "name": "audiobook_chapters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobook_chapters_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobooks": { + "name": "audiobooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobooks_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_previews": { + "name": "document_previews", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + "status", + "lease_until_ms" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "columns": [ + "document_id", + "namespace", + "variant" + ], + "name": "document_previews_document_id_namespace_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_settings": { + "name": "document_settings", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "columns": [ + "document_id", + "user_id" + ], + "name": "document_settings_document_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + "user_id", + "last_modified" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "documents_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_entries": { + "name": "tts_segment_entries", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_version": { + "name": "document_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + "user_id", + "document_id", + "document_version", + "locator_reader_rank", + "locator_spine_index", + "locator_char_offset", + "locator_spine_href", + "locator_page", + "locator_location", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + "user_id", + "document_id", + "document_version", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + "user_id", + "document_id", + "document_version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "columns": [ + "segment_entry_id", + "user_id" + ], + "name": "tts_segment_entries_segment_entry_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_variants": { + "name": "tts_segment_variants", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + "user_id", + "segment_entry_id", + "updated_at" + ], + "isUnique": false + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + "user_id", + "status" + ], + "isUnique": false + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + "user_id", + "segment_entry_id", + "settings_hash" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "columns": [ + "segment_id", + "user_id" + ], + "name": "tts_segment_variants_segment_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_document_progress": { + "name": "user_document_progress", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + "user_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "columns": [ + "user_id", + "document_id" + ], + "name": "user_document_progress_user_id_document_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_job_events": { + "name": "user_job_events", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + "user_id", + "action", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_job_events_user_id_user_id_fk": { + "name": "user_job_events_user_id_user_id_fk", + "tableFrom": "user_job_events", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "columns": [ + "user_id", + "action", + "op_id" + ], + "name": "user_job_events_user_id_action_op_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_preferences": { + "name": "user_preferences", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_tts_chars": { + "name": "user_tts_chars", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + "date" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "columns": [ + "user_id", + "date" + ], + "name": "user_tts_chars_user_id_date_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/sqlite/meta/0011_snapshot.json b/drizzle/sqlite/meta/0011_snapshot.json new file mode 100644 index 0000000..66d871f --- /dev/null +++ b/drizzle/sqlite/meta/0011_snapshot.json @@ -0,0 +1,1892 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d063930a-ae25-4ca0-8b38-7092a16e3503", + "prevId": "a32ae7d4-c074-44f0-9e90-2473949376ef", + "tables": { + "admin_providers": { + "name": "admin_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "admin_settings": { + "name": "admin_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobook_chapters": { + "name": "audiobook_chapters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobook_chapters_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobooks": { + "name": "audiobooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobooks_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_blob_leases": { + "name": "document_blob_leases", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_previews": { + "name": "document_previews", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + "status", + "lease_until_ms" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "columns": [ + "document_id", + "namespace", + "variant" + ], + "name": "document_previews_document_id_namespace_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_settings": { + "name": "document_settings", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "columns": [ + "document_id", + "user_id" + ], + "name": "document_settings_document_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + "user_id", + "last_modified" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "documents_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduled_tasks": { + "name": "scheduled_tasks", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "interval_ms": { + "name": "interval_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_duration_ms": { + "name": "last_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_result_json": { + "name": "last_result_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "run_requested": { + "name": "run_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "running_since": { + "name": "running_since", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "scheduled_tasks_interval_ms_positive": { + "name": "scheduled_tasks_interval_ms_positive", + "value": "\"scheduled_tasks\".\"interval_ms\" > 0" + } + } + }, + "tts_segment_entries": { + "name": "tts_segment_entries", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_version": { + "name": "document_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + "user_id", + "document_id", + "document_version", + "locator_reader_rank", + "locator_spine_index", + "locator_char_offset", + "locator_spine_href", + "locator_page", + "locator_location", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + "user_id", + "document_id", + "document_version", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + "user_id", + "document_id", + "document_version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "columns": [ + "segment_entry_id", + "user_id" + ], + "name": "tts_segment_entries_segment_entry_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_variants": { + "name": "tts_segment_variants", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + "user_id", + "segment_entry_id", + "updated_at" + ], + "isUnique": false + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + "user_id", + "status" + ], + "isUnique": false + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + "user_id", + "segment_entry_id", + "settings_hash" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "columns": [ + "segment_id", + "user_id" + ], + "name": "tts_segment_variants_segment_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_document_progress": { + "name": "user_document_progress", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + "user_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "columns": [ + "user_id", + "document_id" + ], + "name": "user_document_progress_user_id_document_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_job_events": { + "name": "user_job_events", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_job_events_user_action_created": { + "name": "idx_user_job_events_user_action_created", + "columns": [ + "user_id", + "action", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_job_events_user_id_user_id_fk": { + "name": "user_job_events_user_id_user_id_fk", + "tableFrom": "user_job_events", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_job_events_user_id_action_op_id_pk": { + "columns": [ + "user_id", + "action", + "op_id" + ], + "name": "user_job_events_user_id_action_op_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_preferences": { + "name": "user_preferences", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_tts_chars": { + "name": "user_tts_chars", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + "date" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "columns": [ + "user_id", + "date" + ], + "name": "user_tts_chars_user_id_date_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index 3dc4ee8..3222938 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -71,6 +71,20 @@ "when": 1780625663601, "tag": "0009_drop_pdf_parse", "breakpoints": true + }, + { + "idx": 10, + "version": "6", + "when": 1780794515094, + "tag": "0010_user-data-cleanup-cascades", + "breakpoints": true + }, + { + "idx": 11, + "version": "6", + "when": 1780851107505, + "tag": "0011_scheduled-tasks", + "breakpoints": true } ] } \ No newline at end of file diff --git a/playwright.config.ts b/playwright.config.ts index d38c61e..1135007 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -45,6 +45,7 @@ export default defineConfig({ name: 'chromium', use: { ...devices['Desktop Chrome'], + userAgent: `${devices['Desktop Chrome'].userAgent} OpenReader-Playwright/chromium`, extraHTTPHeaders: { 'x-openreader-test-namespace': 'chromium' }, }, }, @@ -53,6 +54,7 @@ export default defineConfig({ name: 'firefox', use: { ...devices['Desktop Firefox'], + userAgent: `${devices['Desktop Firefox'].userAgent} OpenReader-Playwright/firefox`, extraHTTPHeaders: { 'x-openreader-test-namespace': 'firefox' }, }, }, @@ -61,6 +63,7 @@ export default defineConfig({ name: 'webkit', use: { ...devices['Desktop Safari'], + userAgent: `${devices['Desktop Safari'].userAgent} OpenReader-Playwright/webkit`, extraHTTPHeaders: { 'x-openreader-test-namespace': 'webkit' }, }, }, diff --git a/src/app/api/account/delete/route.ts b/src/app/api/account/delete/route.ts index 26e1a0f..e0130bc 100644 --- a/src/app/api/account/delete/route.ts +++ b/src/app/api/account/delete/route.ts @@ -3,7 +3,7 @@ import { NextResponse } from 'next/server'; import { auth } from '@/lib/server/auth/auth'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { deleteUserStorageData } from '@/lib/server/user/data-cleanup'; -import { errorToLog, hashForLog, serverLogger } from '@/lib/server/logger'; +import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; export async function DELETE() { @@ -18,21 +18,12 @@ export async function DELETE() { } try { - // Best-effort cleanup for test namespaced storage using request context. - // The Better Auth beforeDelete hook still runs and handles non-namespaced data. + // Clean test-namespaced storage using request context. The Better Auth + // beforeDelete hook handles non-namespaced storage and blocks deletion if + // either cleanup cannot complete. const testNamespace = getOpenReaderTestNamespace(reqHeaders); if (testNamespace) { - try { - await deleteUserStorageData(session.user.id, testNamespace); - } catch (error) { - serverLogger.warn({ - event: 'account.delete.storage_cleanup_failed', - degraded: true, - step: 'namespaced_storage_cleanup', - userIdHash: hashForLog(session.user.id), - error: errorToLog(error), - }, 'Failed to clean up namespaced user storage before deletion'); - } + await deleteUserStorageData(session.user.id, testNamespace); } // Use Better Auth's built-in deleteUser to handle cascading cleanup diff --git a/src/app/api/admin/tasks/[key]/route.ts b/src/app/api/admin/tasks/[key]/route.ts new file mode 100644 index 0000000..c91bc03 --- /dev/null +++ b/src/app/api/admin/tasks/[key]/route.ts @@ -0,0 +1,62 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdminContext } from '@/lib/server/auth/admin'; +import { updateTask } from '@/lib/server/tasks/engine'; +import { TASK_REGISTRY } from '@/lib/server/tasks/registry'; +import { getTaskSchedulerInfo } from '@/lib/server/tasks/scheduler'; + +export const dynamic = 'force-dynamic'; + +export async function PATCH( + req: NextRequest, + { params }: { params: Promise<{ key: string }> }, +): Promise { + const ctx = await requireAdminContext(req); + if (ctx instanceof Response) return ctx; + + const { key } = await params; + if (!Object.hasOwn(TASK_REGISTRY, key)) { + return NextResponse.json({ error: 'Unknown task' }, { status: 404 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON' }, { status: 400 }); + } + if (!body || typeof body !== 'object') { + return NextResponse.json({ error: 'Expected JSON object' }, { status: 400 }); + } + + const payload = body as { enabled?: unknown; intervalMs?: unknown }; + const patch: { enabled?: boolean; intervalMs?: number } = {}; + if (Object.hasOwn(payload, 'enabled')) { + if (typeof payload.enabled !== 'boolean') { + return NextResponse.json({ error: 'enabled must be a boolean' }, { status: 400 }); + } + patch.enabled = payload.enabled; + } + if (Object.hasOwn(payload, 'intervalMs')) { + if ( + typeof payload.intervalMs !== 'number' + || !Number.isFinite(payload.intervalMs) + || payload.intervalMs <= 0 + ) { + return NextResponse.json({ error: 'intervalMs must be a finite positive number' }, { status: 400 }); + } + patch.intervalMs = Math.max(1, Math.floor(payload.intervalMs)); + const minimumIntervalMs = getTaskSchedulerInfo().minimumIntervalMs; + if (patch.intervalMs < minimumIntervalMs) { + return NextResponse.json( + { error: `intervalMs must be at least ${minimumIntervalMs} for this deployment` }, + { status: 400 }, + ); + } + } + if (Object.keys(patch).length === 0) { + return NextResponse.json({ error: 'Expected enabled or intervalMs' }, { status: 400 }); + } + + await updateTask(key, patch); + return NextResponse.json({ ok: true }); +} diff --git a/src/app/api/admin/tasks/[key]/run/route.ts b/src/app/api/admin/tasks/[key]/run/route.ts new file mode 100644 index 0000000..f09554a --- /dev/null +++ b/src/app/api/admin/tasks/[key]/run/route.ts @@ -0,0 +1,23 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdminContext } from '@/lib/server/auth/admin'; +import { runTaskNow } from '@/lib/server/tasks/engine'; +import { TASK_REGISTRY } from '@/lib/server/tasks/registry'; + +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; + +export async function POST( + req: NextRequest, + { params }: { params: Promise<{ key: string }> }, +): Promise { + const ctx = await requireAdminContext(req); + if (ctx instanceof Response) return ctx; + + const { key } = await params; + if (!Object.hasOwn(TASK_REGISTRY, key)) { + return NextResponse.json({ error: 'Unknown task' }, { status: 404 }); + } + + const ran = await runTaskNow(key); + return NextResponse.json({ ran }); +} diff --git a/src/app/api/admin/tasks/route.ts b/src/app/api/admin/tasks/route.ts new file mode 100644 index 0000000..33dc7c3 --- /dev/null +++ b/src/app/api/admin/tasks/route.ts @@ -0,0 +1,18 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdminContext } from '@/lib/server/auth/admin'; +import { listTasks } from '@/lib/server/tasks/engine'; +import { getTaskSchedulerInfo } from '@/lib/server/tasks/scheduler'; + +export const dynamic = 'force-dynamic'; + +export async function GET(req: NextRequest): Promise { + const ctx = await requireAdminContext(req); + if (ctx instanceof Response) return ctx; + + const scheduler = getTaskSchedulerInfo(); + const tasks = (await listTasks()).map((task) => ({ + ...task, + intervalMs: Math.max(task.intervalMs, scheduler.minimumIntervalMs), + })); + return NextResponse.json({ tasks, scheduler }); +} diff --git a/src/app/api/admin/tasks/tick/route.ts b/src/app/api/admin/tasks/tick/route.ts new file mode 100644 index 0000000..f4d4438 --- /dev/null +++ b/src/app/api/admin/tasks/tick/route.ts @@ -0,0 +1,24 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { runDueTasks } from '@/lib/server/tasks/engine'; + +export const dynamic = 'force-dynamic'; +export const maxDuration = 300; + +/** + * Cron-driven tick for serverless (Vercel) deployments. Vercel automatically + * sends `Authorization: Bearer ${CRON_SECRET}` on scheduled invocations, so we + * require that secret. Self-hosted deployments drive ticks via the in-process + * scheduler and don't need this route. + */ +export async function GET(req: NextRequest): Promise { + const secret = process.env.CRON_SECRET; + if (!secret) { + return NextResponse.json({ error: 'Cron not configured' }, { status: 503 }); + } + if (req.headers.get('authorization') !== `Bearer ${secret}`) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + await runDueTasks(); + return NextResponse.json({ ok: true }); +} diff --git a/src/app/api/documents/blob/get/fallback/route.ts b/src/app/api/documents/blob/get/fallback/route.ts index 1929aeb..0db67e0 100644 --- a/src/app/api/documents/blob/get/fallback/route.ts +++ b/src/app/api/documents/blob/get/fallback/route.ts @@ -9,6 +9,7 @@ import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace' import { isS3Configured } from '@/lib/server/storage/s3'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; +import { deleteOwnedDocument } from '@/lib/server/documents/delete-owned'; export const dynamic = 'force-dynamic'; @@ -80,9 +81,11 @@ export async function GET(req: NextRequest) { }); } catch (error) { if (isMissingBlobError(error)) { - await db - .delete(documents) - .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); + await deleteOwnedDocument({ + userId: doc.userId, + documentId: id, + namespace: testNamespace, + }); return NextResponse.json({ error: 'Not found' }, { status: 404 }); } throw error; diff --git a/src/app/api/documents/blob/preview/fallback/route.ts b/src/app/api/documents/blob/preview/fallback/route.ts index c67da8d..004aa74 100644 --- a/src/app/api/documents/blob/preview/fallback/route.ts +++ b/src/app/api/documents/blob/preview/fallback/route.ts @@ -22,6 +22,7 @@ import { import { extractRawTextSnippet } from '@/lib/server/documents/text-snippets'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; +import { deleteOwnedDocument } from '@/lib/server/documents/delete-owned'; export const dynamic = 'force-dynamic'; @@ -108,9 +109,11 @@ export async function GET(req: NextRequest) { ); } catch (error) { if (isMissingDocumentBlobError(error)) { - await db - .delete(documents) - .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); + await deleteOwnedDocument({ + userId: doc.userId, + documentId: id, + namespace: testNamespace, + }); return NextResponse.json({ error: 'Not found' }, { status: 404 }); } throw error; diff --git a/src/app/api/documents/blob/upload/finalize/route.ts b/src/app/api/documents/blob/upload/finalize/route.ts index b1f5f6e..6944eb6 100644 --- a/src/app/api/documents/blob/upload/finalize/route.ts +++ b/src/app/api/documents/blob/upload/finalize/route.ts @@ -18,6 +18,7 @@ import { } from '@/lib/server/documents/blobstore'; import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert'; import { registerUploadedDocument } from '@/lib/server/documents/register-upload'; +import { withDocumentBlobLease } from '@/lib/server/documents/blob-lease'; import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorResponse } from '@/lib/server/errors/next-response'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -149,47 +150,52 @@ async function finalizeOne(input: { : input.upload.name; const documentId = createHash('sha256').update(finalizedBody).digest('hex'); - try { - await headDocumentBlob(documentId, input.namespace); - } catch (error) { - if (!isMissingBlobError(error)) throw error; - if (!isDocxUpload) { - try { - await copyTempDocumentBlobToDocument( - input.upload.token, - input.userId, - documentId, - input.namespace, - finalizedContentType, - { ifNoneMatch: true }, - ); - } catch (copyError) { - if (!isPreconditionFailed(copyError)) throw copyError; - } - } else { - try { - await putDocumentBlob( - documentId, - finalizedBody, - finalizedContentType, - input.namespace, - { ifNoneMatch: true }, - ); - } catch (putError) { - if (!isPreconditionFailed(putError)) throw putError; + const stored = await withDocumentBlobLease(documentId, async () => { + // Keep the canonical blob and ownership-row write under the same durable + // lease so the orphan reaper cannot delete between its ownership check and + // this registration. + try { + await headDocumentBlob(documentId, input.namespace); + } catch (error) { + if (!isMissingBlobError(error)) throw error; + if (!isDocxUpload) { + try { + await copyTempDocumentBlobToDocument( + input.upload.token, + input.userId, + documentId, + input.namespace, + finalizedContentType, + { ifNoneMatch: true }, + ); + } catch (copyError) { + if (!isPreconditionFailed(copyError)) throw copyError; + } + } else { + try { + await putDocumentBlob( + documentId, + finalizedBody, + finalizedContentType, + input.namespace, + { ifNoneMatch: true }, + ); + } catch (putError) { + if (!isPreconditionFailed(putError)) throw putError; + } } } - } - const canonicalHead = await headDocumentBlob(documentId, input.namespace); - const stored = await registerUploadedDocument({ - documentId, - userId: input.userId, - namespace: input.namespace, - name: finalizedName, - type: finalizedType, - size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength, - lastModified: input.upload.lastModified, + const canonicalHead = await headDocumentBlob(documentId, input.namespace); + return registerUploadedDocument({ + documentId, + userId: input.userId, + namespace: input.namespace, + name: finalizedName, + type: finalizedType, + size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength, + lastModified: input.upload.lastModified, + }); }); await putTempDocumentFinalizeReceipt( diff --git a/src/app/api/documents/blob/upload/presign/route.ts b/src/app/api/documents/blob/upload/presign/route.ts index 183074c..e11cd60 100644 --- a/src/app/api/documents/blob/upload/presign/route.ts +++ b/src/app/api/documents/blob/upload/presign/route.ts @@ -2,8 +2,6 @@ import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { - TEMP_DOCUMENT_UPLOAD_TTL_MS, - deleteExpiredTempDocumentUploads, presignTempPut, } from '@/lib/server/documents/blobstore'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; @@ -72,8 +70,7 @@ export async function POST(req: NextRequest) { } const namespace = getOpenReaderTestNamespace(req.headers); - await deleteExpiredTempDocumentUploads(userId, namespace, Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS) - .catch(() => undefined); + // Expired temp uploads are swept by the cleanup-temp-uploads scheduled task. const signed = await Promise.all( uploads.map(async (upload) => { const token = randomUUID(); diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index fa34474..c7d21d8 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,18 +1,15 @@ import { NextRequest, NextResponse } from 'next/server'; -import { and, count, eq, inArray } from 'drizzle-orm'; +import { and, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; -import { - cleanupDocumentPreviewArtifacts, - deleteDocumentPreviewRows, -} from '@/lib/server/documents/previews'; -import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; +import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; +import { deleteOwnedDocument } from '@/lib/server/documents/delete-owned'; import type { BaseDocument, DocumentType } from '@/types/documents'; export const dynamic = 'force-dynamic'; @@ -114,10 +111,6 @@ export async function DELETE(req: NextRequest) { const targetUserIds = [storageUserId]; - if (targetUserIds.length === 0) { - return NextResponse.json({ success: true, deleted: 0 }); - } - let targetIds: string[] = []; if (idsParam) { targetIds = idsParam @@ -136,52 +129,26 @@ export async function DELETE(req: NextRequest) { return NextResponse.json({ success: true, deleted: 0 }); } - const deletedRows = (await db - .delete(documents) - .where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds))) - .returning({ id: documents.id })) as Array<{ id: string }>; + const ownedRows = (await db + .select({ id: documents.id, userId: documents.userId }) + .from(documents) + .where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))) as Array<{ + id: string; + userId: string; + }>; - const uniqueIds = Array.from(new Set(deletedRows.map((row) => row.id))); - for (const id of uniqueIds) { - const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, id)); - const refCount = Number(ref?.count ?? 0); - if (refCount > 0) continue; - - try { - await deleteDocumentBlob(id, testNamespace); - } catch (error) { - if (!isMissingBlobError(error)) { - serverLogger.warn({ - event: 'documents.delete.blob_cleanup_failed', - degraded: true, - step: 'delete_document_blob', - documentId: id, - error: errorToLog(error), - }, 'Failed to delete document blob during cleanup'); - } + let deleted = 0; + for (const row of ownedRows) { + if (await deleteOwnedDocument({ + userId: row.userId, + documentId: row.id, + namespace: testNamespace, + })) { + deleted += 1; } - - await cleanupDocumentPreviewArtifacts(id, testNamespace).catch((error) => { - serverLogger.warn({ - event: 'documents.delete.preview_artifacts_cleanup_failed', - degraded: true, - step: 'delete_preview_artifacts', - documentId: id, - error: errorToLog(error), - }, 'Failed to cleanup preview artifacts'); - }); - await deleteDocumentPreviewRows(id, testNamespace).catch((error) => { - serverLogger.warn({ - event: 'documents.delete.preview_rows_cleanup_failed', - degraded: true, - step: 'delete_preview_rows', - documentId: id, - error: errorToLog(error), - }, 'Failed to cleanup preview rows'); - }); } - return NextResponse.json({ success: true, deleted: deletedRows.length }); + return NextResponse.json({ success: true, deleted }); } catch (error) { serverLogger.error({ event: 'documents.delete.failed', diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index 4dfb6d9..28d7a0a 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -1,11 +1,12 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { documents, ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; import { isS3Configured, getS3Config } from '@/lib/server/storage/s3'; import { generateTTSBuffer } from '@/lib/server/tts/generate'; import { getTtsSegmentAudioObject, + deleteTtsSegmentAudioObjects, putTtsSegmentAudioObject, } from '@/lib/server/tts/segments-blobstore'; import { @@ -300,6 +301,28 @@ export async function POST(request: NextRequest) { const existingById = new Map(rows.map((row) => [row.segmentId, row])); const manifest: TTSSegmentManifestItem[] = []; + const documentOwnershipExists = async (): Promise => { + const [row] = await db + .select({ id: documents.id }) + .from(documents) + .where(and( + eq(documents.id, parsed.documentId), + eq(documents.userId, scope.storageUserId), + )) + .limit(1); + return Boolean(row); + }; + const assertDocumentOwnership = async (): Promise => { + if (!await documentOwnershipExists()) { + throw new Error('Document ownership was removed during TTS generation'); + } + }; + const deleteAbandonedSegmentMetadata = async (): Promise => { + await db.delete(ttsSegmentEntries).where(and( + eq(ttsSegmentEntries.userId, scope.storageUserId), + eq(ttsSegmentEntries.documentId, parsed.documentId), + )); + }; const upsertSegmentEntry = async (input: { segmentEntryId: string; segmentIndex: number; @@ -308,6 +331,7 @@ export async function POST(request: NextRequest) { textHash: string; textLength: number; }): Promise => { + await assertDocumentOwnership(); await db .insert(ttsSegmentEntries) .values({ @@ -337,6 +361,10 @@ export async function POST(request: NextRequest) { updatedAt: nowMs, }, }); + if (!await documentOwnershipExists()) { + await deleteAbandonedSegmentMetadata(); + throw new Error('Document ownership was removed during TTS metadata persistence'); + } }; for (const segment of normalized) { @@ -657,7 +685,13 @@ export async function POST(request: NextRequest) { failedStage = 's3.put_audio'; const putStartedAt = Date.now(); + await assertDocumentOwnership(); await putTtsSegmentAudioObject(audioKey, ttsBuffer); + if (!await documentOwnershipExists()) { + await deleteTtsSegmentAudioObjects([audioKey]); + await deleteAbandonedSegmentMetadata(); + throw new Error('Document ownership was removed during TTS audio persistence'); + } stageTimings.putAudioMs = Date.now() - putStartedAt; let persistedBuffer = ttsBuffer; diff --git a/src/app/api/user/claim/route.ts b/src/app/api/user/claim/route.ts index 14cd313..5745cd7 100644 --- a/src/app/api/user/claim/route.ts +++ b/src/app/api/user/claim/route.ts @@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { claimAnonymousData } from '@/lib/server/user/claim-data'; import { auth } from '@/lib/server/auth/auth'; import { db } from '@/db'; -import { audiobooks, documents, userDocumentProgress, userPreferences } from '@/db/schema'; +import { audiobooks, documentSettings, documents, userDocumentProgress, userPreferences } from '@/db/schema'; import { count, eq, ne } from 'drizzle-orm'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { errorToLog, serverLogger } from '@/lib/server/logger'; @@ -26,13 +26,14 @@ async function checkClaimMigrationReadiness(): Promise { async function getClaimableCounts( unclaimedUserId: string, -): Promise<{ documents: number; audiobooks: number; preferences: number; progress: number }> { - const [[docCount], [bookCount], [preferencesCount], [progressCount]] = +): Promise<{ documents: number; audiobooks: number; preferences: number; progress: number; documentSettings: number }> { + const [[docCount], [bookCount], [preferencesCount], [progressCount], [settingsCount]] = await Promise.all([ db.select({ count: count() }).from(documents).where(eq(documents.userId, unclaimedUserId)), db.select({ count: count() }).from(audiobooks).where(eq(audiobooks.userId, unclaimedUserId)), db.select({ count: count() }).from(userPreferences).where(eq(userPreferences.userId, unclaimedUserId)), db.select({ count: count() }).from(userDocumentProgress).where(eq(userDocumentProgress.userId, unclaimedUserId)), + db.select({ count: count() }).from(documentSettings).where(eq(documentSettings.userId, unclaimedUserId)), ]); return { @@ -40,6 +41,7 @@ async function getClaimableCounts( audiobooks: Number(bookCount?.count ?? 0), preferences: Number(preferencesCount?.count ?? 0), progress: Number(progressCount?.count ?? 0), + documentSettings: Number(settingsCount?.count ?? 0), }; } diff --git a/src/app/api/user/export/route.ts b/src/app/api/user/export/route.ts index 66920bf..e65b035 100644 --- a/src/app/api/user/export/route.ts +++ b/src/app/api/user/export/route.ts @@ -2,12 +2,26 @@ import { NextRequest, NextResponse } from 'next/server'; import { PassThrough, Readable } from 'stream'; import { auth } from '@/lib/server/auth/auth'; import { db } from '@/db'; -import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress, userTtsChars } from '@/db/schema'; +import { + documents, + audiobooks, + audiobookChapters, + documentSettings, + ttsSegmentEntries, + ttsSegmentVariants, + userDocumentProgress, + userJobEvents, + userPreferences, + userTtsChars, +} from '@/db/schema'; +import * as authSchemaSqlite from '@/db/schema_auth_sqlite'; +import * as authSchemaPostgres from '@/db/schema_auth_postgres'; import { and, desc, eq, inArray } from 'drizzle-orm'; import archiver from 'archiver'; import { appendUserExportArchive } from '@/lib/server/user/data-export'; import { getDocumentBlobStream } from '@/lib/server/documents/blobstore'; import { getAudiobookObjectStream, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore'; +import { getTtsSegmentAudioObjectStream } from '@/lib/server/tts/segments-blobstore'; import { isS3Configured } from '@/lib/server/storage/s3'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { nowTimestampMs } from '@/lib/shared/timestamps'; @@ -17,13 +31,6 @@ import { errorResponse } from '@/lib/server/errors/next-response'; export const dynamic = 'force-dynamic'; export async function GET(req: NextRequest) { - if (!isS3Configured()) { - return NextResponse.json( - { error: 'Export storage is not configured. Set S3_* environment variables.' }, - { status: 503 }, - ); - } - if (!auth) { return errorResponse(new Error('Auth not initialized'), { apiErrorMessage: 'Auth not initialized', @@ -41,11 +48,21 @@ export async function GET(req: NextRequest) { const userId = session.user.id; const testNamespace = getOpenReaderTestNamespace(req.headers); + const storageEnabled = isS3Configured(); + const requireStorage = () => { + if (!storageEnabled) { + throw new Error('Storage is not configured; file content could not be exported'); + } + }; const [ prefs, progress, ttsUsage, + jobEvents, + perDocumentSettings, + segmentEntries, + segmentVariants, userDocs, userAudiobooks, ] = await Promise.all([ @@ -60,6 +77,26 @@ export async function GET(req: NextRequest) { .from(userTtsChars) .where(eq(userTtsChars.userId, userId)) .orderBy(desc(userTtsChars.date)), + db + .select() + .from(userJobEvents) + .where(eq(userJobEvents.userId, userId)) + .orderBy(desc(userJobEvents.createdAt)), + db + .select() + .from(documentSettings) + .where(eq(documentSettings.userId, userId)) + .orderBy(desc(documentSettings.updatedAt)), + db + .select() + .from(ttsSegmentEntries) + .where(eq(ttsSegmentEntries.userId, userId)) + .orderBy(desc(ttsSegmentEntries.updatedAt)), + db + .select() + .from(ttsSegmentVariants) + .where(eq(ttsSegmentVariants.userId, userId)) + .orderBy(desc(ttsSegmentVariants.updatedAt)), db .select() .from(documents) @@ -72,6 +109,36 @@ export async function GET(req: NextRequest) { .orderBy(desc(audiobooks.createdAt)), ]); + const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite; + // Auth exports intentionally select metadata only. Credential and session + // secrets must never be written into the archive. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const database = db as any; + const [authSessions, linkedAccounts] = await Promise.all([ + database + .select({ + id: authSchema.session.id, + expiresAt: authSchema.session.expiresAt, + createdAt: authSchema.session.createdAt, + updatedAt: authSchema.session.updatedAt, + ipAddress: authSchema.session.ipAddress, + userAgent: authSchema.session.userAgent, + }) + .from(authSchema.session) + .where(eq(authSchema.session.userId, userId)), + database + .select({ + id: authSchema.account.id, + accountId: authSchema.account.accountId, + providerId: authSchema.account.providerId, + scope: authSchema.account.scope, + createdAt: authSchema.account.createdAt, + updatedAt: authSchema.account.updatedAt, + }) + .from(authSchema.account) + .where(eq(authSchema.account.userId, userId)), + ]); + const archive = archiver('zip', { zlib: { level: 0 }, forceZip64: true, @@ -114,7 +181,6 @@ export async function GET(req: NextRequest) { const exportedAtMs = nowTimestampMs(); const profileData = { user: session.user, - session: session.session, exportedAtMs, }; @@ -126,13 +192,32 @@ export async function GET(req: NextRequest) { preferences: prefs[0] ?? null, readingHistory: progress, ttsUsage, + jobEvents, + documentSettings: perDocumentSettings, + authSessions, + linkedAccounts, documents: userDocs, audiobooks: userAudiobooks, audiobookChapters: allChapters, - getDocumentBlobStream: async (documentId: string) => getDocumentBlobStream(documentId, testNamespace), - listAudiobookObjects: async (bookId: string, ownerId: string) => listAudiobookObjects(bookId, ownerId, testNamespace), - getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) => - getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace), + ttsSegmentEntries: segmentEntries, + ttsSegmentVariants: segmentVariants, + storageEnabled, + getDocumentBlobStream: async (documentId: string) => { + requireStorage(); + return getDocumentBlobStream(documentId, testNamespace); + }, + listAudiobookObjects: async (bookId: string, ownerId: string) => { + requireStorage(); + return listAudiobookObjects(bookId, ownerId, testNamespace); + }, + getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) => { + requireStorage(); + return getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace); + }, + getTtsSegmentAudioStream: async (audioKey: string) => { + requireStorage(); + return (await getTtsSegmentAudioObjectStream(audioKey)).stream; + }, }); if (!req.signal.aborted) { diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index d730dc8..cdc660f 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -36,6 +36,7 @@ import { import { useRuntimeConfig } from '@/contexts/RuntimeConfigContext'; import { AdminProvidersPanel } from '@/components/admin/AdminProvidersPanel'; import { AdminFeaturesPanel } from '@/components/admin/AdminFeaturesPanel'; +import { AdminTasksPanel } from '@/components/admin/AdminTasksPanel'; import { useSharedProviders } from '@/hooks/useSharedProviders'; import { flushUserPreferencesSync } from '@/lib/client/api/user-state'; import toast from 'react-hot-toast'; @@ -169,7 +170,7 @@ const SIDEBAR_SECTIONS: SidebarSection[] = [ { id: 'admin', label: 'Admin', icon: SettingsIcon, authOnly: true, adminOnly: true }, ]; -type AdminSubTab = 'providers' | 'features'; +type AdminSubTab = 'providers' | 'features' | 'tasks'; export function SettingsTrigger({ className = '', @@ -1054,13 +1055,15 @@ export function SettingsModal({ options={[ { value: 'providers', label: 'Shared providers' }, { value: 'features', label: 'Site features' }, + { value: 'tasks', label: 'Tasks' }, ]} onChange={setAdminSubTab} ariaLabel="Admin tab" - className="grid-cols-2" + className="grid-cols-3" /> {adminSubTab === 'providers' && } {adminSubTab === 'features' && } + {adminSubTab === 'tasks' && } )} diff --git a/src/components/admin/AdminTasksPanel.tsx b/src/components/admin/AdminTasksPanel.tsx new file mode 100644 index 0000000..f386feb --- /dev/null +++ b/src/components/admin/AdminTasksPanel.tsx @@ -0,0 +1,286 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import toast from 'react-hot-toast'; +import { Button, Card, Input, Section, Switch } from '@/components/ui'; +import { ClockIcon, RefreshIcon } from '@/components/icons/Icons'; + +type TaskRunStatus = 'idle' | 'running' | 'ok' | 'error'; + +interface TaskView { + key: string; + name: string; + description?: string; + enabled: boolean; + intervalMs: number; + lastStatus: TaskRunStatus; + lastRunAt: number | null; + lastDurationMs: number | null; + lastError: string | null; + lastResult: string | null; + nextRunAt: number | null; + running: boolean; +} + +interface TaskSchedulerInfo { + mode: 'self-hosted' | 'vercel-cron'; + tickIntervalMs: number; + minimumIntervalMs: number; +} + +const TASKS_QUERY_KEY = ['admin-tasks'] as const; + +async function fetchTasks(): Promise<{ tasks: TaskView[]; scheduler: TaskSchedulerInfo }> { + const res = await fetch('/api/admin/tasks'); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return (await res.json()) as { tasks: TaskView[]; scheduler: TaskSchedulerInfo }; +} + +function RunningDot() { + return ( + + + + + ); +} + +function formatRelative(ms: number | null): string { + if (ms == null) return 'never'; + const diff = ms - Date.now(); + const abs = Math.abs(diff); + const units: Array<[number, string]> = [ + [86_400_000, 'd'], + [3_600_000, 'h'], + [60_000, 'm'], + [1_000, 's'], + ]; + let label = 'now'; + for (const [unitMs, suffix] of units) { + if (abs >= unitMs) { + label = `${Math.round(abs / unitMs)}${suffix}`; + break; + } + } + if (label === 'now') return 'just now'; + return diff < 0 ? `${label} ago` : `in ${label}`; +} + +function formatDuration(ms: number | null): string { + if (ms == null) return '—'; + if (ms < 1000) return `${ms}ms`; + return `${(ms / 1000).toFixed(1)}s`; +} + +export function AdminTasksPanel() { + const queryClient = useQueryClient(); + const { data, error, isPending: isLoading } = useQuery({ + queryKey: TASKS_QUERY_KEY, + queryFn: fetchTasks, + refetchInterval: 5000, + }); + + useEffect(() => { + if (!error) return; + console.error('[AdminTasksPanel] load failed:', error); + toast.error('Failed to load tasks'); + }, [error]); + + const invalidate = () => queryClient.invalidateQueries({ queryKey: TASKS_QUERY_KEY }); + + const patchTask = useMutation({ + mutationFn: async ({ key, patch }: { key: string; patch: { enabled?: boolean; intervalMs?: number } }) => { + const res = await fetch(`/api/admin/tasks/${encodeURIComponent(key)}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(patch), + }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + }, + onSuccess: invalidate, + onError: () => toast.error('Update failed'), + }); + + const runTask = useMutation({ + mutationFn: async (key: string) => { + const res = await fetch(`/api/admin/tasks/${encodeURIComponent(key)}/run`, { method: 'POST' }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return ((await res.json()) as { ran: boolean }).ran; + }, + onSuccess: (ran) => { + toast.success(ran ? 'Task ran' : 'Task already running'); + invalidate(); + }, + onError: () => toast.error('Run failed'), + }); + + return ( +
+ {data?.scheduler.mode === 'vercel-cron' && ( +

+ Vercel Hobby invokes scheduled tasks once daily. Shorter intervals are unavailable on this deployment. +

+ )} + {isLoading ? ( + + ) : ( +
+ {(data?.tasks ?? []).map((task) => ( + patchTask.mutate({ key: task.key, patch: { enabled } })} + onSaveInterval={(intervalMs) => patchTask.mutate({ key: task.key, patch: { intervalMs } })} + onRun={() => runTask.mutate(task.key)} + /> + ))} + {data && data.tasks.length === 0 && ( +

No tasks registered.

+ )} +
+ )} +
+ ); +} + +function TasksSkeleton() { + const rows = Array.from({ length: 2 }); + return ( +
+ {rows.map((_, index) => ( + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + ))} +
+ ); +} + +function TaskRow({ + task, + schedulerMode, + minimumIntervalMs, + busy, + runPending, + onToggle, + onSaveInterval, + onRun, +}: { + task: TaskView; + schedulerMode: TaskSchedulerInfo['mode']; + minimumIntervalMs: number; + busy: boolean; + runPending: boolean; + onToggle: (enabled: boolean) => void; + onSaveInterval: (intervalMs: number) => void; + onRun: () => void; +}) { + const [minutes, setMinutes] = useState(String(task.intervalMs / 60000)); + + useEffect(() => { + setMinutes(String(task.intervalMs / 60000)); + }, [task.intervalMs]); + + const parsedMinutes = parseFloat(minutes); + const newIntervalMs = parsedMinutes * 60000; + const intervalDirty = + Number.isFinite(parsedMinutes) + && newIntervalMs >= minimumIntervalMs + && newIntervalMs !== task.intervalMs; + + const running = task.running || runPending; + + return ( + +
+
+
+
+ {running && } + {task.name} +
+ {task.description &&

{task.description}

} +
+
+ + +
+
+ +
+
+ + + {formatRelative(task.lastRunAt)} + + {task.enabled && schedulerMode === 'vercel-cron' && ( + + + next daily cron + + )} + {task.enabled && schedulerMode !== 'vercel-cron' && task.nextRunAt != null && ( + + + next {formatRelative(task.nextRunAt)} + + )} +
+ +
+ Every + setMinutes(e.target.value)} + aria-label={`${task.name} interval in minutes`} + /> + min + {intervalDirty && ( + + )} +
+
+ + {task.lastStatus === 'error' && task.lastError ? ( +

{task.lastError}

+ ) : ( + task.lastStatus === 'ok' && task.lastResult && ( +

{task.lastResult}

+ ) + )} +
+ + ); +} diff --git a/src/components/auth/ClaimDataModal.tsx b/src/components/auth/ClaimDataModal.tsx index 54f80ec..f1338da 100644 --- a/src/components/auth/ClaimDataModal.tsx +++ b/src/components/auth/ClaimDataModal.tsx @@ -10,6 +10,7 @@ export type ClaimableCounts = { audiobooks: number; preferences: number; progress: number; + documentSettings: number; }; function toClaimableCounts(value: unknown): ClaimableCounts { @@ -19,6 +20,7 @@ function toClaimableCounts(value: unknown): ClaimableCounts { audiobooks: Number(rec.audiobooks ?? 0), preferences: Number(rec.preferences ?? 0), progress: Number(rec.progress ?? 0), + documentSettings: Number(rec.documentSettings ?? 0), }; } @@ -50,8 +52,9 @@ export default function ClaimDataModal({ toast.success( `Successfully claimed ${claimed.documents} documents, ` + `${claimed.audiobooks} audiobooks, ` - + `${claimed.preferences} preference set(s), and ` - + `${claimed.progress} reading progress record(s)!`, + + `${claimed.preferences} preference set(s), ` + + `${claimed.progress} reading progress record(s), and ` + + `${claimed.documentSettings} document setting(s)!`, ); onClaimed(); router.refresh(); @@ -82,6 +85,7 @@ export default function ClaimDataModal({
  • {claimableCounts.audiobooks} audiobook(s)
  • {claimableCounts.preferences} preference set(s)
  • {claimableCounts.progress} reading progress record(s)
    • +
  • {claimableCounts.documentSettings} document setting(s)
    • diff --git a/src/contexts/OnboardingFlowContext.tsx b/src/contexts/OnboardingFlowContext.tsx index e1cadaf..7d1b5de 100644 --- a/src/contexts/OnboardingFlowContext.tsx +++ b/src/contexts/OnboardingFlowContext.tsx @@ -30,6 +30,7 @@ const EMPTY_CLAIM_COUNTS: ClaimableCounts = { audiobooks: 0, preferences: 0, progress: 0, + documentSettings: 0, }; function toClaimableCounts(value: unknown): ClaimableCounts { @@ -39,6 +40,7 @@ function toClaimableCounts(value: unknown): ClaimableCounts { audiobooks: Number(rec.audiobooks ?? 0), preferences: Number(rec.preferences ?? 0), progress: Number(rec.progress ?? 0), + documentSettings: Number(rec.documentSettings ?? 0), }; } @@ -148,7 +150,11 @@ export function OnboardingFlowProvider({ children }: { children: ReactNode }) { if (isClaimEligible) { claimCounts = await fetchClaimableCounts(); - const total = claimCounts.documents + claimCounts.audiobooks + claimCounts.preferences + claimCounts.progress; + const total = claimCounts.documents + + claimCounts.audiobooks + + claimCounts.preferences + + claimCounts.progress + + claimCounts.documentSettings; claimHasData = total > 0; if (!claimHasData && userId) { claimDismissedUsersRef.current.add(userId); diff --git a/src/db/run-in-transaction.ts b/src/db/run-in-transaction.ts new file mode 100644 index 0000000..1ec6ecd --- /dev/null +++ b/src/db/run-in-transaction.ts @@ -0,0 +1,23 @@ +import { db } from '@/db'; + +/** + * Run `fn` with a database connection, wrapped in a transaction on Postgres. + * + * This is the single definition of the SQLite/Postgres bridge, so callers never + * branch on `process.env.POSTGRES_URL` themselves: + * - Postgres: opens a real transaction so a multi-statement read-modify-write + * is atomic, and passes the transaction handle as `conn`. + * - SQLite: better-sqlite3 transactions require synchronous callbacks (they + * cannot be awaited) and the database is a single writer anyway, so `fn` + * runs directly on the shared connection. + */ +export async function runInDbTransaction( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (conn: any) => Promise, +): Promise { + if (process.env.POSTGRES_URL) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (db as any).transaction(async (tx: any) => fn(tx)); + } + return fn(db); +} diff --git a/src/db/schema.ts b/src/db/schema.ts index cefde45..5830ed0 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -19,3 +19,5 @@ export const ttsSegmentEntries = usePostgres ? postgresSchema.ttsSegmentEntries export const ttsSegmentVariants = usePostgres ? postgresSchema.ttsSegmentVariants : sqliteSchema.ttsSegmentVariants; export const adminProviders = usePostgres ? postgresSchema.adminProviders : sqliteSchema.adminProviders; export const adminSettings = usePostgres ? postgresSchema.adminSettings : sqliteSchema.adminSettings; +export const scheduledTasks = usePostgres ? postgresSchema.scheduledTasks : sqliteSchema.scheduledTasks; +export const documentBlobLeases = usePostgres ? postgresSchema.documentBlobLeases : sqliteSchema.documentBlobLeases; diff --git a/src/db/schema_postgres.ts b/src/db/schema_postgres.ts index 63acd4f..b43b97b 100644 --- a/src/db/schema_postgres.ts +++ b/src/db/schema_postgres.ts @@ -1,5 +1,5 @@ import { sql } from 'drizzle-orm'; -import { pgTable, text, integer, real, date, bigint, primaryKey, index, jsonb, foreignKey } from 'drizzle-orm/pg-core'; +import { pgTable, text, integer, real, date, bigint, boolean, primaryKey, index, jsonb, foreignKey, check } from 'drizzle-orm/pg-core'; import { user } from './schema_auth_postgres'; const PG_NOW_MS = sql`(extract(epoch from now()) * 1000)::bigint`; @@ -54,6 +54,7 @@ export const audiobookChapters = pgTable('audiobook_chapters', { // defined here. Only application-specific tables belong in this file. export const userTtsChars = pgTable("user_tts_chars", { + // Also stores device:* and ip:* backstop buckets, so this cannot reference user.id. userId: text('user_id').notNull(), date: date('date').notNull(), charCount: bigint('char_count', { mode: 'number' }).default(0), @@ -71,7 +72,7 @@ export const userTtsChars = pgTable("user_tts_chars", { // worker bounds each op by a hard cap, "ops created in the last hard-cap // window" is an upper bound on in-flight ops. Old rows are pruned opportunistically. export const userJobEvents = pgTable('user_job_events', { - userId: text('user_id').notNull(), + userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), action: text('action').notNull(), opId: text('op_id').notNull(), createdAt: bigint('created_at', { mode: 'number' }).notNull().default(PG_NOW_MS), @@ -209,6 +210,30 @@ export const adminSettings = pgTable('admin_settings', { updatedAt: bigint('updated_at', { mode: 'number' }).notNull().default(PG_NOW_MS), }); +export const scheduledTasks = pgTable('scheduled_tasks', { + key: text('key').primaryKey(), + enabled: boolean('enabled').notNull().default(true), + intervalMs: bigint('interval_ms', { mode: 'number' }).notNull(), + lastStatus: text('last_status').notNull().default('idle'), + leaseOwner: text('lease_owner'), + lastRunAt: bigint('last_run_at', { mode: 'number' }), + lastDurationMs: bigint('last_duration_ms', { mode: 'number' }), + lastError: text('last_error'), + lastResultJson: text('last_result_json'), + nextRunAt: bigint('next_run_at', { mode: 'number' }), + runRequested: boolean('run_requested').notNull().default(false), + runningSince: bigint('running_since', { mode: 'number' }), + updatedAt: bigint('updated_at', { mode: 'number' }).notNull().default(PG_NOW_MS), +}, (table) => [ + check('scheduled_tasks_interval_ms_positive', sql`${table.intervalMs} > 0`), +]); + +export const documentBlobLeases = pgTable('document_blob_leases', { + documentId: text('document_id').primaryKey(), + leaseOwner: text('lease_owner').notNull(), + leaseUntilMs: bigint('lease_until_ms', { mode: 'number' }).notNull(), +}); + export const ttsSegmentVariants = pgTable('tts_segment_variants', { segmentId: text('segment_id').notNull(), userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index 475ebc7..12f7ab7 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer, real, primaryKey, index, foreignKey } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, text, integer, real, primaryKey, index, foreignKey, check } from 'drizzle-orm/sqlite-core'; import { sql } from 'drizzle-orm'; import { user } from './schema_auth_sqlite'; @@ -54,6 +54,7 @@ export const audiobookChapters = sqliteTable('audiobook_chapters', { // defined here. Only application-specific tables belong in this file. export const userTtsChars = sqliteTable("user_tts_chars", { + // Also stores device:* and ip:* backstop buckets, so this cannot reference user.id. userId: text('user_id').notNull(), date: text('date').notNull(), // SQLite doesn't have native DATE type, text YYYY-MM-DD is standard charCount: integer('char_count').default(0), @@ -71,7 +72,7 @@ export const userTtsChars = sqliteTable("user_tts_chars", { // worker bounds each op by a hard cap, "ops created in the last hard-cap // window" is an upper bound on in-flight ops. Old rows are pruned opportunistically. export const userJobEvents = sqliteTable('user_job_events', { - userId: text('user_id').notNull(), + userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), action: text('action').notNull(), opId: text('op_id').notNull(), createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS), @@ -209,6 +210,30 @@ export const adminSettings = sqliteTable('admin_settings', { updatedAt: integer('updated_at').notNull().default(SQLITE_NOW_MS), }); +export const scheduledTasks = sqliteTable('scheduled_tasks', { + key: text('key').primaryKey(), + enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true), + intervalMs: integer('interval_ms').notNull(), + lastStatus: text('last_status').notNull().default('idle'), + leaseOwner: text('lease_owner'), + lastRunAt: integer('last_run_at'), + lastDurationMs: integer('last_duration_ms'), + lastError: text('last_error'), + lastResultJson: text('last_result_json'), + nextRunAt: integer('next_run_at'), + runRequested: integer('run_requested', { mode: 'boolean' }).notNull().default(false), + runningSince: integer('running_since'), + updatedAt: integer('updated_at').notNull().default(SQLITE_NOW_MS), +}, (table) => [ + check('scheduled_tasks_interval_ms_positive', sql`${table.intervalMs} > 0`), +]); + +export const documentBlobLeases = sqliteTable('document_blob_leases', { + documentId: text('document_id').primaryKey(), + leaseOwner: text('lease_owner').notNull(), + leaseUntilMs: integer('lease_until_ms').notNull(), +}); + export const ttsSegmentVariants = sqliteTable('tts_segment_variants', { segmentId: text('segment_id').notNull(), userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), diff --git a/src/instrumentation.node.ts b/src/instrumentation.node.ts new file mode 100644 index 0000000..b4b076c --- /dev/null +++ b/src/instrumentation.node.ts @@ -0,0 +1,5 @@ +import { startTaskScheduler } from '@/lib/server/tasks/scheduler'; + +if (!process.env.VERCEL) { + startTaskScheduler(); +} diff --git a/src/instrumentation.ts b/src/instrumentation.ts new file mode 100644 index 0000000..43739b6 --- /dev/null +++ b/src/instrumentation.ts @@ -0,0 +1,10 @@ +/** + * Next.js startup hook. On a long-lived self-hosted server we run the task + * scheduler in-process. On Vercel (serverless, no persistent process) there is + * nothing to keep a loop alive, so a cron route drives the ticks instead. + */ +export async function register(): Promise { + if (process.env.NEXT_RUNTIME === 'nodejs') { + await import('./instrumentation.node'); + } +} diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index 9f65e4a..99e1b87 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -264,25 +264,13 @@ export async function setRuntimeConfigKey( } const serialized = serializeForStorage(validated); const now = Date.now(); - // Upsert with onConflict. - const isPg = !!process.env.POSTGRES_URL; - if (isPg) { - await db - .insert(adminSettings) - .values({ key, valueJson: serialized as never, source: 'admin', updatedAt: now }) - .onConflictDoUpdate({ - target: adminSettings.key, - set: { valueJson: serialized as never, source: 'admin', updatedAt: now }, - }); - } else { - await db - .insert(adminSettings) - .values({ key, valueJson: serialized as never, source: 'admin', updatedAt: now }) - .onConflictDoUpdate({ - target: adminSettings.key, - set: { valueJson: serialized as never, source: 'admin', updatedAt: now }, - }); - } + await db + .insert(adminSettings) + .values({ key, valueJson: serialized as never, source: 'admin', updatedAt: now }) + .onConflictDoUpdate({ + target: adminSettings.key, + set: { valueJson: serialized as never, source: 'admin', updatedAt: now }, + }); } /** Delete a runtime config row (resets to default/implicit behavior). */ diff --git a/src/lib/server/audiobooks/blobstore.ts b/src/lib/server/audiobooks/blobstore.ts index 4d5549e..a5138d5 100644 --- a/src/lib/server/audiobooks/blobstore.ts +++ b/src/lib/server/audiobooks/blobstore.ts @@ -255,7 +255,11 @@ export async function deleteAudiobookPrefix(prefix: string): Promise { }, }), ); - deleted += deleteRes.Deleted?.length ?? 0; + const errors = deleteRes.Errors ?? []; + if (errors.length > 0) { + throw new Error(`Failed deleting ${errors.length} audiobook storage objects`); + } + deleted += keys.length; } continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; diff --git a/src/lib/server/auth/auth.ts b/src/lib/server/auth/auth.ts index c3dd2cf..d0359d1 100644 --- a/src/lib/server/auth/auth.ts +++ b/src/lib/server/auth/auth.ts @@ -96,8 +96,9 @@ const createAuth = () => betterAuth({ context: { userIdHash: hashForLog(user.id) }, error, }); - // Don't throw – allow the user deletion to proceed even if S3 cleanup fails. - // Orphaned blobs are preferable to a blocked account deletion. + // Without a durable cleanup queue, proceeding would permanently + // orphan user-scoped storage and non-cascading database rows. + throw error; } }, }, @@ -163,134 +164,30 @@ const createAuth = () => betterAuth({ import('@/lib/server/user/claim-data'), ]); - // Transfer rate limiting data (TTS char counts) from anonymous user to authenticated user - try { - await rateLimiter.transferAnonymousUsage(anonymousUser.user.id, newUser.user.id); - serverLogger.info({ - event: 'auth.link_account.transfer.rate_limit.succeeded', - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, 'Transferred rate limit data during account linking'); - } catch (error) { - logDegraded(serverLogger, { - event: 'auth.link_account.transfer.rate_limit.failed', - msg: 'Failed transferring rate limit data during account linking', - step: 'transfer_rate_limit', - context: { - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, - error, - }); - // Don't throw here to prevent blocking the account linking process - } - - // Transfer audiobooks from anonymous user to new authenticated user - try { - const transferred = await claimData.transferUserAudiobooks(anonymousUser.user.id, newUser.user.id); - if (transferred > 0) { - serverLogger.info({ - event: 'auth.link_account.transfer.audiobooks.succeeded', - transferred, - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, 'Transferred audiobooks during account linking'); - } - } catch (error) { - logDegraded(serverLogger, { - event: 'auth.link_account.transfer.audiobooks.failed', - msg: 'Failed transferring audiobooks during account linking', - step: 'transfer_audiobooks', - context: { - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, - error, - }); - // Don't throw here to prevent blocking the account linking process - } - - // Transfer documents from anonymous user to new authenticated user - try { - const transferred = await claimData.transferUserDocuments(anonymousUser.user.id, newUser.user.id); - if (transferred > 0) { - serverLogger.info({ - event: 'auth.link_account.transfer.documents.succeeded', - transferred, - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, 'Transferred documents during account linking'); - } - } catch (error) { - logDegraded(serverLogger, { - event: 'auth.link_account.transfer.documents.failed', - msg: 'Failed transferring documents during account linking', - step: 'transfer_documents', - context: { - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, - error, - }); - // Don't throw here to prevent blocking the account linking process - } - - // Transfer preferences from anonymous user to new authenticated user - try { - const transferred = await claimData.transferUserPreferences(anonymousUser.user.id, newUser.user.id); - if (transferred > 0) { - serverLogger.info({ - event: 'auth.link_account.transfer.preferences.succeeded', - transferred, - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, 'Transferred preferences during account linking'); - } - } catch (error) { - logDegraded(serverLogger, { - event: 'auth.link_account.transfer.preferences.failed', - msg: 'Failed transferring preferences during account linking', - step: 'transfer_preferences', - context: { - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, - error, - }); - // Don't throw here to prevent blocking the account linking process - } - - // Transfer reading progress from anonymous user to new authenticated user - try { - const transferred = await claimData.transferUserProgress(anonymousUser.user.id, newUser.user.id); - if (transferred > 0) { - serverLogger.info({ - event: 'auth.link_account.transfer.progress.succeeded', - transferred, - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, 'Transferred reading progress during account linking'); - } - } catch (error) { - logDegraded(serverLogger, { - event: 'auth.link_account.transfer.progress.failed', - msg: 'Failed transferring reading progress during account linking', - step: 'transfer_progress', - context: { - anonymousUserIdHash: hashForLog(anonymousUser.user.id), - newUserIdHash: hashForLog(newUser.user.id), - }, - error, - }); - // Don't throw here to prevent blocking the account linking process - } + const transferred = await claimData.claimAnonymousData( + newUser.user.id, + anonymousUser.user.id, + null, + { cleanupLegacySources: false }, + ); + await rateLimiter.transferAnonymousUsage(anonymousUser.user.id, newUser.user.id); + const { deleteUserStorageData } = await import('@/lib/server/user/data-cleanup'); + await deleteUserStorageData(anonymousUser.user.id, null); + serverLogger.info({ + event: 'auth.link_account.transfer.succeeded', + transferred, + anonymousUserIdHash: hashForLog(anonymousUser.user.id), + newUserIdHash: hashForLog(newUser.user.id), + }, 'Transferred anonymous user data during account linking'); } catch (error) { logServerError(serverLogger, { event: 'auth.link_account.failed', msg: 'onLinkAccount callback failed', error, }); - // Don't throw here to prevent blocking the account linking process + // Better Auth deletes the anonymous user after this callback. + // Block linking when transfer is incomplete so data remains retryable. + throw error; } // Note: Anonymous user will be automatically deleted after this callback completes }, diff --git a/src/lib/server/documents/blob-lease.ts b/src/lib/server/documents/blob-lease.ts new file mode 100644 index 0000000..c5b82bf --- /dev/null +++ b/src/lib/server/documents/blob-lease.ts @@ -0,0 +1,95 @@ +import { randomUUID } from 'node:crypto'; +import { and, eq, lt } from 'drizzle-orm'; +import { db } from '@/db'; +import { documentBlobLeases } from '@/db/schema'; +import { errorToLog, serverLogger } from '@/lib/server/logger'; + +const DEFAULT_LEASE_MS = 15 * 60 * 1000; +const RETRY_DELAY_MS = 100; +const MAX_RETRY_DELAY_MS = 2_000; + +export type DocumentBlobLease = { + owner: string; + release: () => Promise; +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function tryAcquireDocumentBlobLease( + documentId: string, + options?: { leaseMs?: number }, +): Promise { + const owner = randomUUID(); + const now = Date.now(); + const leaseUntilMs = now + (options?.leaseMs ?? DEFAULT_LEASE_MS); + + const claimed = await db + .insert(documentBlobLeases) + .values({ documentId, leaseOwner: owner, leaseUntilMs }) + .onConflictDoUpdate({ + target: documentBlobLeases.documentId, + set: { leaseOwner: owner, leaseUntilMs }, + where: lt(documentBlobLeases.leaseUntilMs, now), + }) + .returning({ owner: documentBlobLeases.leaseOwner }); + + if (claimed[0]?.owner !== owner) return null; + + return { + owner, + release: async () => { + await db + .delete(documentBlobLeases) + .where(and( + eq(documentBlobLeases.documentId, documentId), + eq(documentBlobLeases.leaseOwner, owner), + )); + }, + }; +} + +export async function withDocumentBlobLease( + documentId: string, + fn: () => Promise, + options?: { waitMs?: number; leaseMs?: number }, +): Promise { + const deadline = Date.now() + (options?.waitMs ?? 30_000); + let lease: DocumentBlobLease | null = null; + let backoffMs = RETRY_DELAY_MS; + + while (!lease && Date.now() < deadline) { + lease = await tryAcquireDocumentBlobLease(documentId, options); + if (lease) break; + + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + + // Exponential backoff with jitter to avoid thundering-herd retries under + // contention, capped and never sleeping past the deadline. + const jitterMs = Math.random() * backoffMs * 0.1; + await sleep(Math.min(backoffMs + jitterMs, remainingMs)); + backoffMs = Math.min(backoffMs * 2, MAX_RETRY_DELAY_MS); + } + if (!lease) { + throw new Error(`Timed out waiting for document blob lease: ${documentId}`); + } + + try { + return await fn(); + } finally { + // Release in its own try/catch so a failing release never masks the + // original error (or result) produced by fn(). + try { + await lease.release(); + } catch (releaseError) { + serverLogger.warn({ + event: 'documents.blobLease.release.failed', + degraded: true, + documentId, + error: errorToLog(releaseError), + }, 'Failed to release document blob lease'); + } + } +} diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index 732ca0b..230bbed 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -10,6 +10,8 @@ import { import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { PDF_PARSER_VERSION, encodeParserVersion } from '@openreader/compute-core/api-contracts'; import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; +import { serverLogger } from '@/lib/server/logger'; +import { logDegraded } from '@/lib/server/errors/logging'; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; @@ -506,11 +508,27 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): const nsSegment = ns ? `ns/${ns}/` : ''; const parsedPrefix = `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/`; + await deleteDocumentPrefix(parsedPrefix); + await deleteDocumentPrefix(`${key}/`); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })); + // Delete the source after the initial derived-artifact cleanup, then sweep + // parsed output once more to catch a worker that finished during deletion. await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key })); - await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })).catch(() => undefined); - await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined); - await deleteDocumentPrefix(parsedPrefix).catch(() => undefined); - await deleteDocumentPrefix(`${key}/`).catch(() => undefined); + // The source blob is already gone at this point. Treat the final sweep as a + // best-effort cleanup: if it throws, rethrowing would make callers roll back + // the document row even though the source is deleted, so log and swallow. + try { + await deleteDocumentPrefix(parsedPrefix); + } catch (error) { + logDegraded(serverLogger, { + event: 'documents.blob_delete.final_parsed_sweep.failed', + msg: 'Failed final parsed-output sweep after document deletion', + step: 'delete_document_parsed_prefix_final', + context: { parsedPrefix }, + error, + }); + } } export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise { @@ -564,7 +582,16 @@ export async function deleteDocumentPrefix(prefix: string): Promise { }, }), ); - deleted += deleteRes.Deleted?.length ?? 0; + const errors = deleteRes.Errors ?? []; + if (errors.length > 0) { + const details = errors + .map((e) => `${e.Key ?? '?'} (${e.Code ?? 'Unknown'}: ${e.Message ?? 'no message'})`) + .join('; '); + throw new Error( + `Failed deleting ${errors.length} document storage object(s) under prefix "${cleanedPrefix}": ${details}`, + ); + } + deleted += keys.length; } continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; @@ -573,52 +600,107 @@ export async function deleteDocumentPrefix(prefix: string): Promise { return deleted; } -export async function deleteExpiredTempDocumentUploads( - userId: string, +/** + * List the source document blobs under a namespace (content-addressed objects + * directly beneath `documents_v1/`, excluding the `parsed_v2/` and `ns/` + * subtrees via the `/` delimiter). Used by the orphaned-blob reaper. + */ +export async function listDocumentSourceBlobs( + namespace: string | null, + options?: { signal?: AbortSignal }, +): Promise> { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + const prefix = `${cfg.prefix}/documents_v1/${nsSegment}`; + const out: Array<{ id: string; lastModifiedMs: number }> = []; + let continuationToken: string | undefined; + + do { + options?.signal?.throwIfAborted(); + const listRes = await client.send( + new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: prefix, + Delimiter: '/', + ContinuationToken: continuationToken, + }), + { abortSignal: options?.signal }, + ); + + for (const item of listRes.Contents ?? []) { + const key = item.Key; + if (!key) continue; + const id = key.slice(prefix.length); + if (!isValidDocumentId(id)) continue; + out.push({ id, lastModifiedMs: item.LastModified?.getTime() ?? 0 }); + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + + return out; +} + +/** + * Delete every temporary upload object (across all users) older than the given + * cutoff. Used by the cleanup-temp-uploads scheduled task. + */ +export async function deleteAllExpiredTempDocumentUploads( namespace: string | null, olderThanMs: number, + options?: { signal?: AbortSignal }, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); - const prefix = tempDocumentUploadPrefix(userId, namespace); + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + const prefix = `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}`; let continuationToken: string | undefined; - const keys: string[] = []; + let deleted = 0; do { + options?.signal?.throwIfAborted(); const listRes = await client.send( new ListObjectsV2Command({ Bucket: cfg.bucket, Prefix: prefix, ContinuationToken: continuationToken, }), + { abortSignal: options?.signal }, ); + const batch: string[] = []; for (const item of listRes.Contents ?? []) { const key = item.Key; const lastModified = item.LastModified?.getTime() ?? 0; if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue; - keys.push(key); + batch.push(key); + } + + if (batch.length > 0) { + const deleteRes = await client.send( + new DeleteObjectsCommand({ + Bucket: cfg.bucket, + Delete: { + Objects: batch.map((Key) => ({ Key })), + Quiet: true, + }, + }), + { abortSignal: options?.signal }, + ); + if (deleteRes.Errors?.length) { + throw new Error( + `Failed to delete temporary uploads from bucket "${cfg.bucket}" ` + + `(keys: ${JSON.stringify(batch)}, errors: ${JSON.stringify(deleteRes.Errors)})`, + ); + } + deleted += batch.length; } continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; } while (continuationToken); - if (keys.length === 0) return 0; - - let deleted = 0; - for (let i = 0; i < keys.length; i += 1000) { - const batch = keys.slice(i, i + 1000); - const deleteRes = await client.send( - new DeleteObjectsCommand({ - Bucket: cfg.bucket, - Delete: { - Objects: batch.map((Key) => ({ Key })), - Quiet: true, - }, - }), - ); - deleted += deleteRes.Deleted?.length ?? 0; - } - return deleted; } diff --git a/src/lib/server/documents/delete-owned.ts b/src/lib/server/documents/delete-owned.ts new file mode 100644 index 0000000..4b2d14e --- /dev/null +++ b/src/lib/server/documents/delete-owned.ts @@ -0,0 +1,41 @@ +import { and, eq } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache'; +import { hashForLog, serverLogger } from '@/lib/server/logger'; +import { logDegraded } from '@/lib/server/errors/logging'; + +/** + * Remove a user's ownership of a document. + * + * Only the per-user TTS segment cache is cleaned inline (it is keyed by userId + * and not reachable afterwards). The shared, content-addressed document blob and + * its previews are reclaimed by the `reap-orphaned-blobs` task once no owner + * remains, so there is no inline blob deletion, last-owner check, or lock. + */ +export async function deleteOwnedDocument(input: { + userId: string; + documentId: string; + namespace: string | null; +}): Promise { + const [removed] = await db + .delete(documents) + .where(and( + eq(documents.id, input.documentId), + eq(documents.userId, input.userId), + )) + .returning(); + if (!removed) return false; + + await deleteDocumentTtsSegmentCache(input).catch((error) => { + logDegraded(serverLogger, { + event: 'documents.delete_owned.tts_cache_cleanup.failed', + msg: 'Failed to clean TTS segment cache after document deletion', + step: 'delete_document_tts_segment_cache', + context: { documentId: input.documentId, userIdHash: hashForLog(input.userId) }, + error, + }); + }); + + return true; +} diff --git a/src/lib/server/documents/previews.ts b/src/lib/server/documents/previews.ts index 44699c6..7c60c98 100644 --- a/src/lib/server/documents/previews.ts +++ b/src/lib/server/documents/previews.ts @@ -8,7 +8,6 @@ import { DOCUMENT_PREVIEW_CONTENT_TYPE, DOCUMENT_PREVIEW_VARIANT, DOCUMENT_PREVIEW_WIDTH, - deleteDocumentPreviewArtifacts, documentPreviewKey, headDocumentPreview, isMissingBlobError, @@ -419,7 +418,3 @@ export async function deleteDocumentPreviewRows(documentId: string, namespace: s .delete(documentPreviews) .where(and(eq(documentPreviews.documentId, documentId), eq(documentPreviews.namespace, namespaceKey))); } - -export async function cleanupDocumentPreviewArtifacts(documentId: string, namespace: string | null): Promise { - await deleteDocumentPreviewArtifacts(documentId, namespace); -} diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts index 9e1d84e..a29feb8 100644 --- a/src/lib/server/documents/register-upload.ts +++ b/src/lib/server/documents/register-upload.ts @@ -1,9 +1,11 @@ import { db } from '@/db'; import { documents } from '@/db/schema'; +import { and, eq } from 'drizzle-orm'; import { enqueueDocumentPreview, } from '@/lib/server/documents/previews'; import { errorToLog, serverLogger } from '@/lib/server/logger'; +import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache'; import type { BaseDocument, DocumentType } from '@/types/documents'; type RegisterUploadedDocumentInput = { @@ -17,6 +19,23 @@ type RegisterUploadedDocumentInput = { }; export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise { + const [existing] = await db + .select({ lastModified: documents.lastModified }) + .from(documents) + .where(and( + eq(documents.id, input.documentId), + eq(documents.userId, input.userId), + )) + .limit(1); + + if (existing && Number(existing.lastModified) !== input.lastModified) { + await deleteDocumentTtsSegmentCache({ + userId: input.userId, + documentId: input.documentId, + namespace: input.namespace, + }); + } + await db .insert(documents) .values({ diff --git a/src/lib/server/rate-limit/job-rate-limiter.ts b/src/lib/server/rate-limit/job-rate-limiter.ts index ef6a45a..9a886ce 100644 --- a/src/lib/server/rate-limit/job-rate-limiter.ts +++ b/src/lib/server/rate-limit/job-rate-limiter.ts @@ -1,4 +1,4 @@ -import { and, eq, gte, lt, sql } from 'drizzle-orm'; +import { and, eq, gte, sql } from 'drizzle-orm'; import { db } from '@/db'; import { userJobEvents } from '@/db/schema'; import { nowTimestampMs } from '@/lib/shared/timestamps'; @@ -143,24 +143,5 @@ export async function recordJobEvent( } catch { // Recording is best-effort; never block op creation on ledger writes. } - - // Opportunistic prune of rows older than the largest configured window so - // the ledger stays small without a separate cron, but never deletes events - // that could still affect an in-window count. - if (Math.random() < 0.05) { - const largestWindowMs = config.windows.reduce( - (max, w) => (Number.isFinite(w.windowMs) && w.windowMs > max ? w.windowMs : max), - 24 * 60 * 60 * 1000, - ); - try { - await safeDb() - .delete(userJobEvents) - .where(and( - eq(userJobEvents.action, action), - lt(userJobEvents.createdAt, now - largestWindowMs), - )); - } catch { - // ignore prune failures - } - } + // Old rows are removed by the prune-job-events scheduled task. } diff --git a/src/lib/server/rate-limit/rate-limiter.ts b/src/lib/server/rate-limit/rate-limiter.ts index b7ce741..a526d0d 100644 --- a/src/lib/server/rate-limit/rate-limiter.ts +++ b/src/lib/server/rate-limit/rate-limiter.ts @@ -1,4 +1,5 @@ import { db } from '@/db'; +import { runInDbTransaction } from '@/db/run-in-transaction'; import { userTtsChars } from '@/db/schema'; import { eq, and, lt, sql } from 'drizzle-orm'; import { nextUtcMidnightTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; @@ -127,25 +128,10 @@ function getRowsAffected(result: unknown): number { export class RateLimiter { constructor() { } - private isPostgres(): boolean { - return Boolean(process.env.POSTGRES_URL); - } - private getUpdatedAtValue(): number { return nowTimestampMs(); } - // Use a transaction only when running with Postgres. - // better-sqlite3 transactions require sync callbacks and cannot be awaited. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private async runMutation(fn: (conn: any) => Promise): Promise { - if (this.isPostgres()) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return safeDb().transaction(async (tx: any) => fn(tx)); - } - return fn(safeDb()); - } - /** * Check if a user can use TTS and increment their char count if allowed */ @@ -189,7 +175,7 @@ export class RateLimiter { try { const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue; - return await this.runMutation(async (conn) => { + return await runInDbTransaction(async (conn) => { // Ensure records exist for each bucket for (const bucket of buckets) { await conn.insert(userTtsChars) @@ -320,36 +306,34 @@ export class RateLimiter { * Transfer char counts when anonymous user creates an account */ async transferAnonymousUsage(anonymousUserId: string, authenticatedUserId: string): Promise { - const today = new Date().toISOString().split('T')[0]; - const dateValue = today as unknown as UserTtsCharsDateValue; const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue; - const anonymousResult = await safeDb().select({ charCount: userTtsChars.charCount }) - .from(userTtsChars) - .where(and(eq(userTtsChars.userId, anonymousUserId), eq(userTtsChars.date, dateValue))); + await runInDbTransaction(async (conn) => { + const anonymousRows = await conn.select({ + date: userTtsChars.date, + charCount: userTtsChars.charCount, + }) + .from(userTtsChars) + .where(eq(userTtsChars.userId, anonymousUserId)); - if (anonymousResult.length === 0) return; + for (const anonymousRow of anonymousRows) { + const dateValue = anonymousRow.date as unknown as UserTtsCharsDateValue; + const anonymousCount = Number(anonymousRow.charCount ?? 0); - const anonymousCount = Number(anonymousResult[0].charCount); - - const existingAuth = await safeDb().select({ charCount: userTtsChars.charCount }) - .from(userTtsChars) - .where(and(eq(userTtsChars.userId, authenticatedUserId), eq(userTtsChars.date, dateValue))); - - if (existingAuth.length === 0) { - await safeDb().insert(userTtsChars) - .values({ userId: authenticatedUserId, date: dateValue, charCount: anonymousCount }); - } else { - const existingCount = Number(existingAuth[0].charCount); - if (anonymousCount > existingCount) { - await safeDb().update(userTtsChars) - .set({ charCount: anonymousCount, updatedAt }) - .where(and(eq(userTtsChars.userId, authenticatedUserId), eq(userTtsChars.date, dateValue))); + await conn.insert(userTtsChars) + .values({ userId: authenticatedUserId, date: dateValue, charCount: anonymousCount }) + .onConflictDoUpdate({ + target: [userTtsChars.userId, userTtsChars.date], + set: { + charCount: sql`${userTtsChars.charCount} + ${anonymousCount}`, + updatedAt, + }, + }); } - } - await safeDb().delete(userTtsChars) - .where(and(eq(userTtsChars.userId, anonymousUserId), eq(userTtsChars.date, dateValue))); + await conn.delete(userTtsChars) + .where(eq(userTtsChars.userId, anonymousUserId)); + }); } /** diff --git a/src/lib/server/tasks/engine.ts b/src/lib/server/tasks/engine.ts new file mode 100644 index 0000000..93822df --- /dev/null +++ b/src/lib/server/tasks/engine.ts @@ -0,0 +1,242 @@ +import { randomUUID } from 'node:crypto'; +import { and, eq, lt, ne, or, sql } from 'drizzle-orm'; +import { db } from '@/db'; +import { scheduledTasks } from '@/db/schema'; +import { serverLogger } from '@/lib/server/logger'; +import { logDegraded } from '@/lib/server/errors/logging'; +import { TASK_REGISTRY } from './registry'; +import type { TaskContext, TaskDef, TaskRegistry, TaskRunStatus } from './types'; + +// A task still marked 'running' after this long is assumed abandoned (process +// crashed mid-run) and may be reclaimed by the next tick. +const STALE_RUNNING_MS = 60 * 60 * 1000; +const DEFAULT_TASK_MAX_RUN_MS = 4 * 60 * 1000; + +/** + * Seed a `scheduled_tasks` row for every registered task. Idempotent: existing + * rows (including user-edited interval/enabled) are left untouched. + */ +async function ensureTaskRows(registry: TaskRegistry = TASK_REGISTRY): Promise { + const now = Date.now(); + for (const [key, def] of Object.entries(registry)) { + await db + .insert(scheduledTasks) + .values({ + key, + enabled: true, + intervalMs: def.defaultIntervalMs, + lastStatus: 'idle', + nextRunAt: now, + }) + .onConflictDoNothing(); + } +} + +/** + * Atomically claim a task for execution. Returns true only if this caller won + * the claim. The single UPDATE flips status to 'running' iff it is not already + * running (or its running marker is stale). A unique owner token fences final + * state updates if a stale runner later completes after being replaced. + */ +async function claimTask(key: string, now: number): Promise { + const leaseOwner = randomUUID(); + const claimed = await db + .update(scheduledTasks) + .set({ + lastStatus: 'running', + leaseOwner, + runningSince: now, + runRequested: false, + updatedAt: now, + }) + .where(and( + eq(scheduledTasks.key, key), + or( + ne(scheduledTasks.lastStatus, 'running'), + lt(scheduledTasks.runningSince, now - STALE_RUNNING_MS), + ), + )) + .returning({ leaseOwner: scheduledTasks.leaseOwner }); + return claimed[0]?.leaseOwner === leaseOwner ? leaseOwner : null; +} + +async function finishTask( + key: string, + leaseOwner: string, + outcome: + | { status: 'ok'; startedAt: number; summary: string | null } + | { status: 'error'; startedAt: number; error: unknown }, +): Promise { + const now = Date.now(); + await db + .update(scheduledTasks) + .set({ + lastStatus: outcome.status, + lastRunAt: now, + lastDurationMs: now - outcome.startedAt, + lastError: outcome.status === 'error' ? String(outcome.error) : null, + lastResultJson: outcome.status === 'ok' && outcome.summary ? outcome.summary : null, + // Schedule the next run off the row's (possibly user-edited) interval. + nextRunAt: sql`${now} + ${scheduledTasks.intervalMs}`, + leaseOwner: null, + runningSince: null, + updatedAt: now, + }) + .where(and( + eq(scheduledTasks.key, key), + eq(scheduledTasks.leaseOwner, leaseOwner), + )); +} + +function taskTimeoutError(key: string, maxRunMs: number): Error { + return new Error(`Scheduled task "${key}" exceeded its ${maxRunMs}ms runtime limit`); +} + +async function executeTask(key: string, def: TaskDef, leaseOwner: string): Promise { + const startedAt = Date.now(); + const maxRunMs = def.maxRunMs ?? DEFAULT_TASK_MAX_RUN_MS; + const controller = new AbortController(); + const context: TaskContext = { + signal: controller.signal, + deadlineAt: startedAt + maxRunMs, + }; + let timeout: ReturnType | undefined; + + try { + const result = await Promise.race([ + def.run(context), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(taskTimeoutError(key, maxRunMs)); + }, maxRunMs); + }), + ]); + const summary = result && typeof result.summary === 'string' ? result.summary : null; + await finishTask(key, leaseOwner, { status: 'ok', startedAt, summary }); + } catch (error) { + logDegraded(serverLogger, { + event: 'tasks.run.failed', + msg: `Scheduled task "${key}" failed`, + step: key, + error, + }); + await finishTask(key, leaseOwner, { status: 'error', startedAt, error }); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +/** + * Run every task that is due (interval elapsed or a manual run was requested), + * claiming each first so concurrent ticks don't double-run. Safe to call from + * any trigger: the self-host interval, a Vercel cron route, or a manual run. + */ +export async function runDueTasks(options?: { registry?: TaskRegistry }): Promise { + const registry = options?.registry ?? TASK_REGISTRY; + await ensureTaskRows(registry); + + const now = Date.now(); + const rows = await db.select().from(scheduledTasks); + + const executions: Promise[] = []; + for (const row of rows) { + const def = registry[row.key]; + if (!def) continue; // orphaned row for a task no longer in the registry + + const due = + row.runRequested || + (row.enabled && row.nextRunAt != null && now >= Number(row.nextRunAt)); + if (!due) continue; + + const leaseOwner = await claimTask(row.key, now); + if (!leaseOwner) continue; + executions.push(executeTask(row.key, def, leaseOwner)); + } + await Promise.all(executions); +} + +/** + * Run a single task immediately, regardless of schedule. Returns false if the + * key is unknown or the task is already running. + */ +export async function runTaskNow(key: string, registry: TaskRegistry = TASK_REGISTRY): Promise { + const def = registry[key]; + if (!def) return false; + await ensureTaskRows(registry); + const leaseOwner = await claimTask(key, Date.now()); + if (!leaseOwner) return false; + await executeTask(key, def, leaseOwner); + return true; +} + +/** Update a task's user-editable fields (enable/disable, run interval). */ +export async function updateTask( + key: string, + patch: { enabled?: boolean; intervalMs?: number }, +): Promise { + const def = Object.hasOwn(TASK_REGISTRY, key) ? TASK_REGISTRY[key] : undefined; + if (!def) return; + + const now = Date.now(); + const set: Record = { updatedAt: now }; + if (typeof patch.enabled === 'boolean') set.enabled = patch.enabled; + if (typeof patch.intervalMs === 'number' && Number.isFinite(patch.intervalMs) && patch.intervalMs > 0) { + set.intervalMs = Math.max(1, Math.floor(patch.intervalMs)); + } + await db + .insert(scheduledTasks) + .values({ + key, + enabled: typeof set.enabled === 'boolean' ? set.enabled : true, + intervalMs: typeof set.intervalMs === 'number' ? set.intervalMs : def.defaultIntervalMs, + lastStatus: 'idle', + nextRunAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: scheduledTasks.key, + set, + }); +} + +export type TaskView = { + key: string; + name: string; + description?: string; + enabled: boolean; + intervalMs: number; + lastStatus: TaskRunStatus; + lastRunAt: number | null; + lastDurationMs: number | null; + lastError: string | null; + lastResult: string | null; + nextRunAt: number | null; + running: boolean; +}; + +/** Combined registry + stored-state view for the admin tasks UI. */ +export async function listTasks(registry: TaskRegistry = TASK_REGISTRY): Promise { + await ensureTaskRows(registry); + const rows = await db.select().from(scheduledTasks); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const byKey = new Map(rows.map((row: any) => [row.key, row])); + + return Object.entries(registry).map(([key, def]) => { + const row = byKey.get(key); + return { + key, + name: def.name, + description: def.description, + enabled: !!row?.enabled, + intervalMs: Number(row?.intervalMs ?? def.defaultIntervalMs), + lastStatus: (row?.lastStatus ?? 'idle') as TaskRunStatus, + lastRunAt: row?.lastRunAt != null ? Number(row.lastRunAt) : null, + lastDurationMs: row?.lastDurationMs != null ? Number(row.lastDurationMs) : null, + lastError: row?.lastError ?? null, + lastResult: row?.lastResultJson ?? null, + nextRunAt: row?.nextRunAt != null ? Number(row.nextRunAt) : null, + running: row?.lastStatus === 'running', + }; + }); +} diff --git a/src/lib/server/tasks/handlers/cleanup-temp-uploads.ts b/src/lib/server/tasks/handlers/cleanup-temp-uploads.ts new file mode 100644 index 0000000..537607d --- /dev/null +++ b/src/lib/server/tasks/handlers/cleanup-temp-uploads.ts @@ -0,0 +1,17 @@ +import { isS3Configured } from '@/lib/server/storage/s3'; +import { + TEMP_DOCUMENT_UPLOAD_TTL_MS, + deleteAllExpiredTempDocumentUploads, +} from '@/lib/server/documents/blobstore'; +import type { TaskContext, TaskResult } from '../types'; + +/** Remove temporary upload objects past their TTL across all users. */ +export async function cleanupTempUploads(context: TaskContext): Promise { + if (!isS3Configured()) { + return { summary: 'Skipped: object storage not configured', deleted: 0 }; + } + + const cutoff = Date.now() - TEMP_DOCUMENT_UPLOAD_TTL_MS; + const deleted = await deleteAllExpiredTempDocumentUploads(null, cutoff, { signal: context.signal }); + return { summary: `Deleted ${deleted} expired upload object(s)`, deleted }; +} diff --git a/src/lib/server/tasks/handlers/prune-job-events.ts b/src/lib/server/tasks/handlers/prune-job-events.ts new file mode 100644 index 0000000..f0ddf18 --- /dev/null +++ b/src/lib/server/tasks/handlers/prune-job-events.ts @@ -0,0 +1,25 @@ +import { lt } from 'drizzle-orm'; +import { db } from '@/db'; +import { userJobEvents } from '@/db/schema'; +import type { TaskResult } from '../types'; + +// Retention far exceeds the largest rate-limit window, so pruning never removes +// an event that could still affect an in-window count. +const RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +function rowsAffected(result: unknown): number { + if (result && typeof result === 'object') { + const rec = result as Record; + if (typeof rec.rowCount === 'number') return rec.rowCount; + if (typeof rec.changes === 'number') return rec.changes; + } + return 0; +} + +/** Delete rate-limit job-event rows older than the retention window. */ +export async function pruneJobEvents(): Promise { + const cutoff = Date.now() - RETENTION_MS; + const result = await db.delete(userJobEvents).where(lt(userJobEvents.createdAt, cutoff)); + const pruned = rowsAffected(result); + return { summary: `Pruned ${pruned} job event(s)`, pruned }; +} diff --git a/src/lib/server/tasks/handlers/prune-tts-usage.ts b/src/lib/server/tasks/handlers/prune-tts-usage.ts new file mode 100644 index 0000000..04f2b91 --- /dev/null +++ b/src/lib/server/tasks/handlers/prune-tts-usage.ts @@ -0,0 +1,10 @@ +import { rateLimiter } from '@/lib/server/rate-limit/rate-limiter'; +import type { TaskResult } from '../types'; + +const RETENTION_DAYS = 30; + +/** Delete TTS usage counter rows (user_tts_chars) older than the retention window. */ +export async function pruneTtsUsage(): Promise { + await rateLimiter.cleanupOldRecords(RETENTION_DAYS); + return { summary: `Pruned TTS usage older than ${RETENTION_DAYS}d` }; +} diff --git a/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts b/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts new file mode 100644 index 0000000..a692370 --- /dev/null +++ b/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts @@ -0,0 +1,88 @@ +import { inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import { deleteDocumentBlob, listDocumentSourceBlobs } from '@/lib/server/documents/blobstore'; +import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; +import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews'; +import { serverLogger } from '@/lib/server/logger'; +import { logDegraded } from '@/lib/server/errors/logging'; +import { tryAcquireDocumentBlobLease } from '@/lib/server/documents/blob-lease'; +import type { TaskContext, TaskResult } from '../types'; + +// Don't reap a blob younger than this — it may belong to an in-flight finalize +// that has written the blob but not yet committed its ownership row. +const GRACE_MS = 60 * 60 * 1000; +const OWNERSHIP_CHECK_BATCH = 200; + +/** + * Delete content-addressed document blobs that no longer have any owner. + * + * Reference count = ownership rows in `documents`. A blob with zero rows and + * age past the grace window is an orphan (e.g. left by a failed inline delete) + * and is safe to remove. Production data is non-namespaced. + */ +export async function reapOrphanedBlobs(context: TaskContext): Promise { + if (!isS3Configured()) { + return { summary: 'Skipped: object storage not configured', reaped: 0 }; + } + + const now = Date.now(); + const blobs = await listDocumentSourceBlobs(null, { signal: context.signal }); + const candidates = blobs.filter((blob) => now - blob.lastModifiedMs > GRACE_MS); + + let reaped = 0; + const failures: unknown[] = []; + for (let i = 0; i < candidates.length; i += OWNERSHIP_CHECK_BATCH) { + context.signal.throwIfAborted(); + const chunk = candidates.slice(i, i + OWNERSHIP_CHECK_BATCH); + const ids = chunk.map((c) => c.id); + const ownedRows = await db + .select({ id: documents.id }) + .from(documents) + .where(inArray(documents.id, ids)); + const owned = new Set(ownedRows.map((row: { id: string }) => row.id)); + + for (const candidate of chunk) { + context.signal.throwIfAborted(); + if (owned.has(candidate.id)) continue; + const lease = await tryAcquireDocumentBlobLease(candidate.id); + if (!lease) continue; + try { + const [owner] = await db + .select({ id: documents.id }) + .from(documents) + .where(inArray(documents.id, [candidate.id])) + .limit(1); + if (owner) continue; + + await deleteDocumentBlob(candidate.id, null); + await deleteDocumentPreviewArtifacts(candidate.id, null); + await deleteDocumentPreviewRows(candidate.id, null); + reaped += 1; + } catch (error) { + failures.push(error); + logDegraded(serverLogger, { + event: 'tasks.reap_orphaned_blobs.delete_failed', + msg: 'Failed to reap orphaned document storage', + step: 'reap_orphaned_blob', + context: { documentId: candidate.id }, + error, + }); + } finally { + await lease.release(); + } + } + } + + if (failures.length > 0) { + throw new AggregateError(failures, `Failed to reap ${failures.length} orphaned blob(s)`); + } + + return { + summary: `Reaped ${reaped} orphaned blob(s)`, + scanned: blobs.length, + candidates: candidates.length, + reaped, + }; +} diff --git a/src/lib/server/tasks/registry.ts b/src/lib/server/tasks/registry.ts new file mode 100644 index 0000000..46c078a --- /dev/null +++ b/src/lib/server/tasks/registry.ts @@ -0,0 +1,37 @@ +import type { TaskRegistry } from './types'; +import { reapOrphanedBlobs } from './handlers/reap-orphaned-blobs'; +import { cleanupTempUploads } from './handlers/cleanup-temp-uploads'; +import { pruneJobEvents } from './handlers/prune-job-events'; +import { pruneTtsUsage } from './handlers/prune-tts-usage'; + +/** + * The catalog of scheduled tasks. Each key is the stable task id stored in the + * `scheduled_tasks` table; renaming a key orphans its row (the engine ignores + * rows with no matching definition). + */ +export const TASK_REGISTRY: TaskRegistry = { + 'reap-orphaned-blobs': { + name: 'Reap orphaned document blobs', + description: 'Delete content-addressed document blobs that no longer have any owner.', + defaultIntervalMs: 6 * 60 * 60 * 1000, + run: reapOrphanedBlobs, + }, + 'cleanup-temp-uploads': { + name: 'Clean up expired uploads', + description: 'Remove temporary upload objects past their TTL.', + defaultIntervalMs: 60 * 60 * 1000, + run: cleanupTempUploads, + }, + 'prune-job-events': { + name: 'Prune job event ledger', + description: 'Delete rate-limit job events older than the retention window.', + defaultIntervalMs: 24 * 60 * 60 * 1000, + run: pruneJobEvents, + }, + 'prune-tts-usage': { + name: 'Prune TTS usage counters', + description: 'Delete TTS usage rows (user_tts_chars) older than 30 days.', + defaultIntervalMs: 24 * 60 * 60 * 1000, + run: pruneTtsUsage, + }, +}; diff --git a/src/lib/server/tasks/scheduler.ts b/src/lib/server/tasks/scheduler.ts new file mode 100644 index 0000000..f60a57a --- /dev/null +++ b/src/lib/server/tasks/scheduler.ts @@ -0,0 +1,55 @@ +import { runDueTasks } from './engine'; +import { serverLogger } from '@/lib/server/logger'; + +const TICK_INTERVAL_MS = 60_000; +const INITIAL_DELAY_MS = 10_000; +export const VERCEL_TICK_INTERVAL_MS = 24 * 60 * 60 * 1000; + +export type TaskSchedulerInfo = { + mode: 'self-hosted' | 'vercel-cron'; + tickIntervalMs: number; + minimumIntervalMs: number; +}; + +export function getTaskSchedulerInfo(): TaskSchedulerInfo { + if (process.env.VERCEL) { + return { + mode: 'vercel-cron', + tickIntervalMs: VERCEL_TICK_INTERVAL_MS, + minimumIntervalMs: VERCEL_TICK_INTERVAL_MS, + }; + } + return { + mode: 'self-hosted', + tickIntervalMs: TICK_INTERVAL_MS, + minimumIntervalMs: TICK_INTERVAL_MS, + }; +} + +let started = false; + +/** + * Start the in-process scheduler loop. Intended for the long-lived self-hosted + * server only — on Vercel a cron route drives the ticks instead. Idempotent: + * repeated calls (e.g. dev HMR) start a single loop. + */ +export function startTaskScheduler(): void { + if (started) return; + started = true; + + const tick = async () => { + try { + await runDueTasks(); + } catch (error) { + serverLogger.warn( + { event: 'tasks.scheduler.tick_failed', error: String(error) }, + 'Task scheduler tick failed', + ); + } + }; + + setTimeout(tick, INITIAL_DELAY_MS); + const handle = setInterval(tick, TICK_INTERVAL_MS); + // Don't keep the process alive solely for the scheduler. + if (typeof handle.unref === 'function') handle.unref(); +} diff --git a/src/lib/server/tasks/types.ts b/src/lib/server/tasks/types.ts new file mode 100644 index 0000000..3c61d7b --- /dev/null +++ b/src/lib/server/tasks/types.ts @@ -0,0 +1,32 @@ +/** Outcome a task handler may return; surfaced in the admin UI as a summary. */ +export type TaskResult = { + /** Short human-readable summary, e.g. "Reaped 3 orphaned blobs". */ + summary?: string; + /** Arbitrary structured detail for debugging. */ + [key: string]: unknown; +}; + +export type TaskContext = { + signal: AbortSignal; + deadlineAt: number; +}; + +export type TaskHandler = (context: TaskContext) => Promise; + +/** Static definition of a task, kept in code (the registry). */ +export type TaskDef = { + /** Display name shown in the admin tasks list. */ + name: string; + /** Optional longer description of what the task does. */ + description?: string; + /** Default run interval in ms; the per-task row may override it. */ + defaultIntervalMs: number; + /** Maximum wall-clock time before the run is marked failed and aborted. */ + maxRunMs?: number; + /** The work to perform. Must be idempotent and safe to re-run. */ + run: TaskHandler; +}; + +export type TaskRegistry = Record; + +export type TaskRunStatus = 'idle' | 'running' | 'ok' | 'error'; diff --git a/src/lib/server/tts/segments-blobstore.ts b/src/lib/server/tts/segments-blobstore.ts index de2d56d..529f69f 100644 --- a/src/lib/server/tts/segments-blobstore.ts +++ b/src/lib/server/tts/segments-blobstore.ts @@ -1,4 +1,5 @@ import { + CopyObjectCommand, DeleteObjectsCommand, GetObjectCommand, ListObjectsV2Command, @@ -244,7 +245,12 @@ export async function deleteTtsSegmentPrefix(prefix: string): Promise { }, }), ); - deleted += deleteRes.Deleted?.length ?? 0; + const errors = deleteRes.Errors ?? []; + if (errors.length > 0) { + throw new Error(`Failed deleting ${errors.length} TTS segment audio objects`); + } + // Quiet=true commonly omits Deleted entries on successful requests. + deleted += keys.length; } continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; @@ -252,3 +258,51 @@ export async function deleteTtsSegmentPrefix(prefix: string): Promise { return deleted; } + +export async function copyTtsSegmentPrefix(sourcePrefix: string, destinationPrefix: string): Promise { + const cfg = getS3Config(); + const client = getS3ProxyClient(); + const source = sourcePrefix.replace(/^\/+/, ''); + const destination = destinationPrefix.replace(/^\/+/, ''); + if (source === destination) return 0; + + let copied = 0; + let continuationToken: string | undefined; + // Track destination keys we have written so a mid-copy failure can be rolled + // back, leaving no orphaned objects behind at the destination prefix. + const copiedKeys: string[] = []; + try { + do { + const listRes = await client.send(new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: source, + ContinuationToken: continuationToken, + })); + const keys = (listRes.Contents ?? []) + .map((item) => item.Key) + .filter((value): value is string => typeof value === 'string' && value.startsWith(source)); + + for (const key of keys) { + const destinationKey = `${destination}${key.slice(source.length)}`; + await client.send(new CopyObjectCommand({ + Bucket: cfg.bucket, + Key: destinationKey, + CopySource: `${cfg.bucket}/${key}`, + ServerSideEncryption: 'AES256', + })); + copiedKeys.push(destinationKey); + copied += 1; + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + } catch (error) { + // Best-effort rollback of the partial copy; surface the original error. + if (copiedKeys.length > 0) { + await deleteTtsSegmentAudioObjects(copiedKeys).catch(() => {}); + } + throw error; + } + + return copied; +} diff --git a/src/lib/server/tts/segments-cache.ts b/src/lib/server/tts/segments-cache.ts index bad4b1f..cc63791 100644 --- a/src/lib/server/tts/segments-cache.ts +++ b/src/lib/server/tts/segments-cache.ts @@ -1,7 +1,12 @@ import { and, eq } from 'drizzle-orm'; import { db } from '@/db'; import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; -import { deleteTtsSegmentAudioObjects } from '@/lib/server/tts/segments-blobstore'; +import { + deleteTtsSegmentAudioObjects, + deleteTtsSegmentPrefix, +} from '@/lib/server/tts/segments-blobstore'; +import { buildTtsSegmentDocumentPrefix } from '@/lib/server/tts/segments'; +import { getS3Config } from '@/lib/server/storage/s3'; import type { ReaderType } from '@/types/user-state'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; @@ -35,6 +40,11 @@ export async function clearTtsSegmentCache( conditions.push(eq(ttsSegmentEntries.readerType, input.readerType)); } + const entryRows = (await db + .select({ segmentEntryId: ttsSegmentEntries.segmentEntryId }) + .from(ttsSegmentEntries) + .where(and(...conditions))) as Array<{ segmentEntryId: string }>; + const rows = (await db .select({ segmentId: ttsSegmentVariants.segmentId, @@ -50,8 +60,6 @@ export async function clearTtsSegmentCache( ) .where(and(...conditions))) as Array<{ segmentId: string; audioKey: string | null }>; - await db.delete(ttsSegmentEntries).where(and(...conditions)); - const audioKeys = rows .map((row) => row.audioKey) .filter((key): key is string => Boolean(key)); @@ -80,10 +88,38 @@ export async function clearTtsSegmentCache( } } + // Keep metadata when storage cleanup is incomplete so a later retry still + // knows which objects must be removed. + if (!warning) { + await db.delete(ttsSegmentEntries).where(and(...conditions)); + } + return { - deletedSegments: rows.length, + deletedSegments: warning ? 0 : new Set(entryRows.map((row) => row.segmentEntryId)).size, requestedAudioObjects: uniqueAudioKeys.length, deletedAudioObjects, ...(warning ? { warning } : {}), }; } + +export async function deleteDocumentTtsSegmentCache(input: { + userId: string; + documentId: string; + namespace: string | null; +}): Promise { + const storagePrefix = getS3Config().prefix; + for (const storageVersion of ['v1', 'v2'] as const) { + await deleteTtsSegmentPrefix(buildTtsSegmentDocumentPrefix({ + storagePrefix, + namespace: input.namespace, + userId: input.userId, + documentId: input.documentId, + storageVersion, + })); + } + + await db.delete(ttsSegmentEntries).where(and( + eq(ttsSegmentEntries.userId, input.userId), + eq(ttsSegmentEntries.documentId, input.documentId), + )); +} diff --git a/src/lib/server/tts/segments.ts b/src/lib/server/tts/segments.ts index e3618ae..08a10a6 100644 --- a/src/lib/server/tts/segments.ts +++ b/src/lib/server/tts/segments.ts @@ -226,6 +226,17 @@ export function buildTtsSegmentAudioKey(input: { return `${input.storagePrefix}/tts_segments_v2/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`; } +export function buildTtsSegmentDocumentPrefix(input: { + storagePrefix: string; + namespace: string | null; + userId: string; + documentId: string; + storageVersion?: 'v1' | 'v2'; +}): string { + const nsSegment = input.namespace ? `ns/${input.namespace}/` : ''; + return `${input.storagePrefix}/tts_segments_${input.storageVersion ?? 'v2'}/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/`; +} + export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise { let workDir: string | null = null; try { diff --git a/src/lib/server/user/claim-data.ts b/src/lib/server/user/claim-data.ts index 24673ec..47a52c9 100644 --- a/src/lib/server/user/claim-data.ts +++ b/src/lib/server/user/claim-data.ts @@ -1,5 +1,14 @@ import { db } from '@/db'; -import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress } from '@/db/schema'; +import { + documents, + audiobooks, + audiobookChapters, + documentSettings, + ttsSegmentEntries, + ttsSegmentVariants, + userPreferences, + userDocumentProgress, +} from '@/db/schema'; import { eq, and, inArray } from 'drizzle-orm'; import { UNCLAIMED_USER_ID } from '../storage/docstore-legacy'; import { cleanupClaimedLegacyFsSources } from './legacy-fs-claim-cleanup'; @@ -12,6 +21,9 @@ import { import { isS3Configured } from '../storage/s3'; import { logDegraded } from '../errors/logging'; import { hashForLog, serverLogger } from '../logger'; +import { getS3Config } from '../storage/s3'; +import { copyTtsSegmentPrefix, deleteTtsSegmentPrefix } from '../tts/segments-blobstore'; +import { buildTtsSegmentDocumentPrefix } from '../tts/segments'; type AudiobookRow = { id: string; @@ -61,7 +73,7 @@ function contentTypeForAudiobookObject(fileName: string): string { return 'application/octet-stream'; } -async function moveAudiobookBlobScope( +async function copyAudiobookBlobScope( bookId: string, fromUserId: string, toUserId: string, @@ -83,15 +95,27 @@ async function moveAudiobookBlobScope( namespace, ); } +} +async function deleteAudiobookBlobScope( + bookId: string, + userId: string, + namespace: string | null, +): Promise { + const objects = await listAudiobookObjects(bookId, userId, namespace); for (const object of objects) { - await deleteAudiobookObject(bookId, fromUserId, object.fileName, namespace).catch(() => {}); + await deleteAudiobookObject(bookId, userId, object.fileName, namespace); } } -export async function claimAnonymousData(userId: string, unclaimedUserId: string = UNCLAIMED_USER_ID, namespace: string | null = null) { +export async function claimAnonymousData( + userId: string, + unclaimedUserId: string = UNCLAIMED_USER_ID, + namespace: string | null = null, + options?: { cleanupLegacySources?: boolean }, +) { if (!userId) { - return { documents: 0, audiobooks: 0, preferences: 0, progress: 0 }; + return { documents: 0, audiobooks: 0, preferences: 0, progress: 0, documentSettings: 0 }; } const [claimableDocumentRows, claimableAudiobookRows] = await Promise.all([ @@ -105,14 +129,18 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string .where(eq(audiobooks.userId, unclaimedUserId)) as Promise>, ]); - const [documentsClaimed, audiobooksClaimed, preferencesClaimed, progressClaimed] = await Promise.all([ - transferUserDocuments(unclaimedUserId, userId), + const [documentsClaimed, audiobooksClaimed, preferencesClaimed, progressClaimed, documentSettingsClaimed] = await Promise.all([ + transferUserDocuments(unclaimedUserId, userId, { namespace, transferTts: true }), transferUserAudiobooks(unclaimedUserId, userId, namespace), transferUserPreferences(unclaimedUserId, userId), transferUserProgress(unclaimedUserId, userId), + transferUserDocumentSettings(unclaimedUserId, userId), ]); - if (claimableDocumentRows.length > 0 || claimableAudiobookRows.length > 0) { + if ( + options?.cleanupLegacySources !== false + && (claimableDocumentRows.length > 0 || claimableAudiobookRows.length > 0) + ) { await cleanupClaimedLegacyFsSources({ documentIds: claimableDocumentRows.map((row) => row.id), audiobookIds: claimableAudiobookRows.map((row) => row.id), @@ -139,6 +167,7 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string audiobooks: audiobooksClaimed, preferences: preferencesClaimed, progress: progressClaimed, + documentSettings: documentSettingsClaimed, }; } @@ -146,7 +175,8 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string * Transfer documents from one userId to another. * * This is used when an anonymous user upgrades to an authenticated account. - * The underlying blob storage is shared (by sha), so this only moves metadata rows. + * The source document blob is shared, while user-scoped TTS metadata and audio + * are copied before the old ownership is removed. * * @returns number of document rows transferred */ @@ -154,26 +184,166 @@ export async function transferUserDocuments( fromUserId: string, toUserId: string, // eslint-disable-next-line @typescript-eslint/no-explicit-any - options?: { db?: any }, + options?: { db?: any; namespace?: string | null; transferTts?: boolean; skipStorage?: boolean }, ): Promise { if (!fromUserId || !toUserId) return 0; if (fromUserId === toUserId) return 0; const database = options?.db ?? db; + // Object storage is always present in a real deployment; `skipStorage` lets + // tests transfer metadata only. The shared, content-addressed document blob is + // never touched here — it stays as long as any owner (the new user) remains. + const copyStorage = !options?.skipStorage && isS3Configured(); const rows = await database.select().from(documents).where(eq(documents.userId, fromUserId)); if (rows.length === 0) return 0; - await database - .insert(documents) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - .values(rows.map((row: any) => ({ ...row, userId: toUserId }))) - .onConflictDoNothing(); - - await database.delete(documents).where(eq(documents.userId, fromUserId)); + for (const row of rows) { + await database + .insert(documents) + .values({ ...row, userId: toUserId }) + .onConflictDoNothing(); + if (options?.transferTts) { + await transferDocumentTtsSegments({ + documentId: row.id, + fromUserId, + toUserId, + namespace: options.namespace ?? null, + database, + copyStorage, + }); + } + await database.delete(documents).where(and( + eq(documents.userId, fromUserId), + eq(documents.id, row.id), + )); + } return rows.length; } +async function transferDocumentTtsSegments(input: { + documentId: string; + fromUserId: string; + toUserId: string; + namespace: string | null; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + database: any; + copyStorage: boolean; +}): Promise { + // Built only when storage is in play — getS3Config() throws if storage is + // unconfigured, which is exactly the metadata-only path tests exercise. + const sourceTtsPrefixes = () => (['v1', 'v2'] as const).map((storageVersion) => ({ + from: buildTtsSegmentDocumentPrefix({ + storagePrefix: getS3Config().prefix, + namespace: input.namespace, + userId: input.fromUserId, + documentId: input.documentId, + storageVersion, + }), + to: buildTtsSegmentDocumentPrefix({ + storagePrefix: getS3Config().prefix, + namespace: input.namespace, + userId: input.toUserId, + documentId: input.documentId, + storageVersion, + }), + })); + + if (input.copyStorage) { + for (const { from, to } of sourceTtsPrefixes()) { + await copyTtsSegmentPrefix(from, to); + } + } + + const entries = await input.database + .select() + .from(ttsSegmentEntries) + .where(and( + eq(ttsSegmentEntries.userId, input.fromUserId), + eq(ttsSegmentEntries.documentId, input.documentId), + )); + const variants = entries.length > 0 + ? await input.database + .select() + .from(ttsSegmentVariants) + .where(and( + eq(ttsSegmentVariants.userId, input.fromUserId), + inArray(ttsSegmentVariants.segmentEntryId, entries.map( + (entry: typeof ttsSegmentEntries.$inferSelect) => entry.segmentEntryId, + )), + )) + : []; + + if (entries.length > 0) { + await input.database.insert(ttsSegmentEntries) + .values(entries.map((entry: typeof ttsSegmentEntries.$inferSelect) => ({ + ...entry, + userId: input.toUserId, + }))) + .onConflictDoNothing(); + } + + const encodedFrom = encodeURIComponent(input.fromUserId); + const encodedTo = encodeURIComponent(input.toUserId); + const sourceAudioKeyPrefix = `/users/${encodedFrom}/docs/${input.documentId}/`; + const destAudioKeyPrefix = `/users/${encodedTo}/docs/${input.documentId}/`; + if (variants.length > 0) { + await input.database.insert(ttsSegmentVariants) + .values(variants.map((variant: typeof ttsSegmentVariants.$inferSelect) => { + const audioKey = variant.audioKey ?? null; + if (!audioKey || audioKey.includes(sourceAudioKeyPrefix)) { + return { + ...variant, + userId: input.toUserId, + audioKey: audioKey?.replace(sourceAudioKeyPrefix, destAudioKeyPrefix) ?? null, + }; + } + // The key did not contain the expected source path, so it cannot be + // safely remapped. Leaving it would point the new owner at the source + // user's (soon-deleted) audio, so null it out and log for investigation. + logDegraded(serverLogger, { + event: 'user.claim.tts_variant_audio_key.unmapped', + msg: 'TTS segment variant audioKey did not match expected source path during claim', + step: 'remap_tts_variant_audio_key', + context: { + originalAudioKey: audioKey, + fromUserIdHash: hashForLog(input.fromUserId), + toUserIdHash: hashForLog(input.toUserId), + documentId: input.documentId, + }, + }); + return { + ...variant, + userId: input.toUserId, + audioKey: null, + }; + })) + .onConflictDoNothing(); + } + + // Always remove the source metadata: the source account (e.g. the persistent + // 'unclaimed' user) is not deleted, so nothing would cascade it away. + if (variants.length > 0) { + await input.database.delete(ttsSegmentVariants).where(and( + eq(ttsSegmentVariants.userId, input.fromUserId), + inArray(ttsSegmentVariants.segmentId, variants.map( + (variant: typeof ttsSegmentVariants.$inferSelect) => variant.segmentId, + )), + )); + } + await input.database.delete(ttsSegmentEntries).where(and( + eq(ttsSegmentEntries.userId, input.fromUserId), + eq(ttsSegmentEntries.documentId, input.documentId), + )); + + // Remove the now-copied source audio objects too (only when storage is in play). + if (input.copyStorage) { + for (const { from } of sourceTtsPrefixes()) { + await deleteTtsSegmentPrefix(from); + } + } +} + /** * Transfer audiobooks from one user to another. * Used when an anonymous user creates a real account. @@ -195,7 +365,7 @@ export async function transferUserAudiobooks( if (isS3Configured()) { for (const book of books) { - await moveAudiobookBlobScope(book.id, fromUserId, toUserId, namespace); + await copyAudiobookBlobScope(book.id, fromUserId, toUserId, namespace); } } @@ -215,6 +385,12 @@ export async function transferUserAudiobooks( .onConflictDoNothing(); } + if (isS3Configured()) { + for (const book of books) { + await deleteAudiobookBlobScope(book.id, fromUserId, namespace); + } + } + await db.delete(audiobookChapters).where(eq(audiobookChapters.userId, fromUserId)); await db.delete(audiobooks).where(eq(audiobooks.userId, fromUserId)); @@ -309,3 +485,57 @@ export async function transferUserProgress(fromUserId: string, toUserId: string) await db.delete(userDocumentProgress).where(eq(userDocumentProgress.userId, fromUserId)); return fromRows.length; } + +export async function transferUserDocumentSettings( + fromUserId: string, + toUserId: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options?: { db?: any }, +): Promise { + if (!fromUserId || !toUserId || fromUserId === toUserId) return 0; + + const database = options?.db ?? db; + const rows = await database + .select() + .from(documentSettings) + .where(eq(documentSettings.userId, fromUserId)); + const documentIds = rows.map((row: { documentId: string }) => row.documentId); + const existingRows = documentIds.length > 0 + ? await database + .select({ + documentId: documentSettings.documentId, + clientUpdatedAtMs: documentSettings.clientUpdatedAtMs, + }) + .from(documentSettings) + .where(and( + eq(documentSettings.userId, toUserId), + inArray(documentSettings.documentId, documentIds), + )) + : []; + const existingByDocumentId = new Map( + existingRows.map((row: { documentId: string; clientUpdatedAtMs: number | null }) => [ + row.documentId, + row, + ] as const), + ); + + for (const row of rows) { + const existing = existingByDocumentId.get(row.documentId); + if (existing && Number(existing.clientUpdatedAtMs ?? 0) >= Number(row.clientUpdatedAtMs ?? 0)) { + continue; + } + await database + .insert(documentSettings) + .values({ ...row, userId: toUserId }) + .onConflictDoUpdate({ + target: [documentSettings.documentId, documentSettings.userId], + set: { + dataJson: row.dataJson, + clientUpdatedAtMs: row.clientUpdatedAtMs, + updatedAt: row.updatedAt, + }, + }); + } + await database.delete(documentSettings).where(eq(documentSettings.userId, fromUserId)); + return rows.length; +} diff --git a/src/lib/server/user/data-cleanup.ts b/src/lib/server/user/data-cleanup.ts index 43f18b2..fd3845f 100644 --- a/src/lib/server/user/data-cleanup.ts +++ b/src/lib/server/user/data-cleanup.ts @@ -1,14 +1,26 @@ /** - * Cleans up all S3 storage artifacts belonging to a user. - * Called from Better Auth's `beforeDelete` hook so that blobs are removed - * before the DB cascade wipes the metadata rows we query against. + * Cleans up user-scoped storage that the orphaned-blob reaper cannot reach. + * + * Called from Better Auth's `beforeDelete` hook (canonical pass) and the + * account-delete route (test-namespaced pass). Shared, content-addressed + * document blobs + previews are NOT deleted here on the canonical pass — they + * are reclaimed by the `reap-orphaned-blobs` task once their ownership rows are + * gone. Per-user storage (audiobooks, TTS segments, temp uploads) is keyed by + * userId and would be unreachable after the cascade, so it is deleted inline + * and failures block the deletion. */ import { db } from '@/db'; -import { documents, audiobooks } from '@/db/schema'; +import { documents, audiobooks, userJobEvents, userTtsChars } from '@/db/schema'; +import * as authSchemaSqlite from '@/db/schema_auth_sqlite'; +import * as authSchemaPostgres from '@/db/schema_auth_postgres'; import { eq } from 'drizzle-orm'; import { getS3Config, isS3Configured } from '@/lib/server/storage/s3'; -import { deleteDocumentBlob } from '@/lib/server/documents/blobstore'; +import { + deleteDocumentBlob, + deleteDocumentPrefix, + tempDocumentUploadPrefix, +} from '@/lib/server/documents/blobstore'; import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews'; import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore'; @@ -16,87 +28,50 @@ import { deleteTtsSegmentPrefix } from '@/lib/server/tts/segments-blobstore'; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; -type DocumentRow = { id: string }; type AudiobookRow = { id: string }; -/** - * Delete all S3 blobs owned by `userId`. - * - * This covers: - * - Document file blobs - * - Document preview images - * - Audiobook audio files (chapter mp3s, metadata json, etc.) - * - * Each item is cleaned up independently; a failure on one does not block the rest. - */ export async function deleteUserStorageData( userId: string, namespace: string | null, ): Promise { const s3Enabled = isS3Configured(); + const failures: unknown[] = []; // eslint-disable-next-line @typescript-eslint/no-explicit-any const database = db as any; + const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite; - // --- Documents & previews --- - const userDocs: DocumentRow[] = await database - .select({ id: documents.id }) - .from(documents) - .where(eq(documents.userId, userId)); - - let docsDeleted = 0; - for (const doc of userDocs) { - if (s3Enabled) { + // --- Document blobs & previews --- + // Canonical pass: deferred to the reap-orphaned-blobs task (the rows cascade + // away with the user, then the reaper reclaims the now-orphaned blobs). The + // reaper only runs for the canonical namespace, so test-namespaced storage is + // deleted inline here. + let docBlobsDeleted = 0; + if (s3Enabled && namespace !== null) { + const userDocs = (await database + .select({ id: documents.id }) + .from(documents) + .where(eq(documents.userId, userId))) as Array<{ id: string }>; + for (const doc of userDocs) { try { await deleteDocumentBlob(doc.id, namespace); - docsDeleted++; - } catch (error) { - logDegraded(serverLogger, { - event: 'user.data_cleanup.document_blob_delete.failed', - msg: 'Failed to delete document blob', - step: 'delete_document_blob', - context: { - documentId: doc.id, - userIdHash: hashForLog(userId), - }, - error, - }); - } - - try { await deleteDocumentPreviewArtifacts(doc.id, namespace); + await deleteDocumentPreviewRows(doc.id, namespace); + docBlobsDeleted++; } catch (error) { + failures.push(error); logDegraded(serverLogger, { - event: 'user.data_cleanup.document_preview_delete.failed', - msg: 'Failed to delete preview artifacts', - step: 'delete_document_preview_artifacts', - context: { - documentId: doc.id, - userIdHash: hashForLog(userId), - }, + event: 'user.data_cleanup.document_storage_delete.failed', + msg: 'Failed to delete namespaced document storage', + step: 'delete_namespaced_document_storage', + context: { documentId: doc.id, userIdHash: hashForLog(userId) }, error, }); } } - - // Always clean up DB rows — documentPreviews has no FK cascade on user. - try { - await deleteDocumentPreviewRows(doc.id, namespace); - } catch (error) { - logDegraded(serverLogger, { - event: 'user.data_cleanup.document_preview_rows_delete.failed', - msg: 'Failed to delete preview rows', - step: 'delete_document_preview_rows', - context: { - documentId: doc.id, - userIdHash: hashForLog(userId), - }, - error, - }); - } } - // --- Audiobooks --- + // --- Audiobooks (per-user object storage; not reaped) --- const userBooks: AudiobookRow[] = s3Enabled ? await database .select({ id: audiobooks.id }) @@ -107,26 +82,36 @@ export async function deleteUserStorageData( let booksDeleted = 0; for (const book of userBooks) { try { - const prefix = audiobookPrefix(book.id, userId, namespace); - await deleteAudiobookPrefix(prefix); + await deleteAudiobookPrefix(audiobookPrefix(book.id, userId, namespace)); booksDeleted++; } catch (error) { + failures.push(error); logDegraded(serverLogger, { event: 'user.data_cleanup.audiobook_blobs_delete.failed', msg: 'Failed to delete audiobook blobs', step: 'delete_audiobook_prefix', - context: { - bookId: book.id, - userIdHash: hashForLog(userId), - }, + context: { bookId: book.id, userIdHash: hashForLog(userId) }, error, }); } } - // --- TTS segments --- + // --- Temp uploads + TTS segments (per-user object storage; not reaped) --- let segmentsDeleted = 0; if (s3Enabled) { + try { + await deleteDocumentPrefix(tempDocumentUploadPrefix(userId, namespace)); + } catch (error) { + failures.push(error); + logDegraded(serverLogger, { + event: 'user.data_cleanup.temp_document_uploads_delete.failed', + msg: 'Failed to delete temporary document uploads', + step: 'delete_temp_document_upload_prefix', + context: { userIdHash: hashForLog(userId) }, + error, + }); + } + try { const cfg = getS3Config(); const nsSegment = namespace ? `ns/${namespace}/` : ''; @@ -135,6 +120,7 @@ export async function deleteUserStorageData( segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV1); segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV2); } catch (error) { + failures.push(error); logDegraded(serverLogger, { event: 'user.data_cleanup.tts_segments_delete.failed', msg: 'Failed to delete TTS segment blobs', @@ -145,15 +131,48 @@ export async function deleteUserStorageData( } } - if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) { + // Block deletion if any non-reapable storage cleanup failed — proceeding + // would permanently orphan it. Nothing was removed from the database yet, so + // there is nothing to roll back. + if (failures.length > 0) { + throw new AggregateError(failures, `User storage cleanup failed in ${failures.length} operation(s)`); + } + + // Namespaced cleanup is a storage-only pass; database rows are global and are + // only removed on the canonical (non-namespaced) pass. + if (namespace === null) { + // Explicit for compatibility with pre-cascade installations and to remove + // auth verification tokens, which cannot carry a user FK. + for (const { table, userColumn, step } of [ + { table: userTtsChars, userColumn: userTtsChars.userId, step: 'delete_user_tts_usage_rows' }, + { table: userJobEvents, userColumn: userJobEvents.userId, step: 'delete_user_job_event_rows' }, + { table: authSchema.verification, userColumn: authSchema.verification.value, step: 'delete_user_verification_rows' }, + ]) { + await database.delete(table).where(eq(userColumn, userId)).catch((error: unknown) => { + failures.push(error); + logDegraded(serverLogger, { + event: 'user.data_cleanup.db_rows_delete.failed', + msg: 'Failed to delete non-cascading user database rows', + step, + context: { userIdHash: hashForLog(userId) }, + error, + }); + }); + } + } + + if (docBlobsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) { serverLogger.info({ event: 'user.data_cleanup.completed', userIdHash: hashForLog(userId), - docsDeleted, - totalDocs: userDocs.length, + docBlobsDeleted, booksDeleted, totalBooks: userBooks.length, segmentsDeleted, }, 'Completed user storage cleanup'); } + + if (failures.length > 0) { + throw new AggregateError(failures, `User database cleanup failed in ${failures.length} operation(s)`); + } } diff --git a/src/lib/server/user/data-export.ts b/src/lib/server/user/data-export.ts index f81de15..8e173f3 100644 --- a/src/lib/server/user/data-export.ts +++ b/src/lib/server/user/data-export.ts @@ -11,7 +11,7 @@ export type ExportBlobBody = | ArrayBufferView | { transformToByteArray: () => Promise }; -type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list'; +type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list' | 'tts_segment'; type ExportIssue = { scope: ExportIssueScope; @@ -41,6 +41,20 @@ type ExportAudiobookObject = { [key: string]: unknown; }; +type ExportTtsSegmentEntry = { + segmentEntryId: string; + documentId: string; + [key: string]: unknown; +}; + +type ExportTtsSegmentVariant = { + segmentId: string; + segmentEntryId: string; + audioKey?: string | null; + audioFormat?: string | null; + [key: string]: unknown; +}; + export type AppendUserExportArchiveInput = { archive: Archiver; userId: string; @@ -49,12 +63,20 @@ export type AppendUserExportArchiveInput = { preferences: unknown | null; readingHistory: unknown[]; ttsUsage: unknown[]; + jobEvents: unknown[]; + documentSettings: unknown[]; + authSessions: unknown[]; + linkedAccounts: unknown[]; documents: ExportDocument[]; audiobooks: ExportAudiobook[]; audiobookChapters: ExportAudiobookChapter[]; + ttsSegmentEntries: ExportTtsSegmentEntry[]; + ttsSegmentVariants: ExportTtsSegmentVariant[]; + storageEnabled: boolean; getDocumentBlobStream: (documentId: string) => Promise; listAudiobookObjects: (bookId: string, userId: string) => Promise; getAudiobookObjectStream: (bookId: string, userId: string, fileName: string) => Promise; + getTtsSegmentAudioStream: (audioKey: string) => Promise; }; function isNodeReadableStream(value: unknown): value is Readable { @@ -124,17 +146,26 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu preferences, readingHistory, ttsUsage, + jobEvents, + documentSettings, + authSessions, + linkedAccounts, documents, audiobooks, audiobookChapters, + ttsSegmentEntries, + ttsSegmentVariants, + storageEnabled, getDocumentBlobStream, listAudiobookObjects, getAudiobookObjectStream, + getTtsSegmentAudioStream, } = input; const issues: ExportIssue[] = []; let documentFilesExported = 0; let audiobookFilesExported = 0; + let ttsSegmentFilesExported = 0; appendJson(archive, 'profile.json', profileData); if (preferences) { @@ -142,7 +173,13 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu } appendJson(archive, 'reading_history.json', readingHistory); appendJson(archive, 'tts_usage.json', ttsUsage); + appendJson(archive, 'job_events.json', jobEvents); + appendJson(archive, 'document_settings.json', documentSettings); + appendJson(archive, 'auth_sessions.json', authSessions); + appendJson(archive, 'linked_accounts.json', linkedAccounts); appendJson(archive, 'library_documents.json', documents); + appendJson(archive, 'tts_segment_entries.json', ttsSegmentEntries); + appendJson(archive, 'tts_segment_variants.json', ttsSegmentVariants); const chaptersByBookId = new Map(); for (const chapter of audiobookChapters) { @@ -157,7 +194,7 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu })); appendJson(archive, 'library_audiobooks.json', audiobooksWithChapters); - for (const doc of documents) { + for (const doc of storageEnabled ? documents : []) { const documentId = toSafePathSegment(doc.id, 'document'); const fileName = toSafePathSegment(doc.name || `${doc.id}.bin`, `${documentId}.bin`); const entryName = `files/documents/${documentId}/${fileName}`; @@ -177,7 +214,7 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu } } - for (const book of audiobooks) { + for (const book of storageEnabled ? audiobooks : []) { let objects: ExportAudiobookObject[] = []; try { objects = await listAudiobookObjects(book.id, userId); @@ -215,8 +252,38 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu } } + const documentIdByEntryId = new Map( + ttsSegmentEntries.map((entry) => [entry.segmentEntryId, entry.documentId]), + ); + for (const variant of storageEnabled ? ttsSegmentVariants : []) { + const audioKey = typeof variant.audioKey === 'string' ? variant.audioKey : ''; + if (!audioKey) continue; + + const documentId = toSafePathSegment( + documentIdByEntryId.get(variant.segmentEntryId) ?? 'unknown-document', + 'unknown-document', + ); + const segmentId = toSafePathSegment(variant.segmentId, 'segment'); + const format = toSafePathSegment(variant.audioFormat || 'mp3', 'mp3'); + const entryName = `files/tts_segments/${documentId}/${segmentId}.${format}`; + + try { + const body = await getTtsSegmentAudioStream(audioKey); + const stream = await bodyToNodeReadable(body); + archive.append(stream, { name: entryName }); + ttsSegmentFilesExported += 1; + } catch (error) { + issues.push({ + scope: 'tts_segment', + id: variant.segmentId, + fileName: entryName, + message: normalizeErrorMessage(error), + }); + } + } + const manifest = { - formatVersion: 2, + formatVersion: 3, exportedAtMs, userId, scope: 'owned', @@ -224,14 +291,26 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu documentsMetadata: documents.length, audiobooksMetadata: audiobooks.length, audiobookChaptersMetadata: audiobookChapters.length, + documentSettingsMetadata: documentSettings.length, + ttsSegmentEntriesMetadata: ttsSegmentEntries.length, + ttsSegmentVariantsMetadata: ttsSegmentVariants.length, + authSessionsMetadata: authSessions.length, + linkedAccountsMetadata: linkedAccounts.length, + jobEventsMetadata: jobEvents.length, documentFiles: documentFilesExported, audiobookFiles: audiobookFilesExported, + ttsSegmentFiles: ttsSegmentFilesExported, issues: issues.length, }, includes: { metadata: true, - documentFiles: true, - audiobookFiles: true, + documentFiles: storageEnabled, + audiobookFiles: storageEnabled, + ttsSegmentFiles: storageEnabled, + credentialSecrets: false, + temporaryUploads: false, + derivedDocumentPreviews: false, + derivedParsedDocuments: false, filesystemSources: false, }, }; diff --git a/tests/global-teardown.ts b/tests/global-teardown.ts index fa6356f..826a85e 100644 --- a/tests/global-teardown.ts +++ b/tests/global-teardown.ts @@ -1,11 +1,13 @@ import { ListObjectsV2Command } from '@aws-sdk/client-s3'; -import { and, eq, inArray, like } from 'drizzle-orm'; +import { and, eq, inArray, like, ne } from 'drizzle-orm'; import { db } from '../src/db'; -import { audiobooks, audiobookChapters, documents } from '../src/db/schema'; +import { audiobooks, audiobookChapters, documentPreviews, documents } from '../src/db/schema'; import { deleteDocumentPrefix } from '../src/lib/server/documents/blobstore'; import { deleteAudiobookPrefix } from '../src/lib/server/audiobooks/blobstore'; import { deleteTtsSegmentPrefix } from '../src/lib/server/tts/segments-blobstore'; import { getS3Client, getS3Config, isS3Configured } from '../src/lib/server/storage/s3'; +import * as authSchemaSqlite from '../src/db/schema_auth_sqlite'; +import * as authSchemaPostgres from '../src/db/schema_auth_postgres'; function chunk(items: T[], size: number): T[][] { if (items.length === 0) return []; @@ -66,46 +68,65 @@ function parseAudiobookScopeFromKey( } export default async function globalTeardown(): Promise { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const authSchema: any = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite; + const testSessions = await db + .select({ userId: authSchema.session.userId }) + .from(authSchema.session) + .where(like(authSchema.session.userAgent, '%OpenReader-Playwright/%')) as Array<{ userId: string }>; + const testUserIds = Array.from(new Set(testSessions.map((row) => row.userId))); + // Always clear namespaced unclaimed SQL rows from prior runs. await db.delete(audiobookChapters).where(like(audiobookChapters.userId, 'unclaimed::%')); await db.delete(audiobooks).where(like(audiobooks.userId, 'unclaimed::%')); await db.delete(documents).where(like(documents.userId, 'unclaimed::%')); + await db.delete(documentPreviews).where(ne(documentPreviews.namespace, '')); - if (!isS3Configured()) return; + if (isS3Configured()) { + const config = getS3Config(); + const docsNsRootPrefix = `${config.prefix}/documents_v1/ns/`; + const documentPreviewsNsRootPrefix = `${config.prefix}/document_previews_v1/ns/`; + const tempUploadsNsRootPrefix = `${config.prefix}/document_uploads_temp_v1/ns/`; + const audiobooksNsRootPrefix = `${config.prefix}/audiobooks_v1/ns/`; + const ttsSegmentsNsRootPrefixV1 = `${config.prefix}/tts_segments_v1/ns/`; + const ttsSegmentsNsRootPrefixV2 = `${config.prefix}/tts_segments_v2/ns/`; - const config = getS3Config(); - const docsNsRootPrefix = `${config.prefix}/documents_v1/ns/`; - const audiobooksNsRootPrefix = `${config.prefix}/audiobooks_v1/ns/`; - const ttsSegmentsNsRootPrefixV1 = `${config.prefix}/tts_segments_v1/ns/`; - const ttsSegmentsNsRootPrefixV2 = `${config.prefix}/tts_segments_v2/ns/`; - - // Remove SQL audiobook rows for namespaced objects (covers auth claim flows too). - const audiobookKeys = await listKeysByPrefix(audiobooksNsRootPrefix); - const byUser = new Map>(); - for (const key of audiobookKeys) { - const scope = parseAudiobookScopeFromKey(key, audiobooksNsRootPrefix); - if (!scope) continue; - let set = byUser.get(scope.userId); - if (!set) { - set = new Set(); - byUser.set(scope.userId, set); + // Remove SQL audiobook rows for namespaced objects (covers auth claim flows too). + const audiobookKeys = await listKeysByPrefix(audiobooksNsRootPrefix); + const byUser = new Map>(); + for (const key of audiobookKeys) { + const scope = parseAudiobookScopeFromKey(key, audiobooksNsRootPrefix); + if (!scope) continue; + let set = byUser.get(scope.userId); + if (!set) { + set = new Set(); + byUser.set(scope.userId, set); + } + set.add(scope.bookId); } - set.add(scope.bookId); + + for (const [userId, bookIds] of byUser) { + for (const ids of chunk(Array.from(bookIds), 200)) { + await db + .delete(audiobookChapters) + .where(and(eq(audiobookChapters.userId, userId), inArray(audiobookChapters.bookId, ids))); + await db + .delete(audiobooks) + .where(and(eq(audiobooks.userId, userId), inArray(audiobooks.id, ids))); + } + } + + await deleteDocumentPrefix(docsNsRootPrefix); + await deleteDocumentPrefix(documentPreviewsNsRootPrefix); + await deleteDocumentPrefix(tempUploadsNsRootPrefix); + await deleteAudiobookPrefix(audiobooksNsRootPrefix); + await deleteTtsSegmentPrefix(ttsSegmentsNsRootPrefixV1); + await deleteTtsSegmentPrefix(ttsSegmentsNsRootPrefixV2); } - for (const [userId, bookIds] of byUser) { - for (const ids of chunk(Array.from(bookIds), 200)) { - await db - .delete(audiobookChapters) - .where(and(eq(audiobookChapters.userId, userId), inArray(audiobookChapters.bookId, ids))); - await db - .delete(audiobooks) - .where(and(eq(audiobooks.userId, userId), inArray(audiobooks.id, ids))); - } + for (const ids of chunk(testUserIds, 200)) { + await db + .delete(authSchema.user) + .where(and(inArray(authSchema.user.id, ids), eq(authSchema.user.isAnonymous, true))); } - - await deleteDocumentPrefix(docsNsRootPrefix); - await deleteAudiobookPrefix(audiobooksNsRootPrefix); - await deleteTtsSegmentPrefix(ttsSegmentsNsRootPrefixV1); - await deleteTtsSegmentPrefix(ttsSegmentsNsRootPrefixV2); } diff --git a/tests/unit/blob-upload-finalize-docx.vitest.spec.ts b/tests/unit/blob-upload-finalize-docx.vitest.spec.ts index b818d66..fec1efc 100644 --- a/tests/unit/blob-upload-finalize-docx.vitest.spec.ts +++ b/tests/unit/blob-upload-finalize-docx.vitest.spec.ts @@ -24,6 +24,10 @@ vi.mock('@/lib/server/documents/register-upload', () => ({ registerUploadedDocument: hoisted.registerUploadedDocument, })); +vi.mock('@/lib/server/documents/blob-lease', () => ({ + withDocumentBlobLease: vi.fn(async (_documentId: string, fn: () => Promise) => fn()), +})); + vi.mock('@/lib/server/documents/docx-convert', () => ({ convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer, })); diff --git a/tests/unit/data-export.vitest.spec.ts b/tests/unit/data-export.vitest.spec.ts new file mode 100644 index 0000000..bfc7806 --- /dev/null +++ b/tests/unit/data-export.vitest.spec.ts @@ -0,0 +1,154 @@ +import { describe, expect, test, vi } from 'vitest'; +import type { Archiver } from 'archiver'; +import { appendUserExportArchive } from '../../src/lib/server/user/data-export'; + +describe('user data export archive', () => { + test('includes user metadata and cached TTS audio without credential secrets', async () => { + const entries = new Map(); + const archive = { + append(value: unknown, options: { name: string }) { + entries.set(options.name, value); + return archive; + }, + } as unknown as Archiver; + + await appendUserExportArchive({ + archive, + userId: 'user-1', + exportedAtMs: 123, + profileData: { user: { id: 'user-1', email: 'person@example.com' }, exportedAtMs: 123 }, + preferences: { dataJson: '{}' }, + readingHistory: [{ documentId: 'doc-1' }], + ttsUsage: [{ charCount: 10 }], + jobEvents: [{ action: 'pdf-layout' }], + documentSettings: [{ documentId: 'doc-1', dataJson: '{}' }], + authSessions: [{ id: 'session-1', ipAddress: null }], + linkedAccounts: [{ id: 'account-1', providerId: 'credential' }], + documents: [], + audiobooks: [], + audiobookChapters: [], + ttsSegmentEntries: [{ segmentEntryId: 'entry-1', documentId: 'doc-1' }], + ttsSegmentVariants: [{ + segmentId: 'segment-1', + segmentEntryId: 'entry-1', + audioKey: 'private/audio/key', + audioFormat: 'mp3', + }], + storageEnabled: true, + getDocumentBlobStream: async () => new Uint8Array(), + listAudiobookObjects: async () => [], + getAudiobookObjectStream: async () => new Uint8Array(), + getTtsSegmentAudioStream: async () => new Uint8Array([1, 2, 3]), + }); + + expect(entries.has('document_settings.json')).toBe(true); + expect(entries.has('auth_sessions.json')).toBe(true); + expect(entries.has('linked_accounts.json')).toBe(true); + expect(entries.has('tts_segment_entries.json')).toBe(true); + expect(entries.has('tts_segment_variants.json')).toBe(true); + expect(entries.has('files/tts_segments/doc-1/segment-1.mp3')).toBe(true); + + const manifest = JSON.parse(String(entries.get('export_manifest.json'))); + expect(manifest).toMatchObject({ + formatVersion: 3, + counts: { + ttsSegmentEntriesMetadata: 1, + ttsSegmentVariantsMetadata: 1, + ttsSegmentFiles: 1, + }, + includes: { + credentialSecrets: false, + }, + }); + }); + + test('exports every TTS variant even when variants share an audio key', async () => { + const entries = new Map(); + const archive = { + append(value: unknown, options: { name: string }) { + entries.set(options.name, value); + return archive; + }, + } as unknown as Archiver; + const getTtsSegmentAudioStream = vi.fn(async () => new Uint8Array([1])); + + await appendUserExportArchive({ + archive, + userId: 'user-1', + exportedAtMs: 123, + profileData: {}, + preferences: null, + readingHistory: [], + ttsUsage: [], + jobEvents: [], + documentSettings: [], + authSessions: [], + linkedAccounts: [], + documents: [], + audiobooks: [], + audiobookChapters: [], + ttsSegmentEntries: [{ segmentEntryId: 'entry-1', documentId: 'doc-1' }], + ttsSegmentVariants: [ + { segmentId: 'segment-1', segmentEntryId: 'entry-1', audioKey: 'shared-key', audioFormat: 'mp3' }, + { segmentId: 'segment-2', segmentEntryId: 'entry-1', audioKey: 'shared-key', audioFormat: 'mp3' }, + ], + storageEnabled: true, + getDocumentBlobStream: async () => new Uint8Array(), + listAudiobookObjects: async () => [], + getAudiobookObjectStream: async () => new Uint8Array(), + getTtsSegmentAudioStream, + }); + + expect(entries.has('files/tts_segments/doc-1/segment-1.mp3')).toBe(true); + expect(entries.has('files/tts_segments/doc-1/segment-2.mp3')).toBe(true); + expect(getTtsSegmentAudioStream).toHaveBeenCalledTimes(2); + }); + + test('reports file buckets as excluded when storage is disabled', async () => { + const entries = new Map(); + const archive = { + append(value: unknown, options: { name: string }) { + entries.set(options.name, value); + return archive; + }, + } as unknown as Archiver; + const getDocumentBlobStream = vi.fn(async () => new Uint8Array()); + + await appendUserExportArchive({ + archive, + userId: 'user-1', + exportedAtMs: 123, + profileData: {}, + preferences: null, + readingHistory: [], + ttsUsage: [], + jobEvents: [], + documentSettings: [], + authSessions: [], + linkedAccounts: [], + documents: [{ id: 'doc-1', name: 'doc.pdf' }], + audiobooks: [], + audiobookChapters: [], + ttsSegmentEntries: [], + ttsSegmentVariants: [], + storageEnabled: false, + getDocumentBlobStream, + listAudiobookObjects: async () => [], + getAudiobookObjectStream: async () => new Uint8Array(), + getTtsSegmentAudioStream: async () => new Uint8Array(), + }); + + const manifest = JSON.parse(String(entries.get('export_manifest.json'))); + expect(manifest.includes).toMatchObject({ + documentFiles: false, + audiobookFiles: false, + ttsSegmentFiles: false, + }); + expect(manifest.counts).toMatchObject({ + documentFiles: 0, + audiobookFiles: 0, + ttsSegmentFiles: 0, + }); + expect(getDocumentBlobStream).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/document-blob-lease.vitest.spec.ts b/tests/unit/document-blob-lease.vitest.spec.ts new file mode 100644 index 0000000..684d75b --- /dev/null +++ b/tests/unit/document-blob-lease.vitest.spec.ts @@ -0,0 +1,59 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import * as sqliteSchema from '../../src/db/schema_sqlite'; + +const holder = vi.hoisted(() => ({ db: null as unknown as ReturnType })); +vi.mock('@/db', () => ({ + get db() { + return holder.db; + }, +})); + +import { tryAcquireDocumentBlobLease } from '../../src/lib/server/documents/blob-lease'; + +beforeEach(() => { + const sqlite = new Database(':memory:'); + sqlite.exec(`CREATE TABLE document_blob_leases ( + document_id text PRIMARY KEY NOT NULL, + lease_owner text NOT NULL, + lease_until_ms integer NOT NULL + );`); + holder.db = drizzle(sqlite, { schema: sqliteSchema }); +}); + +describe('document blob lease', () => { + test('allows only one owner until the lease is released', async () => { + const first = await tryAcquireDocumentBlobLease('doc-1'); + const blocked = await tryAcquireDocumentBlobLease('doc-1'); + + expect(first).not.toBeNull(); + expect(blocked).toBeNull(); + + await first?.release(); + await expect(tryAcquireDocumentBlobLease('doc-1')).resolves.not.toBeNull(); + }); + + test('allows an expired lease to be reclaimed', async () => { + const first = await tryAcquireDocumentBlobLease('doc-1'); + expect(first).not.toBeNull(); + + await holder.db + .update(sqliteSchema.documentBlobLeases) + .set({ leaseUntilMs: Date.now() - 1 }); + const replacement = await tryAcquireDocumentBlobLease('doc-1'); + + expect(replacement).not.toBeNull(); + expect(replacement?.owner).not.toBe(first?.owner); + + // Releasing the now-stale original lease must not delete the replacement, + // since release() is scoped to the original owner. + await first?.release(); + + const rows = await holder.db + .select() + .from(sqliteSchema.documentBlobLeases); + expect(rows).toHaveLength(1); + expect(rows[0]?.leaseOwner).toBe(replacement?.owner); + }); +}); diff --git a/tests/unit/document-delete-cleanup.vitest.spec.ts b/tests/unit/document-delete-cleanup.vitest.spec.ts new file mode 100644 index 0000000..962905e --- /dev/null +++ b/tests/unit/document-delete-cleanup.vitest.spec.ts @@ -0,0 +1,80 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + deleteResults: [] as unknown[][], + deleteDocumentTtsSegmentCache: vi.fn(async () => undefined), +})); + +function resultBuilder(result: unknown[]) { + return { + then: (resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) => + Promise.resolve(result).then(resolve, reject), + }; +} + +vi.mock('@/db', () => { + const database = { + delete: vi.fn(() => ({ + where: vi.fn(() => ({ + returning: () => resultBuilder(mocks.deleteResults.shift() ?? []), + })), + })), + }; + return { db: database }; +}); + +vi.mock('@/lib/server/tts/segments-cache', () => ({ + deleteDocumentTtsSegmentCache: mocks.deleteDocumentTtsSegmentCache, +})); + +vi.mock('@/lib/server/logger', () => ({ + hashForLog: () => 'hash', + serverLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock('@/lib/server/errors/logging', () => ({ logDegraded: vi.fn() })); + +import { deleteOwnedDocument } from '../../src/lib/server/documents/delete-owned'; + +describe('owned document cleanup', () => { + beforeEach(() => { + mocks.deleteResults = []; + mocks.deleteDocumentTtsSegmentCache.mockReset(); + mocks.deleteDocumentTtsSegmentCache.mockResolvedValue(undefined); + }); + + test('removes the ownership row and cleans the per-user TTS cache', async () => { + mocks.deleteResults = [[{ id: 'doc-1', userId: 'user-1' }]]; + + await expect(deleteOwnedDocument({ + userId: 'user-1', + documentId: 'doc-1', + namespace: null, + })).resolves.toBe(true); + + expect(mocks.deleteDocumentTtsSegmentCache).toHaveBeenCalledOnce(); + }); + + test('returns false and skips cleanup when no row was owned', async () => { + mocks.deleteResults = [[]]; + + await expect(deleteOwnedDocument({ + userId: 'user-1', + documentId: 'doc-1', + namespace: null, + })).resolves.toBe(false); + + expect(mocks.deleteDocumentTtsSegmentCache).not.toHaveBeenCalled(); + }); + + test('still succeeds if TTS cache cleanup fails (best effort)', async () => { + mocks.deleteResults = [[{ id: 'doc-1', userId: 'user-1' }]]; + mocks.deleteDocumentTtsSegmentCache.mockRejectedValueOnce(new Error('storage unavailable')); + + await expect(deleteOwnedDocument({ + userId: 'user-1', + documentId: 'doc-1', + namespace: null, + })).resolves.toBe(true); + }); +}); diff --git a/tests/unit/reap-orphaned-blobs.vitest.spec.ts b/tests/unit/reap-orphaned-blobs.vitest.spec.ts new file mode 100644 index 0000000..667a242 --- /dev/null +++ b/tests/unit/reap-orphaned-blobs.vitest.spec.ts @@ -0,0 +1,109 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + del: vi.fn(async () => undefined), + owned: [] as Array<{ id: string }>, + ownedAfterLease: [] as Array<{ id: string }>, + leaseRelease: vi.fn(async () => undefined), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ isS3Configured: () => true })); +vi.mock('@/lib/server/documents/blobstore', () => ({ + listDocumentSourceBlobs: mocks.list, + deleteDocumentBlob: mocks.del, +})); +vi.mock('@/lib/server/documents/previews-blobstore', () => ({ + deleteDocumentPreviewArtifacts: vi.fn(async () => 0), +})); +vi.mock('@/lib/server/documents/previews', () => ({ + deleteDocumentPreviewRows: vi.fn(async () => undefined), +})); +vi.mock('@/lib/server/documents/blob-lease', () => ({ + tryAcquireDocumentBlobLease: vi.fn(async () => ({ + owner: 'lease-owner', + release: mocks.leaseRelease, + })), +})); +vi.mock('@/lib/server/errors/logging', () => ({ logDegraded: vi.fn() })); +vi.mock('@/db', () => ({ + db: { + select: () => ({ + from: () => ({ + where: () => { + const rows = mocks.owned; + mocks.owned = mocks.ownedAfterLease; + return Object.assign(Promise.resolve(rows), { + limit: () => Promise.resolve(rows.slice(0, 1)), + }); + }, + }), + }), + }, +})); + +import { reapOrphanedBlobs } from '../../src/lib/server/tasks/handlers/reap-orphaned-blobs'; + +const TWO_HOURS = 2 * 60 * 60 * 1000; +const context = () => ({ signal: new AbortController().signal, deadlineAt: Date.now() + 60_000 }); + +beforeEach(() => { + mocks.list.mockReset(); + mocks.del.mockReset(); + mocks.del.mockResolvedValue(undefined); + mocks.owned = []; + mocks.ownedAfterLease = []; + mocks.leaseRelease.mockClear(); +}); + +describe('reap-orphaned-blobs', () => { + test('reaps only old blobs with no owner', async () => { + const now = Date.now(); + mocks.list.mockResolvedValue([ + { id: 'orphan-old', lastModifiedMs: now - TWO_HOURS }, + { id: 'owned-old', lastModifiedMs: now - TWO_HOURS }, + { id: 'orphan-young', lastModifiedMs: now - 60_000 }, + ]); + mocks.owned = [{ id: 'owned-old' }]; + + const result = await reapOrphanedBlobs(context()); + + expect(mocks.del).toHaveBeenCalledTimes(1); + expect(mocks.del).toHaveBeenCalledWith('orphan-old', null); + expect(result.reaped).toBe(1); + expect(result.scanned).toBe(3); + expect(result.candidates).toBe(2); + }); + + test('reaps nothing when every old blob still has an owner', async () => { + const now = Date.now(); + mocks.list.mockResolvedValue([{ id: 'kept', lastModifiedMs: now - TWO_HOURS }]); + mocks.owned = [{ id: 'kept' }]; + + const result = await reapOrphanedBlobs(context()); + + expect(mocks.del).not.toHaveBeenCalled(); + expect(result.reaped).toBe(0); + }); + + test('does not delete when an owner appears before the post-lease recheck', async () => { + const now = Date.now(); + mocks.list.mockResolvedValue([{ id: 'claimed-during-run', lastModifiedMs: now - TWO_HOURS }]); + mocks.ownedAfterLease = [{ id: 'claimed-during-run' }]; + + const result = await reapOrphanedBlobs(context()); + + expect(mocks.del).not.toHaveBeenCalled(); + expect(result.reaped).toBe(0); + expect(mocks.leaseRelease).toHaveBeenCalledTimes(1); + }); + + test('fails the task when an orphan deletion fails', async () => { + const now = Date.now(); + mocks.list.mockResolvedValue([{ id: 'failed-orphan', lastModifiedMs: now - TWO_HOURS }]); + mocks.del.mockRejectedValue(new Error('storage failed')); + + await expect(reapOrphanedBlobs(context())).rejects.toThrow('Failed to reap 1'); + expect(mocks.leaseRelease).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/unit/scheduled-tasks-engine.vitest.spec.ts b/tests/unit/scheduled-tasks-engine.vitest.spec.ts new file mode 100644 index 0000000..715bac4 --- /dev/null +++ b/tests/unit/scheduled-tasks-engine.vitest.spec.ts @@ -0,0 +1,243 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { eq } from 'drizzle-orm'; +import * as sqliteSchema from '../../src/db/schema_sqlite'; + +// Back the engine with a real in-memory SQLite so the CAS claim, due-detection, +// and nextRunAt arithmetic are exercised against real SQL rather than mocks. +const holder = vi.hoisted(() => ({ db: null as unknown as ReturnType })); +vi.mock('@/db', () => ({ + get db() { + return holder.db; + }, +})); + +// Keep the error-path test from printing the expected failure to the console. +vi.mock('@/lib/server/errors/logging', () => ({ logDegraded: vi.fn() })); + +import { runDueTasks, updateTask } from '../../src/lib/server/tasks/engine'; +import type { TaskRegistry } from '../../src/lib/server/tasks/types'; + +const tasks = sqliteSchema.scheduledTasks; + +const CREATE_TABLE = `CREATE TABLE scheduled_tasks ( + key text PRIMARY KEY NOT NULL, + enabled integer DEFAULT true NOT NULL, + interval_ms integer NOT NULL, + last_status text DEFAULT 'idle' NOT NULL, + lease_owner text, + last_run_at integer, + last_duration_ms integer, + last_error text, + last_result_json text, + next_run_at integer, + run_requested integer DEFAULT false NOT NULL, + running_since integer, + updated_at integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL, + CONSTRAINT scheduled_tasks_interval_ms_positive CHECK(interval_ms > 0) +);`; + +const KEY = 'test-task'; + +async function seedRow(overrides: Partial) { + await holder.db.insert(tasks).values({ + key: KEY, + enabled: true, + intervalMs: 1000, + lastStatus: 'idle', + nextRunAt: Date.now() - 1, + runRequested: false, + ...overrides, + }); +} + +async function readRow() { + const rows = await holder.db.select().from(tasks).where(eq(tasks.key, KEY)); + return rows[0]; +} + +function registryWith(run: () => Promise<{ summary?: string } | void>): TaskRegistry { + return { [KEY]: { name: 'Test task', defaultIntervalMs: 1000, run } }; +} + +beforeEach(() => { + const sqlite = new Database(':memory:'); + sqlite.exec(CREATE_TABLE); + holder.db = drizzle(sqlite, { schema: sqliteSchema }); +}); + +describe('scheduled task engine', () => { + test('runs a due task and records success + next run', async () => { + const handler = vi.fn(async () => ({ summary: 'did 3 things' })); + await seedRow({ nextRunAt: Date.now() - 1 }); + + await runDueTasks({ registry: registryWith(handler) }); + + expect(handler).toHaveBeenCalledTimes(1); + const row = await readRow(); + expect(row.lastStatus).toBe('ok'); + expect(row.lastRunAt).not.toBeNull(); + expect(row.lastResultJson).toBe('did 3 things'); + expect(row.runningSince).toBeNull(); + expect(Number(row.nextRunAt)).toBeGreaterThan(Date.now()); + }); + + test('does not run a task that is not yet due', async () => { + const handler = vi.fn(async () => undefined); + await seedRow({ nextRunAt: Date.now() + 60_000, lastStatus: 'idle' }); + + await runDueTasks({ registry: registryWith(handler) }); + + expect(handler).not.toHaveBeenCalled(); + expect((await readRow()).lastStatus).toBe('idle'); + }); + + test('runs when a manual run is requested even if not due', async () => { + const handler = vi.fn(async () => ({ summary: 'manual' })); + await seedRow({ nextRunAt: Date.now() + 60_000, runRequested: true }); + + await runDueTasks({ registry: registryWith(handler) }); + + expect(handler).toHaveBeenCalledTimes(1); + const row = await readRow(); + expect(row.lastStatus).toBe('ok'); + expect(row.runRequested).toBe(false); + }); + + test('records errors without throwing', async () => { + const handler = vi.fn(async () => { + throw new Error('boom'); + }); + await seedRow({ nextRunAt: Date.now() - 1 }); + + await expect(runDueTasks({ registry: registryWith(handler) })).resolves.toBeUndefined(); + + const row = await readRow(); + expect(row.lastStatus).toBe('error'); + expect(row.lastError).toContain('boom'); + }); + + test('single-flight: skips a task already marked running', async () => { + const handler = vi.fn(async () => undefined); + await seedRow({ nextRunAt: Date.now() - 1, lastStatus: 'running', runningSince: Date.now() }); + + await runDueTasks({ registry: registryWith(handler) }); + + expect(handler).not.toHaveBeenCalled(); + expect((await readRow()).lastStatus).toBe('running'); + }); + + test('runs a requested task even when disabled', async () => { + const handler = vi.fn(async () => undefined); + await seedRow({ nextRunAt: Date.now() + 60_000, enabled: false, runRequested: true }); + + await runDueTasks({ registry: registryWith(handler) }); + + expect(handler).toHaveBeenCalledTimes(1); + }); + + test('preserves a manual rerun requested while a task is running', async () => { + const handler = vi.fn(async () => { + await holder.db + .update(tasks) + .set({ runRequested: true }) + .where(eq(tasks.key, KEY)); + }); + await seedRow({ nextRunAt: Date.now() - 1 }); + + await runDueTasks({ registry: registryWith(handler) }); + + expect(handler).toHaveBeenCalledTimes(1); + expect((await readRow()).runRequested).toBe(true); + }); + + test('does not let a stale runner overwrite its replacement run', async () => { + let resolveFirst!: (value: { summary: string }) => void; + const firstResult = new Promise<{ summary: string }>((resolve) => { + resolveFirst = resolve; + }); + const handler = vi.fn() + .mockImplementationOnce(() => firstResult) + .mockResolvedValueOnce({ summary: 'replacement' }); + await seedRow({ nextRunAt: Date.now() - 1 }); + + let now = Date.now(); + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => now); + const firstRun = runDueTasks({ registry: registryWith(handler) }); + await vi.waitFor(() => expect(handler).toHaveBeenCalledTimes(1)); + + now += 2 * 60 * 60 * 1000; + await runDueTasks({ registry: registryWith(handler) }); + expect((await readRow()).lastResultJson).toBe('replacement'); + + resolveFirst({ summary: 'stale original' }); + await firstRun; + expect((await readRow()).lastResultJson).toBe('replacement'); + nowSpy.mockRestore(); + }); + + test('starts independent due tasks concurrently', async () => { + const started: string[] = []; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const registry: TaskRegistry = { + first: { + name: 'First', + defaultIntervalMs: 1000, + run: async () => { + started.push('first'); + await gate; + }, + }, + second: { + name: 'Second', + defaultIntervalMs: 1000, + run: async () => { + started.push('second'); + await gate; + }, + }, + }; + + const running = runDueTasks({ registry }); + await vi.waitFor(() => expect(started).toEqual(['first', 'second'])); + release(); + await running; + }); + + test('aborts and records an error when a task exceeds its runtime limit', async () => { + const handler = vi.fn(async ({ signal }: { signal: AbortSignal }) => { + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }); + await seedRow({ nextRunAt: Date.now() - 1 }); + const registry: TaskRegistry = { + [KEY]: { name: 'Timed task', defaultIntervalMs: 1000, maxRunMs: 5, run: handler }, + }; + + await runDueTasks({ registry }); + + expect(handler).toHaveBeenCalledTimes(1); + expect((await readRow()).lastError).toContain('runtime limit'); + }); + + test('rejects non-positive intervals at the database boundary', async () => { + await expect(seedRow({ intervalMs: 0 })).rejects.toThrow(); + }); + + test('updates a registered task even when its row has not been seeded', async () => { + await updateTask('prune-job-events', { enabled: false, intervalMs: 12_345 }); + + const [row] = await holder.db + .select() + .from(tasks) + .where(eq(tasks.key, 'prune-job-events')); + expect(row).toEqual(expect.objectContaining({ + enabled: false, + intervalMs: 12_345, + lastStatus: 'idle', + })); + }); +}); diff --git a/tests/unit/scheduled-tasks-routes.vitest.spec.ts b/tests/unit/scheduled-tasks-routes.vitest.spec.ts new file mode 100644 index 0000000..cfb418f --- /dev/null +++ b/tests/unit/scheduled-tasks-routes.vitest.spec.ts @@ -0,0 +1,98 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const mocks = vi.hoisted(() => ({ + requireAdminContext: vi.fn(), + listTasks: vi.fn(), + runDueTasks: vi.fn(), + updateTask: vi.fn(), +})); + +vi.mock('@/lib/server/auth/admin', () => ({ + requireAdminContext: mocks.requireAdminContext, +})); +vi.mock('@/lib/server/tasks/engine', () => ({ + listTasks: mocks.listTasks, + runDueTasks: mocks.runDueTasks, + updateTask: mocks.updateTask, +})); + +describe('scheduled task routes', () => { + const originalCronSecret = process.env.CRON_SECRET; + const originalVercel = process.env.VERCEL; + + beforeEach(() => { + mocks.requireAdminContext.mockReset(); + mocks.requireAdminContext.mockResolvedValue({ userId: 'admin-1' }); + mocks.listTasks.mockReset(); + mocks.listTasks.mockResolvedValue([]); + mocks.runDueTasks.mockReset(); + mocks.runDueTasks.mockResolvedValue(undefined); + mocks.updateTask.mockReset(); + mocks.updateTask.mockResolvedValue(undefined); + delete process.env.CRON_SECRET; + delete process.env.VERCEL; + }); + + afterEach(() => { + if (originalCronSecret === undefined) delete process.env.CRON_SECRET; + else process.env.CRON_SECRET = originalCronSecret; + if (originalVercel === undefined) delete process.env.VERCEL; + else process.env.VERCEL = originalVercel; + }); + + test('cron tick requires the configured bearer secret', async () => { + const { GET } = await import('../../src/app/api/admin/tasks/tick/route'); + + const unconfigured = await GET(new NextRequest('http://localhost/api/admin/tasks/tick')); + expect(unconfigured.status).toBe(503); + + process.env.CRON_SECRET = 'cron-secret'; + const unauthorized = await GET(new NextRequest('http://localhost/api/admin/tasks/tick')); + expect(unauthorized.status).toBe(401); + + const authorized = await GET(new NextRequest('http://localhost/api/admin/tasks/tick', { + headers: { authorization: 'Bearer cron-secret' }, + })); + expect(authorized.status).toBe(200); + expect(mocks.runDueTasks).toHaveBeenCalledTimes(1); + }); + + test('admin task list enforces admin authorization', async () => { + const denied = new Response('Forbidden', { status: 403 }); + mocks.requireAdminContext.mockResolvedValue(denied); + const { GET } = await import('../../src/app/api/admin/tasks/route'); + + const response = await GET(new NextRequest('http://localhost/api/admin/tasks')); + + expect(response).toBe(denied); + expect(mocks.listTasks).not.toHaveBeenCalled(); + }); + + test('reports daily Vercel cadence and rejects shorter intervals', async () => { + process.env.VERCEL = '1'; + mocks.listTasks.mockResolvedValue([{ + key: 'cleanup-temp-uploads', + intervalMs: 60 * 60 * 1000, + }]); + const { GET } = await import('../../src/app/api/admin/tasks/route'); + const listResponse = await GET(new NextRequest('http://localhost/api/admin/tasks')); + await expect(listResponse.json()).resolves.toMatchObject({ + tasks: [{ intervalMs: 86_400_000 }], + scheduler: { + mode: 'vercel-cron', + minimumIntervalMs: 86_400_000, + }, + }); + + const { PATCH } = await import('../../src/app/api/admin/tasks/[key]/route'); + const patchResponse = await PATCH(new NextRequest('http://localhost/api/admin/tasks/cleanup-temp-uploads', { + method: 'PATCH', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ intervalMs: 60 * 60 * 1000 }), + }), { params: Promise.resolve({ key: 'cleanup-temp-uploads' }) }); + + expect(patchResponse.status).toBe(400); + expect(mocks.updateTask).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/storage-prefix-cleanup.vitest.spec.ts b/tests/unit/storage-prefix-cleanup.vitest.spec.ts new file mode 100644 index 0000000..7afb7de --- /dev/null +++ b/tests/unit/storage-prefix-cleanup.vitest.spec.ts @@ -0,0 +1,132 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + send: vi.fn(), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + getS3Config: () => ({ bucket: 'test-bucket', prefix: 'openreader-test' }), + getS3Client: () => ({ send: mocks.send }), + getS3ProxyClient: () => ({ send: mocks.send }), +})); + +import { deleteAudiobookPrefix } from '../../src/lib/server/audiobooks/blobstore'; +import { + deleteAllExpiredTempDocumentUploads, + deleteDocumentBlob, + deleteDocumentPrefix, +} from '../../src/lib/server/documents/blobstore'; + +describe('storage prefix cleanup', () => { + beforeEach(() => { + mocks.send.mockReset(); + }); + + test.each([ + ['document', deleteDocumentPrefix], + ['audiobook', deleteAudiobookPrefix], + ])('%s cleanup counts successful quiet deletes', async (_name, removePrefix) => { + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'prefix/a' }, { Key: 'prefix/b' }], + IsTruncated: false, + }) + .mockResolvedValueOnce({}); + + await expect(removePrefix('prefix/')).resolves.toBe(2); + }); + + test.each([ + ['document', deleteDocumentPrefix], + ['audiobook', deleteAudiobookPrefix], + ])('%s cleanup fails on per-object storage errors', async (_name, removePrefix) => { + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'prefix/a' }], + IsTruncated: false, + }) + .mockResolvedValueOnce({ + Errors: [{ Key: 'prefix/a', Code: 'AccessDenied' }], + }); + + await expect(removePrefix('prefix/')).rejects.toThrow('Failed deleting 1'); + }); + + test('deletes the source after derived artifacts and then sweeps late parsed output', async () => { + mocks.send + .mockResolvedValueOnce({ Contents: [], IsTruncated: false }) + .mockResolvedValueOnce({ Contents: [], IsTruncated: false }) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({ Contents: [], IsTruncated: false }); + + const documentId = 'a'.repeat(64); + await deleteDocumentBlob(documentId, null); + + const commands = mocks.send.mock.calls.map(([command]) => command); + const sourceDeleteIndex = commands.findIndex((command) => + command.constructor.name === 'DeleteObjectCommand' + && command.input.Key === `openreader-test/documents_v1/${documentId}`); + const finalCommand = commands.at(-1); + + expect(sourceDeleteIndex).toBeGreaterThan(1); + expect(finalCommand?.constructor.name).toBe('ListObjectsV2Command'); + expect(finalCommand?.input.Prefix).toBe( + `openreader-test/documents_v1/parsed_v2/${documentId}/`, + ); + }); + + test('keeps the source document when derived-artifact cleanup fails', async () => { + mocks.send.mockRejectedValueOnce(new Error('list failed')); + + const documentId = 'b'.repeat(64); + await expect(deleteDocumentBlob(documentId, null)).rejects.toThrow('list failed'); + + const sourceDelete = mocks.send.mock.calls + .map(([command]) => command) + .find((command) => command.constructor.name === 'DeleteObjectCommand' + && command.input.Key === `openreader-test/documents_v1/${documentId}`); + expect(sourceDelete).toBeUndefined(); + }); + + test('deletes expired temporary uploads page by page', async () => { + const old = new Date(Date.now() - 60_000); + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'temp/a', LastModified: old }], + IsTruncated: true, + NextContinuationToken: 'next', + }) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({ + Contents: [{ Key: 'temp/b', LastModified: old }], + IsTruncated: false, + }) + .mockResolvedValueOnce({}); + + await expect(deleteAllExpiredTempDocumentUploads(null, Date.now())).resolves.toBe(2); + + expect(mocks.send.mock.calls.map(([command]) => command.constructor.name)).toEqual([ + 'ListObjectsV2Command', + 'DeleteObjectsCommand', + 'ListObjectsV2Command', + 'DeleteObjectsCommand', + ]); + }); + + test('temporary upload cleanup fails on per-object storage errors', async () => { + const old = new Date(Date.now() - 60_000); + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'temp/a', LastModified: old }], + IsTruncated: false, + }) + .mockResolvedValueOnce({ + Errors: [{ Key: 'temp/a', Code: 'AccessDenied' }], + }); + + await expect(deleteAllExpiredTempDocumentUploads(null, Date.now())) + .rejects.toThrow('test-bucket'); + }); +}); diff --git a/tests/unit/transfer-user-documents.vitest.spec.ts b/tests/unit/transfer-user-documents.vitest.spec.ts index f1c0bed..1444d1d 100644 --- a/tests/unit/transfer-user-documents.vitest.spec.ts +++ b/tests/unit/transfer-user-documents.vitest.spec.ts @@ -2,8 +2,13 @@ import { describe, expect, test } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; import { eq } from 'drizzle-orm'; -import { documents } from '../../src/db/schema_sqlite'; -import { transferUserDocuments } from '../../src/lib/server/user/claim-data'; +import { + documentSettings, + documents, + ttsSegmentEntries, + ttsSegmentVariants, +} from '../../src/db/schema_sqlite'; +import { transferUserDocumentSettings, transferUserDocuments } from '../../src/lib/server/user/claim-data'; describe('transferUserDocuments', () => { test('moves document rows to new user without PK conflicts', async () => { @@ -73,4 +78,174 @@ describe('transferUserDocuments', () => { const ids = remainingTo.map((r) => r.id).sort(); expect(ids).toEqual(['doc-a', 'doc-b']); }); + + test('moves document settings while preserving newer destination settings', async () => { + const sqlite = new Database(':memory:'); + sqlite.exec(` + CREATE TABLE document_settings ( + document_id TEXT NOT NULL, + user_id TEXT NOT NULL, + data_json TEXT NOT NULL DEFAULT '{}', + client_updated_at_ms INTEGER NOT NULL DEFAULT 0, + created_at INTEGER, + updated_at INTEGER, + PRIMARY KEY (document_id, user_id) + ); + `); + const settingsDb = drizzle(sqlite); + + await settingsDb.insert(documentSettings).values([ + { + documentId: 'doc-newer-anon', + userId: 'anon', + dataJson: '{"source":"anon"}', + clientUpdatedAtMs: 20, + }, + { + documentId: 'doc-newer-user', + userId: 'anon', + dataJson: '{"source":"anon"}', + clientUpdatedAtMs: 10, + }, + { + documentId: 'doc-newer-user', + userId: 'user', + dataJson: '{"source":"user"}', + clientUpdatedAtMs: 30, + }, + ]); + + const transferred = await transferUserDocumentSettings('anon', 'user', { + db: settingsDb, + }); + expect(transferred).toBe(2); + + const rows = await settingsDb.select().from(documentSettings); + expect(rows).toHaveLength(2); + expect(rows).toEqual(expect.arrayContaining([ + expect.objectContaining({ + documentId: 'doc-newer-anon', + userId: 'user', + dataJson: '{"source":"anon"}', + clientUpdatedAtMs: 20, + }), + expect.objectContaining({ + documentId: 'doc-newer-user', + userId: 'user', + dataJson: '{"source":"user"}', + clientUpdatedAtMs: 30, + }), + ])); + }); + + test('moves TTS metadata without requiring S3', async () => { + const sqlite = new Database(':memory:'); + sqlite.exec(` + CREATE TABLE documents ( + id TEXT NOT NULL, + user_id TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + size INTEGER NOT NULL, + last_modified INTEGER NOT NULL, + file_path TEXT NOT NULL, + parse_state TEXT, + parsed_json_key TEXT, + created_at INTEGER, + PRIMARY KEY (id, user_id) + ); + CREATE TABLE tts_segment_entries ( + segment_entry_id TEXT NOT NULL, + user_id TEXT NOT NULL, + document_id TEXT NOT NULL, + reader_type TEXT NOT NULL, + document_version INTEGER NOT NULL, + segment_index INTEGER NOT NULL, + segment_key TEXT, + locator_reader_rank INTEGER NOT NULL, + locator_reader_type TEXT NOT NULL, + locator_page INTEGER NOT NULL, + locator_spine_index INTEGER NOT NULL, + locator_spine_href TEXT NOT NULL, + locator_char_offset INTEGER NOT NULL, + locator_location TEXT NOT NULL, + locator_identity_key TEXT NOT NULL, + text_hash TEXT NOT NULL, + text_length INTEGER NOT NULL DEFAULT 0, + created_at INTEGER, + updated_at INTEGER, + PRIMARY KEY (segment_entry_id, user_id) + ); + CREATE TABLE tts_segment_variants ( + segment_id TEXT NOT NULL, + user_id TEXT NOT NULL, + segment_entry_id TEXT NOT NULL, + settings_hash TEXT NOT NULL, + settings_json TEXT NOT NULL, + audio_key TEXT, + audio_format TEXT NOT NULL DEFAULT 'mp3', + duration_ms INTEGER, + alignment_json TEXT, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + created_at INTEGER, + updated_at INTEGER, + PRIMARY KEY (segment_id, user_id) + ); + `); + const transferDb = drizzle(sqlite); + + await transferDb.insert(documents).values({ + id: 'doc', + userId: 'anon', + name: 'doc.pdf', + type: 'pdf', + size: 1, + lastModified: 1, + filePath: 'doc__doc.pdf', + }); + await transferDb.insert(ttsSegmentEntries).values({ + segmentEntryId: 'entry', + userId: 'anon', + documentId: 'doc', + readerType: 'pdf', + documentVersion: 1, + segmentIndex: 0, + locatorReaderRank: 0, + locatorReaderType: 'pdf', + locatorPage: 1, + locatorSpineIndex: 0, + locatorSpineHref: '', + locatorCharOffset: 0, + locatorLocation: '1', + locatorIdentityKey: 'page:1', + textHash: 'hash', + }); + await transferDb.insert(ttsSegmentVariants).values({ + segmentId: 'segment', + userId: 'anon', + segmentEntryId: 'entry', + settingsHash: 'settings', + settingsJson: '{}', + audioKey: 'openreader/tts_segments_v2/users/anon/docs/doc/1/settings/segment.mp3', + status: 'completed', + }); + + await transferUserDocuments('anon', 'user', { + db: transferDb, + transferTts: true, + skipStorage: true, + }); + + expect(await transferDb.select().from(ttsSegmentEntries)).toEqual([ + expect.objectContaining({ segmentEntryId: 'entry', userId: 'user', documentId: 'doc' }), + ]); + expect(await transferDb.select().from(ttsSegmentVariants)).toEqual([ + expect.objectContaining({ + segmentId: 'segment', + userId: 'user', + audioKey: 'openreader/tts_segments_v2/users/user/docs/doc/1/settings/segment.mp3', + }), + ]); + }); }); diff --git a/tests/unit/tts-segments-blobstore.vitest.spec.ts b/tests/unit/tts-segments-blobstore.vitest.spec.ts new file mode 100644 index 0000000..19be3f2 --- /dev/null +++ b/tests/unit/tts-segments-blobstore.vitest.spec.ts @@ -0,0 +1,76 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + send: vi.fn(), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + getS3Config: () => ({ bucket: 'test-bucket' }), + getS3Client: () => ({ send: mocks.send }), + getS3ProxyClient: () => ({ send: mocks.send }), +})); + +import { copyTtsSegmentPrefix, deleteTtsSegmentPrefix } from '../../src/lib/server/tts/segments-blobstore'; + +describe('TTS segment blob cleanup', () => { + beforeEach(() => { + mocks.send.mockReset(); + }); + + test('counts successful quiet deletes by requested object count', async () => { + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'prefix/a.mp3' }, { Key: 'prefix/b.mp3' }], + IsTruncated: false, + }) + .mockResolvedValueOnce({}); + + await expect(deleteTtsSegmentPrefix('prefix/')).resolves.toBe(2); + }); + + test('fails cleanup when storage reports per-object deletion errors', async () => { + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'prefix/a.mp3' }], + IsTruncated: false, + }) + .mockResolvedValueOnce({ + Errors: [{ Key: 'prefix/a.mp3', Code: 'AccessDenied' }], + }); + + await expect(deleteTtsSegmentPrefix('prefix/')).rejects.toThrow( + 'Failed deleting 1 TTS segment audio objects', + ); + }); + + test('copies a user-scoped prefix without deleting its source objects', async () => { + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'source/doc/audio.mp3' }], + IsTruncated: false, + }) + .mockResolvedValueOnce({}); + + await expect(copyTtsSegmentPrefix('source/doc/', 'destination/doc/')).resolves.toBe(1); + + const copyCommand = mocks.send.mock.calls[1]?.[0]; + expect(copyCommand.constructor.name).toBe('CopyObjectCommand'); + expect(copyCommand.input).toMatchObject({ + Key: 'destination/doc/audio.mp3', + CopySource: 'test-bucket/source/doc/audio.mp3', + }); + expect(mocks.send).toHaveBeenCalledTimes(2); + }); + + test('does not delete source objects when a TTS prefix copy fails', async () => { + mocks.send + .mockResolvedValueOnce({ + Contents: [{ Key: 'source/doc/audio.mp3' }], + IsTruncated: false, + }) + .mockRejectedValueOnce(new Error('copy failed')); + + await expect(copyTtsSegmentPrefix('source/doc/', 'destination/doc/')).rejects.toThrow('copy failed'); + expect(mocks.send).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/unit/tts-segments-cache.vitest.spec.ts b/tests/unit/tts-segments-cache.vitest.spec.ts new file mode 100644 index 0000000..18ce89b --- /dev/null +++ b/tests/unit/tts-segments-cache.vitest.spec.ts @@ -0,0 +1,78 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + selectResults: [] as unknown[][], + deleteWhere: vi.fn(async () => undefined), + deleteTtsSegmentAudioObjects: vi.fn(async () => 0), +})); + +function resultBuilder(result: unknown[]) { + return { + innerJoin: vi.fn(() => ({ + where: vi.fn(async () => result), + })), + where: vi.fn(async () => result), + }; +} + +vi.mock('@/db', () => ({ + db: { + select: vi.fn(() => ({ + from: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])), + })), + delete: vi.fn(() => ({ + where: mocks.deleteWhere, + })), + }, +})); + +vi.mock('@/lib/server/tts/segments-blobstore', () => ({ + deleteTtsSegmentAudioObjects: mocks.deleteTtsSegmentAudioObjects, + deleteTtsSegmentPrefix: vi.fn(async () => 0), +})); + +vi.mock('@/lib/server/storage/s3', () => ({ + getS3Config: () => ({ prefix: 'openreader-test' }), +})); + +vi.mock('@/lib/server/logger', () => ({ + serverLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock('@/lib/server/errors/logging', () => ({ + logDegraded: vi.fn(), +})); + +import { clearTtsSegmentCache } from '../../src/lib/server/tts/segments-cache'; + +describe('TTS segment cache cleanup', () => { + beforeEach(() => { + mocks.selectResults = []; + mocks.deleteWhere.mockReset(); + mocks.deleteWhere.mockResolvedValue(undefined); + mocks.deleteTtsSegmentAudioObjects.mockReset(); + mocks.deleteTtsSegmentAudioObjects.mockResolvedValue(2); + }); + + test('counts deleted entries rather than joined variants', async () => { + mocks.selectResults = [ + [{ segmentEntryId: 'entry-1' }, { segmentEntryId: 'entry-2' }], + [ + { segmentId: 'variant-1', audioKey: 'audio-1' }, + { segmentId: 'variant-2', audioKey: 'audio-2' }, + { segmentId: 'variant-3', audioKey: 'audio-2' }, + ], + ]; + + const result = await clearTtsSegmentCache({ + userId: 'user-1', + documentId: 'doc-1', + }); + + expect(result).toMatchObject({ + deletedSegments: 2, + requestedAudioObjects: 2, + deletedAudioObjects: 2, + }); + }); +}); diff --git a/tests/unit/tts-segments.vitest.spec.ts b/tests/unit/tts-segments.vitest.spec.ts index a448afe..a257c23 100644 --- a/tests/unit/tts-segments.vitest.spec.ts +++ b/tests/unit/tts-segments.vitest.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'vitest'; import { + buildTtsSegmentDocumentPrefix, buildProportionalAlignment, buildTtsSegmentEntryId, buildTtsSegmentId, @@ -12,6 +13,15 @@ import { } from '../../src/lib/server/tts/segments'; describe('tts segment helpers', () => { + test('builds a user/document-scoped audio prefix across every version and variant', () => { + expect(buildTtsSegmentDocumentPrefix({ + storagePrefix: 'openreader', + namespace: 'test namespace', + userId: 'user/name', + documentId: 'doc-id', + })).toBe('openreader/tts_segments_v2/ns/test namespace/users/user%2Fname/docs/doc-id/'); + }); + test('builds stable settings hash', () => { const a = buildTtsSegmentSettingsHash({ providerRef: 'openai', diff --git a/tests/unit/user-data-cleanup.vitest.spec.ts b/tests/unit/user-data-cleanup.vitest.spec.ts new file mode 100644 index 0000000..ff8496b --- /dev/null +++ b/tests/unit/user-data-cleanup.vitest.spec.ts @@ -0,0 +1,134 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + selectResults: [] as unknown[][], + deleteWhere: vi.fn(async () => undefined), + deleteDocumentBlob: vi.fn(async () => undefined), + deleteDocumentPrefix: vi.fn(async () => 0), + deleteDocumentPreviewArtifacts: vi.fn(async () => 0), + deleteDocumentPreviewRows: vi.fn(async () => undefined), + deleteAudiobookPrefix: vi.fn(async () => 0), + deleteTtsSegmentPrefix: vi.fn(async () => 0), +})); + +function resultBuilder(result: unknown[]) { + return { + then: (resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) => + Promise.resolve(result).then(resolve, reject), + }; +} + +vi.mock('@/db', () => { + const database = { + select: vi.fn(() => ({ + from: vi.fn(() => ({ + where: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])), + })), + })), + delete: vi.fn(() => ({ + where: vi.fn(() => { + const promise = mocks.deleteWhere(); + return { + then: promise.then.bind(promise), + catch: promise.catch.bind(promise), + }; + }), + })), + }; + return { db: database }; +}); + +vi.mock('@/lib/server/storage/s3', () => ({ + isS3Configured: () => true, + getS3Config: () => ({ prefix: 'openreader-test' }), +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + deleteDocumentBlob: mocks.deleteDocumentBlob, + deleteDocumentPrefix: mocks.deleteDocumentPrefix, + tempDocumentUploadPrefix: () => 'temp/user/', +})); + +vi.mock('@/lib/server/documents/previews-blobstore', () => ({ + deleteDocumentPreviewArtifacts: mocks.deleteDocumentPreviewArtifacts, +})); + +vi.mock('@/lib/server/documents/previews', () => ({ + deleteDocumentPreviewRows: mocks.deleteDocumentPreviewRows, +})); + +vi.mock('@/lib/server/audiobooks/blobstore', () => ({ + audiobookPrefix: () => 'audiobooks/user/', + deleteAudiobookPrefix: mocks.deleteAudiobookPrefix, +})); + +vi.mock('@/lib/server/tts/segments-blobstore', () => ({ + deleteTtsSegmentPrefix: mocks.deleteTtsSegmentPrefix, +})); + +vi.mock('@/lib/server/logger', () => ({ + hashForLog: () => 'hash', + serverLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock('@/lib/server/errors/logging', () => ({ + logDegraded: vi.fn(), +})); + +import { deleteUserStorageData } from '../../src/lib/server/user/data-cleanup'; + +describe('user data cleanup', () => { + beforeEach(() => { + mocks.selectResults = []; + for (const mock of Object.values(mocks)) { + if (typeof mock === 'function' && 'mockReset' in mock) { + mock.mockReset(); + } + } + mocks.deleteWhere.mockResolvedValue(undefined); + mocks.deleteDocumentBlob.mockResolvedValue(undefined); + mocks.deleteDocumentPrefix.mockResolvedValue(0); + mocks.deleteDocumentPreviewArtifacts.mockResolvedValue(0); + mocks.deleteDocumentPreviewRows.mockResolvedValue(undefined); + mocks.deleteAudiobookPrefix.mockResolvedValue(0); + mocks.deleteTtsSegmentPrefix.mockResolvedValue(0); + }); + + test('defers document blobs/previews to the reaper on the canonical pass', async () => { + mocks.selectResults = [[]]; // no audiobooks + + await deleteUserStorageData('user-1', null); + + // Shared document storage is reclaimed by the reap-orphaned-blobs task, not here. + expect(mocks.deleteDocumentBlob).not.toHaveBeenCalled(); + expect(mocks.deleteDocumentPreviewArtifacts).not.toHaveBeenCalled(); + expect(mocks.deleteDocumentPreviewRows).not.toHaveBeenCalled(); + // Only the three non-cascading DB row deletes (tts usage, job events, verification). + expect(mocks.deleteWhere).toHaveBeenCalledTimes(3); + }); + + test('blocks database cleanup when storage cleanup fails', async () => { + mocks.selectResults = [[]]; // no audiobooks + mocks.deleteDocumentPrefix.mockRejectedValueOnce(new Error('storage unavailable')); + + await expect(deleteUserStorageData('user-1', null)).rejects.toThrow( + 'User storage cleanup failed', + ); + expect(mocks.deleteWhere).not.toHaveBeenCalled(); + }); + + test('deletes namespaced document storage inline and skips global DB rows', async () => { + mocks.selectResults = [ + [{ id: 'doc-1' }], // userDocs (namespaced pass) + [], // audiobooks + ]; + + await deleteUserStorageData('user-1', 'test-ns'); + + expect(mocks.deleteDocumentBlob).toHaveBeenCalledWith('doc-1', 'test-ns'); + expect(mocks.deleteDocumentPreviewArtifacts).toHaveBeenCalledWith('doc-1', 'test-ns'); + expect(mocks.deleteDocumentPreviewRows).toHaveBeenCalledWith('doc-1', 'test-ns'); + // Global DB rows are only removed on the canonical pass. + expect(mocks.deleteWhere).not.toHaveBeenCalled(); + }); +}); diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..805b91f --- /dev/null +++ b/vercel.json @@ -0,0 +1,8 @@ +{ + "crons": [ + { + "path": "/api/admin/tasks/tick", + "schedule": "0 0 * * *" + } + ] +}