From 9f25e05cce4d3beb0f4bd0f95b7a9b2a124a9b91 Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 12 Feb 2026 16:05:03 -0700 Subject: [PATCH] feat(previews): implement document preview generation and caching - Add document preview caching logic with in-memory and persisted storage. - Implement S3 blobstore functions for managing document previews. - Create rendering functions for PDF and EPUB cover images to JPEG format. - Introduce database schema and functions for managing document preview metadata. - Add unit tests for rendering PDF and EPUB previews. --- docs-site/docs/configure/database.md | 1 + .../docs/configure/object-blob-storage.md | 7 + ...ect_risque.sql => 0000_solid_colossus.sql} | 43 ++ drizzle/postgres/0001_user_state_minimal.sql | 21 - drizzle/postgres/meta/0000_snapshot.json | 298 +++++++++++- drizzle/postgres/meta/_journal.json | 13 +- ...ly_korvac.sql => 0000_cool_gwen_stacy.sql} | 43 ++ drizzle/sqlite/0001_user_state_minimal.sql | 21 - drizzle/sqlite/meta/0000_snapshot.json | 293 +++++++++++- drizzle/sqlite/meta/_journal.json | 13 +- next.config.ts | 4 +- package.json | 6 +- pnpm-lock.yaml | 134 ++++++ .../api/documents/blob/get/fallback/route.ts | 85 ++++ .../api/documents/blob/get/presign/route.ts | 67 +++ .../documents/blob/preview/ensure/route.ts | 105 +++++ .../documents/blob/preview/fallback/route.ts | 143 ++++++ .../documents/blob/preview/presign/route.ts | 103 +++++ src/app/api/documents/blob/route.ts | 19 +- .../api/documents/docx-to-pdf/upload/route.ts | 12 + src/app/api/documents/route.ts | 23 + src/app/epub/[id]/page.tsx | 30 +- src/app/html/[id]/page.tsx | 30 +- src/app/pdf/[id]/page.tsx | 30 +- src/components/SettingsModal.tsx | 7 +- src/components/doclist/DocumentPreview.tsx | 147 ++++-- src/db/schema.ts | 1 + src/db/schema_postgres.ts | 23 + src/db/schema_sqlite.ts | 23 + src/lib/client-documents.ts | 122 ++++- src/lib/dexie.ts | 284 +++++++++++- src/lib/document-preview-cache.ts | 125 +++++ src/lib/server/document-previews-blobstore.ts | 148 ++++++ src/lib/server/document-previews-render.ts | 258 +++++++++++ src/lib/server/document-previews.ts | 427 ++++++++++++++++++ src/lib/server/documents-blobstore.ts | 18 + tests/unit/document-previews-render.spec.ts | 29 ++ 37 files changed, 3005 insertions(+), 151 deletions(-) rename drizzle/postgres/{0000_perfect_risque.sql => 0000_solid_colossus.sql} (64%) delete mode 100644 drizzle/postgres/0001_user_state_minimal.sql rename drizzle/sqlite/{0000_lively_korvac.sql => 0000_cool_gwen_stacy.sql} (63%) delete mode 100644 drizzle/sqlite/0001_user_state_minimal.sql create mode 100644 src/app/api/documents/blob/get/fallback/route.ts create mode 100644 src/app/api/documents/blob/get/presign/route.ts create mode 100644 src/app/api/documents/blob/preview/ensure/route.ts create mode 100644 src/app/api/documents/blob/preview/fallback/route.ts create mode 100644 src/app/api/documents/blob/preview/presign/route.ts create mode 100644 src/lib/document-preview-cache.ts create mode 100644 src/lib/server/document-previews-blobstore.ts create mode 100644 src/lib/server/document-previews-render.ts create mode 100644 src/lib/server/document-previews.ts create mode 100644 tests/unit/document-previews-render.spec.ts diff --git a/docs-site/docs/configure/database.md b/docs-site/docs/configure/database.md index c3140be..88c79f5 100644 --- a/docs-site/docs/configure/database.md +++ b/docs-site/docs/configure/database.md @@ -16,6 +16,7 @@ This page covers database mode selection for OpenReader WebUI. - TTS character usage counters (`user_tts_chars`) for daily rate limiting (when enabled). - User settings preferences (`user_preferences`) when auth is enabled. - User reading progress (`user_document_progress`) when auth is enabled. +- Document preview job/asset metadata (`document_previews`) for server-side PDF/EPUB thumbnails. ## Related variables diff --git a/docs-site/docs/configure/object-blob-storage.md b/docs-site/docs/configure/object-blob-storage.md index 9664ea1..ff46fd1 100644 --- a/docs-site/docs/configure/object-blob-storage.md +++ b/docs-site/docs/configure/object-blob-storage.md @@ -28,6 +28,13 @@ Storage variables are documented in [Environment Variables](../reference/environ - Primary path: browser uploads to presigned URL from `/api/documents/blob/upload/presign`. - Fallback path: `/api/documents/blob/upload/fallback` when direct upload fails/unreachable. - Read/download path: blob/content serving route `/api/documents/blob` (not the upload fallback route). +- Preview path: `/api/documents/blob/preview` (returns `202` while a preview is generating; serves/redirects when ready). + +## Document previews + +- PDF/EPUB previews are generated server-side and stored in object storage under `document_previews_v1`. +- Preview generation is triggered on upload registration and also backfills on first preview request for older docs. +- Preview artifacts are temporary-cache friendly and can be regenerated from the source document blob. ## FS / Volume Mounts diff --git a/drizzle/postgres/0000_perfect_risque.sql b/drizzle/postgres/0000_solid_colossus.sql similarity index 64% rename from drizzle/postgres/0000_perfect_risque.sql rename to drizzle/postgres/0000_solid_colossus.sql index 25c0a97..aff2619 100644 --- a/drizzle/postgres/0000_perfect_risque.sql +++ b/drizzle/postgres/0000_solid_colossus.sql @@ -38,6 +38,27 @@ CREATE TABLE "audiobooks" ( CONSTRAINT "audiobooks_id_user_id_pk" PRIMARY KEY("id","user_id") ); --> statement-breakpoint +CREATE TABLE "document_previews" ( + "document_id" text NOT NULL, + "namespace" text DEFAULT '' NOT NULL, + "variant" text DEFAULT 'card-240-jpeg' NOT NULL, + "status" text DEFAULT 'queued' NOT NULL, + "source_last_modified_ms" bigint NOT NULL, + "object_key" text NOT NULL, + "content_type" text DEFAULT 'image/jpeg' NOT NULL, + "width" integer DEFAULT 240 NOT NULL, + "height" integer, + "byte_size" bigint, + "etag" text, + "lease_owner" text, + "lease_until_ms" bigint DEFAULT 0 NOT NULL, + "attempt_count" integer DEFAULT 0 NOT NULL, + "last_error" text, + "created_at_ms" bigint DEFAULT 0 NOT NULL, + "updated_at_ms" bigint DEFAULT 0 NOT NULL, + CONSTRAINT "document_previews_document_id_namespace_variant_pk" PRIMARY KEY("document_id","namespace","variant") +); +--> statement-breakpoint CREATE TABLE "documents" ( "id" text NOT NULL, "user_id" text NOT NULL, @@ -74,6 +95,26 @@ CREATE TABLE "user" ( CONSTRAINT "user_email_unique" UNIQUE("email") ); --> statement-breakpoint +CREATE TABLE "user_document_progress" ( + "user_id" text NOT NULL, + "document_id" text NOT NULL, + "reader_type" text NOT NULL, + "location" text NOT NULL, + "progress" real, + "client_updated_at_ms" bigint DEFAULT 0 NOT NULL, + "created_at" timestamp DEFAULT now(), + "updated_at" timestamp DEFAULT now(), + CONSTRAINT "user_document_progress_user_id_document_id_pk" PRIMARY KEY("user_id","document_id") +); +--> statement-breakpoint +CREATE TABLE "user_preferences" ( + "user_id" text PRIMARY KEY NOT NULL, + "data_json" jsonb DEFAULT '{}'::jsonb NOT NULL, + "client_updated_at_ms" bigint DEFAULT 0 NOT NULL, + "created_at" timestamp DEFAULT now(), + "updated_at" timestamp DEFAULT now() +); +--> statement-breakpoint CREATE TABLE "user_tts_chars" ( "user_id" text NOT NULL, "date" date NOT NULL, @@ -92,6 +133,8 @@ CREATE TABLE "verification" ( --> statement-breakpoint ALTER TABLE "account" ADD CONSTRAINT "account_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint ALTER TABLE "session" ADD CONSTRAINT "session_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "idx_document_previews_status_lease" ON "document_previews" USING btree ("status","lease_until_ms");--> statement-breakpoint CREATE INDEX "idx_documents_user_id" ON "documents" USING btree ("user_id");--> statement-breakpoint CREATE INDEX "idx_documents_user_id_last_modified" ON "documents" USING btree ("user_id","last_modified");--> statement-breakpoint +CREATE INDEX "idx_user_document_progress_user_id_updated_at" ON "user_document_progress" USING btree ("user_id","updated_at");--> statement-breakpoint CREATE INDEX "idx_user_tts_chars_date" ON "user_tts_chars" USING btree ("date"); \ No newline at end of file diff --git a/drizzle/postgres/0001_user_state_minimal.sql b/drizzle/postgres/0001_user_state_minimal.sql deleted file mode 100644 index 0f8ef23..0000000 --- a/drizzle/postgres/0001_user_state_minimal.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE "user_document_progress" ( - "user_id" text NOT NULL, - "document_id" text NOT NULL, - "reader_type" text NOT NULL, - "location" text NOT NULL, - "progress" real, - "client_updated_at_ms" bigint DEFAULT 0 NOT NULL, - "created_at" timestamp DEFAULT now(), - "updated_at" timestamp DEFAULT now(), - CONSTRAINT "user_document_progress_user_id_document_id_pk" PRIMARY KEY("user_id","document_id") -); ---> statement-breakpoint -CREATE TABLE "user_preferences" ( - "user_id" text PRIMARY KEY NOT NULL, - "data_json" jsonb DEFAULT '{}'::jsonb NOT NULL, - "client_updated_at_ms" bigint DEFAULT 0 NOT NULL, - "created_at" timestamp DEFAULT now(), - "updated_at" timestamp DEFAULT now() -); ---> statement-breakpoint -CREATE INDEX "idx_user_document_progress_user_id_updated_at" ON "user_document_progress" USING btree ("user_id","updated_at"); diff --git a/drizzle/postgres/meta/0000_snapshot.json b/drizzle/postgres/meta/0000_snapshot.json index 1493370..7ceeb38 100644 --- a/drizzle/postgres/meta/0000_snapshot.json +++ b/drizzle/postgres/meta/0000_snapshot.json @@ -1,5 +1,5 @@ { - "id": "03395f16-5c34-409a-9dc1-69329d67bcd9", + "id": "161f4dc9-0c06-4d77-87e5-3496ffad1b97", "prevId": "00000000-0000-0000-0000-000000000000", "version": "7", "dialect": "postgresql", @@ -250,6 +250,161 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.document_previews": { + "name": "document_previews", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'card-240-jpeg'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 240 + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_until_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "name": "document_previews_document_id_namespace_variant_pk", + "columns": [ + "document_id", + "namespace", + "variant" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.documents": { "name": "documents", "schema": "", @@ -509,6 +664,147 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.user_document_progress": { + "name": "user_document_progress", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "name": "user_document_progress_user_id_document_id_pk", + "columns": [ + "user_id", + "document_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preferences": { + "name": "user_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.user_tts_chars": { "name": "user_tts_chars", "schema": "", diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index 9fea0ea..bd88253 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -5,16 +5,9 @@ { "idx": 0, "version": "7", - "when": 1770191341206, - "tag": "0000_perfect_risque", - "breakpoints": true - }, - { - "idx": 1, - "version": "7", - "when": 1770843600000, - "tag": "0001_user_state_minimal", + "when": 1770846273728, + "tag": "0000_solid_colossus", "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/drizzle/sqlite/0000_lively_korvac.sql b/drizzle/sqlite/0000_cool_gwen_stacy.sql similarity index 63% rename from drizzle/sqlite/0000_lively_korvac.sql rename to drizzle/sqlite/0000_cool_gwen_stacy.sql index ac57079..0c37a89 100644 --- a/drizzle/sqlite/0000_lively_korvac.sql +++ b/drizzle/sqlite/0000_cool_gwen_stacy.sql @@ -39,6 +39,28 @@ CREATE TABLE `audiobooks` ( PRIMARY KEY(`id`, `user_id`) ); --> statement-breakpoint +CREATE TABLE `document_previews` ( + `document_id` text NOT NULL, + `namespace` text DEFAULT '' NOT NULL, + `variant` text DEFAULT 'card-240-jpeg' NOT NULL, + `status` text DEFAULT 'queued' NOT NULL, + `source_last_modified_ms` integer NOT NULL, + `object_key` text NOT NULL, + `content_type` text DEFAULT 'image/jpeg' NOT NULL, + `width` integer DEFAULT 240 NOT NULL, + `height` integer, + `byte_size` integer, + `etag` text, + `lease_owner` text, + `lease_until_ms` integer DEFAULT 0 NOT NULL, + `attempt_count` integer DEFAULT 0 NOT NULL, + `last_error` text, + `created_at_ms` integer DEFAULT 0 NOT NULL, + `updated_at_ms` integer DEFAULT 0 NOT NULL, + PRIMARY KEY(`document_id`, `namespace`, `variant`) +); +--> statement-breakpoint +CREATE INDEX `idx_document_previews_status_lease` ON `document_previews` (`status`,`lease_until_ms`);--> statement-breakpoint CREATE TABLE `documents` ( `id` text NOT NULL, `user_id` text NOT NULL, @@ -78,6 +100,27 @@ CREATE TABLE `user` ( ); --> statement-breakpoint CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint +CREATE TABLE `user_document_progress` ( + `user_id` text NOT NULL, + `document_id` text NOT NULL, + `reader_type` text NOT NULL, + `location` text NOT NULL, + `progress` real, + `client_updated_at_ms` integer DEFAULT 0 NOT NULL, + `created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000), + `updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000), + PRIMARY KEY(`user_id`, `document_id`) +); +--> statement-breakpoint +CREATE INDEX `idx_user_document_progress_user_id_updated_at` ON `user_document_progress` (`user_id`,`updated_at`);--> statement-breakpoint +CREATE TABLE `user_preferences` ( + `user_id` text PRIMARY KEY NOT NULL, + `data_json` text DEFAULT '{}' NOT NULL, + `client_updated_at_ms` integer DEFAULT 0 NOT NULL, + `created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000), + `updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000) +); +--> statement-breakpoint CREATE TABLE `user_tts_chars` ( `user_id` text NOT NULL, `date` text NOT NULL, diff --git a/drizzle/sqlite/0001_user_state_minimal.sql b/drizzle/sqlite/0001_user_state_minimal.sql deleted file mode 100644 index 24514c4..0000000 --- a/drizzle/sqlite/0001_user_state_minimal.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE `user_document_progress` ( - `user_id` text NOT NULL, - `document_id` text NOT NULL, - `reader_type` text NOT NULL, - `location` text NOT NULL, - `progress` real, - `client_updated_at_ms` integer DEFAULT 0 NOT NULL, - `created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000), - `updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000), - PRIMARY KEY(`user_id`, `document_id`) -); ---> statement-breakpoint -CREATE TABLE `user_preferences` ( - `user_id` text PRIMARY KEY NOT NULL, - `data_json` text DEFAULT '{}' NOT NULL, - `client_updated_at_ms` integer DEFAULT 0 NOT NULL, - `created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000), - `updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000) -); ---> statement-breakpoint -CREATE INDEX `idx_user_document_progress_user_id_updated_at` ON `user_document_progress` (`user_id`,`updated_at`); diff --git a/drizzle/sqlite/meta/0000_snapshot.json b/drizzle/sqlite/meta/0000_snapshot.json index 0a7d123..6ca5cca 100644 --- a/drizzle/sqlite/meta/0000_snapshot.json +++ b/drizzle/sqlite/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "9c9e61fc-1f90-463a-99dc-232397f316e7", + "id": "eb884adf-c9c7-494b-b0a1-3601f63bcbbe", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "account": { @@ -270,6 +270,162 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "document_previews": { + "name": "document_previews", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'card-240-jpeg'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 240 + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + "status", + "lease_until_ms" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "columns": [ + "document_id", + "namespace", + "variant" + ], + "name": "document_previews_document_id_namespace_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "documents": { "name": "documents", "columns": { @@ -523,6 +679,141 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "user_document_progress": { + "name": "user_document_progress", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(strftime('%s','now') as int) * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(strftime('%s','now') as int) * 1000)" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + "user_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "columns": [ + "user_id", + "document_id" + ], + "name": "user_document_progress_user_id_document_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_preferences": { + "name": "user_preferences", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(strftime('%s','now') as int) * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(strftime('%s','now') as int) * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "user_tts_chars": { "name": "user_tts_chars", "columns": { diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index cc48afa..4edffc4 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -5,16 +5,9 @@ { "idx": 0, "version": "6", - "when": 1770191340954, - "tag": "0000_lively_korvac", - "breakpoints": true - }, - { - "idx": 1, - "version": "7", - "when": 1770843600000, - "tag": "0001_user_state_minimal", + "when": 1770846273460, + "tag": "0000_cool_gwen_stacy", "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/next.config.ts b/next.config.ts index 42eb5d2..f7c6ffb 100644 --- a/next.config.ts +++ b/next.config.ts @@ -3,10 +3,10 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { turbopack: { resolveAlias: { - canvas: './empty-module.ts', + canvas: '@napi-rs/canvas', }, }, - serverExternalPackages: ["better-sqlite3", "ffmpeg-static", "ffprobe-static"], + serverExternalPackages: ["@napi-rs/canvas", "better-sqlite3", "ffmpeg-static", "ffprobe-static"], outputFileTracingIncludes: { '/api/audiobook': [ './node_modules/ffmpeg-static/ffmpeg', diff --git a/package.json b/package.json index 52e1678..4d17007 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@aws-sdk/client-s3": "^3.987.0", "@aws-sdk/s3-request-presigner": "^3.987.0", "@headlessui/react": "^2.2.9", + "@napi-rs/canvas": "^0.1.91", "@vercel/analytics": "^1.6.1", "better-auth": "^1.4.18", "better-sqlite3": "^12.6.2", @@ -37,9 +38,11 @@ "drizzle-kit": "^0.31.9", "drizzle-orm": "^0.45.1", "epubjs": "^0.3.93", + "fast-xml-parser": "^5.3.5", "ffmpeg-static": "^5.3.0", "ffprobe-static": "^3.1.0", "howler": "^2.2.4", + "jszip": "^3.10.1", "lru-cache": "^11.2.6", "next": "^15.5.12", "openai": "^6.21.0", @@ -63,11 +66,11 @@ "@tailwindcss/typography": "^0.5.19", "@types/better-sqlite3": "^7.6.13", "@types/ffprobe-static": "^2.0.3", + "@types/howler": "^2.2.12", "@types/node": "^20.19.33", "@types/pg": "^8.16.0", "@types/react": "^19.2.13", "@types/react-dom": "^19.2.3", - "@types/howler": "^2.2.12", "@types/uuid": "^10.0.0", "eslint": "^9.39.2", "eslint-config-next": "^15.5.12", @@ -77,6 +80,7 @@ }, "pnpm": { "onlyBuiltDependencies": [ + "@napi-rs/canvas", "better-sqlite3", "ffmpeg-static" ], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 972419d..09bc895 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,9 @@ importers: '@headlessui/react': specifier: ^2.2.9 version: 2.2.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@napi-rs/canvas': + specifier: ^0.1.91 + version: 0.1.91 '@vercel/analytics': specifier: ^1.6.1 version: 1.6.1(next@15.5.12(@playwright/test@1.58.2)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4) @@ -58,6 +61,9 @@ importers: epubjs: specifier: ^0.3.93 version: 0.3.93 + fast-xml-parser: + specifier: ^5.3.5 + version: 5.3.5 ffmpeg-static: specifier: ^5.3.0 version: 5.3.0 @@ -67,6 +73,9 @@ importers: howler: specifier: ^2.2.4 version: 2.2.4 + jszip: + specifier: ^3.10.1 + version: 3.10.1 lru-cache: specifier: ^11.2.6 version: 11.2.6 @@ -915,6 +924,76 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@napi-rs/canvas-android-arm64@0.1.91': + resolution: {integrity: sha512-SLLzXXgSnfct4zy/BVAfweZQkYkPJsNsJ2e5DOE8DFEHC6PufyUrwb12yqeu2So2IOIDpWJJaDAxKY/xpy6MYQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/canvas-darwin-arm64@0.1.91': + resolution: {integrity: sha512-bzdbCjIjw3iRuVFL+uxdSoMra/l09ydGNX9gsBxO/zg+5nlppscIpj6gg+nL6VNG85zwUarDleIrUJ+FWHvmuA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/canvas-darwin-x64@0.1.91': + resolution: {integrity: sha512-q3qpkpw0IsG9fAS/dmcGIhCVoNxj8ojbexZKWwz3HwxlEWsLncEQRl4arnxrwbpLc2nTNTyj4WwDn7QR5NDAaA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.91': + resolution: {integrity: sha512-Io3g8wJZVhK8G+Fpg1363BE90pIPqg+ZbeehYNxPWDSzbgwU3xV0l8r/JBzODwC7XHi1RpFEk+xyUTMa2POj6w==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/canvas-linux-arm64-gnu@0.1.91': + resolution: {integrity: sha512-HBnto+0rxx1bQSl8bCWA9PyBKtlk2z/AI32r3cu4kcNO+M/5SD4b0v1MWBWZyqMQyxFjWgy3ECyDjDKMC6tY1A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-arm64-musl@0.1.91': + resolution: {integrity: sha512-/eJtVe2Xw9A86I4kwXpxxoNagdGclu12/NSMsfoL8q05QmeRCbfjhg1PJS7ENAuAvaiUiALGrbVfeY1KU1gztQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-riscv64-gnu@0.1.91': + resolution: {integrity: sha512-floNK9wQuRWevUhhXRcuis7h0zirdytVxPgkonWO+kQlbvxV7gEUHGUFQyq4n55UHYFwgck1SAfJ1HuXv/+ppQ==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/canvas-linux-x64-gnu@0.1.91': + resolution: {integrity: sha512-c3YDqBdf7KETuZy2AxsHFMsBBX1dWT43yFfWUq+j1IELdgesWtxf/6N7csi3VPf6VA3PmnT9EhMyb+M1wfGtqw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-linux-x64-musl@0.1.91': + resolution: {integrity: sha512-RpZ3RPIwgEcNBHSHSX98adm+4VP8SMT5FN6250s5jQbWpX/XNUX5aLMfAVJS/YnDjS1QlsCgQxFOPU0aCCWgag==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-win32-arm64-msvc@0.1.91': + resolution: {integrity: sha512-gF8MBp4X134AgVurxqlCdDA2qO0WaDdi9o6Sd5rWRVXRhWhYQ6wkdEzXNLIrmmros0Tsp2J0hQzx4ej/9O8trQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/canvas-win32-x64-msvc@0.1.91': + resolution: {integrity: sha512-++gtW9EV/neKI8TshD8WFxzBYALSPag2kFRahIJV+LYsyt5kBn21b1dBhEUDHf7O+wiZmuFCeUa7QKGHnYRZBA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas@0.1.91': + resolution: {integrity: sha512-eeIe1GoB74P1B0Nkw6pV8BCQ3hfCfvyYr4BntzlCsnFXzVJiPMDnLeIx3gVB0xQMblHYnjK/0nCLvirEhOjr5g==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} @@ -2307,6 +2386,10 @@ packages: resolution: {integrity: sha512-EFd6afGmXlCx8H8WTZHhAoDaWaGyuIBoZJ2mknrNxug+aZKjkp0a0dlars9Izl+jF+7Gu1/5f/2h68cQpe0IiA==} hasBin: true + fast-xml-parser@5.3.5: + resolution: {integrity: sha512-JeaA2Vm9ffQKp9VjvfzObuMCjUYAp5WDYhRYL5LrBPY/jUDlUtOvDfot0vKSkB9tuX885BDHjtw4fZadD95wnA==} + hasBin: true + fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -4675,6 +4758,53 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@napi-rs/canvas-android-arm64@0.1.91': + optional: true + + '@napi-rs/canvas-darwin-arm64@0.1.91': + optional: true + + '@napi-rs/canvas-darwin-x64@0.1.91': + optional: true + + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.91': + optional: true + + '@napi-rs/canvas-linux-arm64-gnu@0.1.91': + optional: true + + '@napi-rs/canvas-linux-arm64-musl@0.1.91': + optional: true + + '@napi-rs/canvas-linux-riscv64-gnu@0.1.91': + optional: true + + '@napi-rs/canvas-linux-x64-gnu@0.1.91': + optional: true + + '@napi-rs/canvas-linux-x64-musl@0.1.91': + optional: true + + '@napi-rs/canvas-win32-arm64-msvc@0.1.91': + optional: true + + '@napi-rs/canvas-win32-x64-msvc@0.1.91': + optional: true + + '@napi-rs/canvas@0.1.91': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 0.1.91 + '@napi-rs/canvas-darwin-arm64': 0.1.91 + '@napi-rs/canvas-darwin-x64': 0.1.91 + '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.91 + '@napi-rs/canvas-linux-arm64-gnu': 0.1.91 + '@napi-rs/canvas-linux-arm64-musl': 0.1.91 + '@napi-rs/canvas-linux-riscv64-gnu': 0.1.91 + '@napi-rs/canvas-linux-x64-gnu': 0.1.91 + '@napi-rs/canvas-linux-x64-musl': 0.1.91 + '@napi-rs/canvas-win32-arm64-msvc': 0.1.91 + '@napi-rs/canvas-win32-x64-msvc': 0.1.91 + '@napi-rs/wasm-runtime@0.2.12': dependencies: '@emnapi/core': 1.8.1 @@ -6246,6 +6376,10 @@ snapshots: dependencies: strnum: 2.1.2 + fast-xml-parser@5.3.5: + dependencies: + strnum: 2.1.2 + fastq@1.20.1: dependencies: reusify: 1.1.0 diff --git a/src/app/api/documents/blob/get/fallback/route.ts b/src/app/api/documents/blob/get/fallback/route.ts new file mode 100644 index 0000000..043bfcd --- /dev/null +++ b/src/app/api/documents/blob/get/fallback/route.ts @@ -0,0 +1,85 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth'; +import { contentTypeForName } from '@/lib/server/library'; +import { getDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function streamBuffer(buffer: Buffer): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(buffer)); + controller.close(); + }, + }); +} + +export async function GET(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const url = new URL(req.url); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); + } + + const rows = (await db + .select({ id: documents.id, userId: documents.userId, name: documents.name }) + .from(documents) + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ + id: string; + userId: string; + name: string; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + const filename = doc.name || `${id}.bin`; + const responseType = contentTypeForName(filename); + + try { + const content = await getDocumentBlob(id, testNamespace); + return new NextResponse(streamBuffer(content), { + headers: { + 'Content-Type': responseType, + 'Cache-Control': 'no-store', + }, + }); + } catch (error) { + if (isMissingBlobError(error)) { + await db + .delete(documents) + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + throw error; + } + } catch (error) { + console.error('Error loading document content fallback:', error); + return NextResponse.json({ error: 'Failed to load document content' }, { status: 500 }); + } +} diff --git a/src/app/api/documents/blob/get/presign/route.ts b/src/app/api/documents/blob/get/presign/route.ts new file mode 100644 index 0000000..4785b4c --- /dev/null +++ b/src/app/api/documents/blob/get/presign/route.ts @@ -0,0 +1,67 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth'; +import { isValidDocumentId, presignGet } from '@/lib/server/documents-blobstore'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +export async function GET(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const url = new URL(req.url); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); + } + + const rows = (await db + .select({ id: documents.id, userId: documents.userId }) + .from(documents) + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ + id: string; + userId: string; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + const fallbackUrl = `/api/documents/blob/get/fallback?id=${encodeURIComponent(doc.id)}`; + const directUrl = await presignGet(doc.id, testNamespace).catch(() => null); + if (!directUrl) { + return NextResponse.redirect(fallbackUrl, { + status: 307, + headers: { 'Cache-Control': 'no-store' }, + }); + } + + return NextResponse.redirect(directUrl, { + status: 307, + headers: { 'Cache-Control': 'no-store' }, + }); + } catch (error) { + console.error('Error creating document download signature:', error); + return NextResponse.json({ error: 'Failed to prepare document download' }, { status: 500 }); + } +} diff --git a/src/app/api/documents/blob/preview/ensure/route.ts b/src/app/api/documents/blob/preview/ensure/route.ts new file mode 100644 index 0000000..8d417ad --- /dev/null +++ b/src/app/api/documents/blob/preview/ensure/route.ts @@ -0,0 +1,105 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth'; +import { isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { presignDocumentPreviewGet } from '@/lib/server/document-previews-blobstore'; +import { ensureDocumentPreview, isPreviewableDocumentType } from '@/lib/server/document-previews'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +export async function GET(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const url = new URL(req.url); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); + } + + const rows = (await db + .select({ + id: documents.id, + userId: documents.userId, + type: documents.type, + lastModified: documents.lastModified, + }) + .from(documents) + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ + id: string; + userId: string; + type: string; + lastModified: number; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + if (!isPreviewableDocumentType(doc.type)) { + return NextResponse.json({ error: `Preview not supported for type ${doc.type}` }, { status: 415 }); + } + + const presignUrl = `/api/documents/blob/preview/presign?id=${encodeURIComponent(id)}`; + const fallbackUrl = `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`; + const preview = await ensureDocumentPreview( + { + id: doc.id, + type: doc.type, + lastModified: Number(doc.lastModified), + }, + testNamespace, + ); + + if (preview.state !== 'ready') { + return NextResponse.json( + { + status: preview.status, + retryAfterMs: preview.retryAfterMs, + presignUrl, + fallbackUrl, + }, + { + status: 202, + headers: { 'Cache-Control': 'no-store' }, + }, + ); + } + + const directUrl = await presignDocumentPreviewGet(doc.id, testNamespace).catch(() => null); + return NextResponse.json( + { + status: 'ready', + presignUrl, + fallbackUrl, + ...(directUrl ? { directUrl } : {}), + }, + { + headers: { 'Cache-Control': 'no-store' }, + }, + ); + } catch (error) { + console.error('Error ensuring document preview:', error); + return NextResponse.json({ error: 'Failed to ensure document preview' }, { status: 500 }); + } +} diff --git a/src/app/api/documents/blob/preview/fallback/route.ts b/src/app/api/documents/blob/preview/fallback/route.ts new file mode 100644 index 0000000..36cadff --- /dev/null +++ b/src/app/api/documents/blob/preview/fallback/route.ts @@ -0,0 +1,143 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth'; +import { isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { + getDocumentPreviewBuffer, + isMissingBlobError, +} from '@/lib/server/document-previews-blobstore'; +import { + ensureDocumentPreview, + enqueueDocumentPreview, + isPreviewableDocumentType, +} from '@/lib/server/document-previews'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function streamBuffer(buffer: Buffer): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(buffer)); + controller.close(); + }, + }); +} + +export async function GET(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const url = new URL(req.url); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); + const presignUrl = `/api/documents/blob/preview/presign?id=${encodeURIComponent(id)}`; + const fallbackUrl = `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`; + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); + } + + const rows = (await db + .select({ + id: documents.id, + userId: documents.userId, + type: documents.type, + lastModified: documents.lastModified, + }) + .from(documents) + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ + id: string; + userId: string; + type: string; + lastModified: number; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + if (!isPreviewableDocumentType(doc.type)) { + return NextResponse.json({ error: `Preview not supported for type ${doc.type}` }, { status: 415 }); + } + + const preview = await ensureDocumentPreview( + { + id: doc.id, + type: doc.type, + lastModified: Number(doc.lastModified), + }, + testNamespace, + ); + + if (preview.state !== 'ready') { + return NextResponse.json( + { + status: preview.status, + retryAfterMs: preview.retryAfterMs, + presignUrl, + fallbackUrl, + }, + { + status: 202, + headers: { 'Cache-Control': 'no-store' }, + }, + ); + } + + try { + const content = await getDocumentPreviewBuffer(doc.id, testNamespace); + return new NextResponse(streamBuffer(content), { + headers: { + 'Content-Type': preview.contentType, + 'Cache-Control': 'no-store', + 'Content-Length': String(content.byteLength), + }, + }); + } catch (error) { + if (isMissingBlobError(error)) { + await enqueueDocumentPreview( + { + id: doc.id, + type: doc.type, + lastModified: Number(doc.lastModified), + }, + testNamespace, + ).catch(() => {}); + return NextResponse.json( + { + status: 'queued', + retryAfterMs: 1500, + presignUrl, + fallbackUrl, + }, + { + status: 202, + headers: { 'Cache-Control': 'no-store' }, + }, + ); + } + throw error; + } + } catch (error) { + console.error('Error loading document preview fallback:', error); + return NextResponse.json({ error: 'Failed to load document preview' }, { status: 500 }); + } +} diff --git a/src/app/api/documents/blob/preview/presign/route.ts b/src/app/api/documents/blob/preview/presign/route.ts new file mode 100644 index 0000000..d52e0f5 --- /dev/null +++ b/src/app/api/documents/blob/preview/presign/route.ts @@ -0,0 +1,103 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth'; +import { isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { presignDocumentPreviewGet } from '@/lib/server/document-previews-blobstore'; +import { ensureDocumentPreview, isPreviewableDocumentType } from '@/lib/server/document-previews'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; + +export const dynamic = 'force-dynamic'; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +export async function GET(req: NextRequest) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = ctxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const url = new URL(req.url); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); + } + + const rows = (await db + .select({ + id: documents.id, + userId: documents.userId, + type: documents.type, + lastModified: documents.lastModified, + }) + .from(documents) + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ + id: string; + userId: string; + type: string; + lastModified: number; + }>; + + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + if (!isPreviewableDocumentType(doc.type)) { + return NextResponse.json({ error: `Preview not supported for type ${doc.type}` }, { status: 415 }); + } + + const fallbackUrl = `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`; + const preview = await ensureDocumentPreview( + { + id: doc.id, + type: doc.type, + lastModified: Number(doc.lastModified), + }, + testNamespace, + ); + + if (preview.state !== 'ready') { + return NextResponse.json( + { + status: preview.status, + retryAfterMs: preview.retryAfterMs, + fallbackUrl, + }, + { + status: 202, + headers: { 'Cache-Control': 'no-store' }, + }, + ); + } + + const directUrl = await presignDocumentPreviewGet(doc.id, testNamespace).catch(() => null); + if (!directUrl) { + return NextResponse.redirect(fallbackUrl, { + status: 307, + headers: { 'Cache-Control': 'no-store' }, + }); + } + + return NextResponse.redirect(directUrl, { + status: 307, + headers: { 'Cache-Control': 'no-store' }, + }); + } catch (error) { + console.error('Error creating document preview signature:', error); + return NextResponse.json({ error: 'Failed to prepare document preview' }, { status: 500 }); + } +} diff --git a/src/app/api/documents/blob/route.ts b/src/app/api/documents/blob/route.ts index 04c183d..11366dc 100644 --- a/src/app/api/documents/blob/route.ts +++ b/src/app/api/documents/blob/route.ts @@ -5,7 +5,13 @@ import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth'; import { contentTypeForName } from '@/lib/server/library'; import { extractRawTextSnippet } from '@/lib/text-snippets'; -import { getDocumentBlob, getDocumentRange, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { + getDocumentBlob, + getDocumentRange, + isMissingBlobError, + isValidDocumentId, + presignGet, +} from '@/lib/server/documents-blobstore'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; import { isS3Configured } from '@/lib/server/s3'; @@ -38,6 +44,7 @@ export async function GET(req: NextRequest) { const url = new URL(req.url); const id = (url.searchParams.get('id') || '').trim().toLowerCase(); const format = (url.searchParams.get('format') || '').toLowerCase().trim(); + const prefersProxy = (url.searchParams.get('proxy') || '').trim() === '1'; if (!isValidDocumentId(id)) { return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); } @@ -83,6 +90,16 @@ export async function GET(req: NextRequest) { } try { + if (!prefersProxy) { + const directUrl = await presignGet(id, testNamespace).catch(() => null); + if (directUrl) { + return NextResponse.redirect(directUrl, { + status: 307, + headers: { 'Cache-Control': 'no-store' }, + }); + } + } + const content = await getDocumentBlob(id, testNamespace); const stream = new ReadableStream({ start(controller) { diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 0b4a3bd..4e79309 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -9,6 +9,7 @@ import { requireAuthContext } from '@/lib/server/auth'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { safeDocumentName } from '@/lib/server/documents-utils'; +import { enqueueDocumentPreview } from '@/lib/server/document-previews'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; import { isS3Configured } from '@/lib/server/s3'; import { putDocumentBlob } from '@/lib/server/documents-blobstore'; @@ -147,6 +148,17 @@ export async function POST(req: NextRequest) { }, }); + await enqueueDocumentPreview( + { + id, + type: 'pdf', + lastModified, + }, + testNamespace, + ).catch((error) => { + console.error(`Failed to enqueue preview for converted DOCX ${id}:`, error); + }); + return NextResponse.json({ stored: { id, diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 023afdf..5492b18 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -4,6 +4,11 @@ import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth'; import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents-utils'; +import { + cleanupDocumentPreviewArtifacts, + deleteDocumentPreviewRows, + enqueueDocumentPreview, +} from '@/lib/server/document-previews'; import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; import { isS3Configured } from '@/lib/server/s3'; @@ -124,6 +129,17 @@ export async function POST(req: NextRequest) { lastModified: doc.lastModified, scope: storageUserId === unclaimedUserId ? 'unclaimed' : 'user', }); + + await enqueueDocumentPreview( + { + id: doc.id, + type: doc.type, + lastModified: doc.lastModified, + }, + testNamespace, + ).catch((error) => { + console.error(`Failed to enqueue preview for document ${doc.id}:`, error); + }); } return NextResponse.json({ success: true, stored }); @@ -269,6 +285,13 @@ export async function DELETE(req: NextRequest) { throw error; } } + + await cleanupDocumentPreviewArtifacts(id, testNamespace).catch((error) => { + console.error(`Failed to cleanup preview artifacts for document ${id}:`, error); + }); + await deleteDocumentPreviewRows(id, testNamespace).catch((error) => { + console.error(`Failed to cleanup preview rows for document ${id}:`, error); + }); } return NextResponse.json({ success: true, deleted: deletedRows.length }); diff --git a/src/app/epub/[id]/page.tsx b/src/app/epub/[id]/page.tsx index 51d5d0e..b2b4c97 100644 --- a/src/app/epub/[id]/page.tsx +++ b/src/app/epub/[id]/page.tsx @@ -2,7 +2,7 @@ import { useParams, useRouter } from "next/navigation"; import Link from 'next/link'; -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { useEPUB } from '@/contexts/EPUBContext'; import { DocumentSkeleton } from '@/components/DocumentSkeleton'; import { EPUBViewer } from '@/components/views/EPUBViewer'; @@ -34,11 +34,20 @@ export default function EPUBPage() { const [containerHeight, setContainerHeight] = useState('auto'); const [padPct, setPadPct] = useState(100); // 0..100 (100 = full width, 0 = max padding) const [maxPadPx, setMaxPadPx] = useState(0); + const inFlightDocIdRef = useRef(null); + const loadedDocIdRef = useRef(null); + + useEffect(() => { + setIsLoading(true); + setError(null); + inFlightDocIdRef.current = null; + loadedDocIdRef.current = null; + }, [id]); const loadDocument = useCallback(async () => { console.log('Loading new epub (from page.tsx)'); - stop(); // Reset TTS when loading new document let didRedirect = false; + let startedLoad = false; try { if (!id) { setError('Document not found'); @@ -50,12 +59,27 @@ export default function EPUBPage() { router.replace(`/epub/${resolved}`); return; } + + if (loadedDocIdRef.current === resolved) { + return; + } + if (inFlightDocIdRef.current === resolved) { + return; + } + + startedLoad = true; + inFlightDocIdRef.current = resolved; + stop(); // Reset TTS when loading new document await setCurrentDocument(resolved); + loadedDocIdRef.current = resolved; } catch (err) { console.error('Error loading document:', err); setError('Failed to load document'); } finally { - if (!didRedirect) { + if (startedLoad) { + inFlightDocIdRef.current = null; + } + if (!didRedirect && startedLoad) { setIsLoading(false); } } diff --git a/src/app/html/[id]/page.tsx b/src/app/html/[id]/page.tsx index a3d0811..c9cfba0 100644 --- a/src/app/html/[id]/page.tsx +++ b/src/app/html/[id]/page.tsx @@ -2,7 +2,7 @@ import { useParams, useRouter } from "next/navigation"; import Link from 'next/link'; -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { useHTML } from '@/contexts/HTMLContext'; import { DocumentSkeleton } from '@/components/DocumentSkeleton'; import { HTMLViewer } from '@/components/views/HTMLViewer'; @@ -28,12 +28,21 @@ export default function HTMLPage() { const [containerHeight, setContainerHeight] = useState('auto'); const [padPct, setPadPct] = useState(100); // 0..100 (100 = full width) const [maxPadPx, setMaxPadPx] = useState(0); + const inFlightDocIdRef = useRef(null); + const loadedDocIdRef = useRef(null); + + useEffect(() => { + setIsLoading(true); + setError(null); + inFlightDocIdRef.current = null; + loadedDocIdRef.current = null; + }, [id]); const loadDocument = useCallback(async () => { if (!isLoading) return; console.log('Loading new HTML document (from page.tsx)'); - stop(); let didRedirect = false; + let startedLoad = false; try { if (!id) { setError('Document not found'); @@ -45,12 +54,27 @@ export default function HTMLPage() { router.replace(`/html/${resolved}`); return; } + + if (loadedDocIdRef.current === resolved) { + return; + } + if (inFlightDocIdRef.current === resolved) { + return; + } + + startedLoad = true; + inFlightDocIdRef.current = resolved; + stop(); await setCurrentDocument(resolved); + loadedDocIdRef.current = resolved; } catch (err) { console.error('Error loading document:', err); setError('Failed to load document'); } finally { - if (!didRedirect) { + if (startedLoad) { + inFlightDocIdRef.current = null; + } + if (!didRedirect && startedLoad) { setIsLoading(false); } } diff --git a/src/app/pdf/[id]/page.tsx b/src/app/pdf/[id]/page.tsx index 1c534a1..ff25cb8 100644 --- a/src/app/pdf/[id]/page.tsx +++ b/src/app/pdf/[id]/page.tsx @@ -4,7 +4,7 @@ import dynamic from 'next/dynamic'; import { usePDF } from '@/contexts/PDFContext'; import { useParams, useRouter } from 'next/navigation'; import Link from 'next/link'; -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { DocumentSkeleton } from '@/components/DocumentSkeleton'; import { useTTS } from '@/contexts/TTSContext'; import { DocumentSettings } from '@/components/DocumentSettings'; @@ -42,12 +42,21 @@ export default function PDFViewerPage() { const [isSettingsOpen, setIsSettingsOpen] = useState(false); const [isAudiobookModalOpen, setIsAudiobookModalOpen] = useState(false); const [containerHeight, setContainerHeight] = useState('auto'); + const inFlightDocIdRef = useRef(null); + const loadedDocIdRef = useRef(null); + + useEffect(() => { + setIsLoading(true); + setError(null); + inFlightDocIdRef.current = null; + loadedDocIdRef.current = null; + }, [id]); const loadDocument = useCallback(async () => { if (!isLoading) return; // Prevent calls when not loading new doc console.log('Loading new document (from page.tsx)'); - stop(); // Reset TTS when loading new document let didRedirect = false; + let startedLoad = false; try { if (!id) { setError('Document not found'); @@ -59,12 +68,27 @@ export default function PDFViewerPage() { router.replace(`/pdf/${resolved}`); return; } + + if (loadedDocIdRef.current === resolved) { + return; + } + if (inFlightDocIdRef.current === resolved) { + return; + } + + startedLoad = true; + inFlightDocIdRef.current = resolved; + stop(); // Reset TTS when loading new document await setCurrentDocument(resolved); + loadedDocIdRef.current = resolved; } catch (err) { console.error('Error loading document:', err); setError('Failed to load document'); } finally { - if (!didRedirect) { + if (startedLoad) { + inFlightDocIdRef.current = null; + } + if (!didRedirect && startedLoad) { setIsLoading(false); } } diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index 58c9386..15c1609 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -38,6 +38,7 @@ import { useRouter } from 'next/navigation'; import { showPrivacyModal } from '@/components/PrivacyModal'; import { deleteDocuments, mimeTypeForDoc, uploadDocuments } from '@/lib/client-documents'; import { cacheStoredDocumentFromBytes, clearDocumentCache } from '@/lib/document-cache'; +import { clearAllDocumentPreviewCaches, clearInMemoryDocumentPreviewCache } from '@/lib/document-preview-cache'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -188,6 +189,7 @@ export function SettingsModal({ className = '' }: { className?: string }) { const handleRefresh = async () => { try { + clearInMemoryDocumentPreviewCache(); await refreshDocuments(); } catch (error) { console.error('Failed to refresh documents:', error); @@ -196,7 +198,10 @@ export function SettingsModal({ className = '' }: { className?: string }) { const handleClearCache = async () => { try { - await clearDocumentCache(); + await Promise.all([ + clearDocumentCache(), + clearAllDocumentPreviewCaches(), + ]); } catch (error) { console.error('Failed to clear cache:', error); } diff --git a/src/components/doclist/DocumentPreview.tsx b/src/components/doclist/DocumentPreview.tsx index d6bcab9..699adb9 100644 --- a/src/components/doclist/DocumentPreview.tsx +++ b/src/components/doclist/DocumentPreview.tsx @@ -2,11 +2,16 @@ import { DocumentListDocument } from '@/types/documents'; import { PDFIcon, EPUBIcon, FileIcon } from '@/components/icons/Icons'; import { useEffect, useMemo, useRef, useState } from 'react'; import { - extractEpubCoverToDataUrl, - renderPdfFirstPageToDataUrl, -} from '@/lib/documentPreview'; -import { getDocumentContentSnippet } from '@/lib/client-documents'; -import { ensureCachedDocument } from '@/lib/document-cache'; + documentPreviewFallbackUrl, + getDocumentContentSnippet, + getDocumentPreviewStatus, +} from '@/lib/client-documents'; +import { + getInMemoryDocumentPreviewUrl, + getPersistedDocumentPreviewUrl, + primeDocumentPreviewCache, + setInMemoryDocumentPreviewUrl, +} from '@/lib/document-preview-cache'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; @@ -14,7 +19,6 @@ interface DocumentPreviewProps { doc: DocumentListDocument; } -const imagePreviewCache = new Map(); const textPreviewCache = new Map(); export function DocumentPreview({ doc }: DocumentPreviewProps) { @@ -33,20 +37,11 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { const containerRef = useRef(null); const [isVisible, setIsVisible] = useState(false); const [imagePreview, setImagePreview] = useState(null); + const [isImageReady, setIsImageReady] = useState(false); const [textPreview, setTextPreview] = useState(null); const [isGenerating, setIsGenerating] = useState(false); const previewKey = useMemo(() => `${doc.type}:${doc.id}`, [doc.id, doc.type]); - const cacheMeta = useMemo( - () => ({ - id: doc.id, - name: doc.name, - type: doc.type, - size: doc.size, - lastModified: doc.lastModified, - }), - [doc.id, doc.lastModified, doc.name, doc.size, doc.type], - ); useEffect(() => { const el = containerRef.current; @@ -69,7 +64,7 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { useEffect(() => { if (!isVisible) return; - const cachedImage = imagePreviewCache.get(previewKey); + const cachedImage = getInMemoryDocumentPreviewUrl(previewKey); if (cachedImage) { setImagePreview(cachedImage); setTextPreview(null); @@ -89,32 +84,65 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { const run = async () => { setIsGenerating(true); try { - const targetWidth = 240; - - if (doc.type === 'pdf') { - const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal }); - if (cached.type !== 'pdf') return; - const data = cached.data; - if (cancelled) return; - const dataUrl = await renderPdfFirstPageToDataUrl(data, targetWidth); - if (cancelled) return; - imagePreviewCache.set(previewKey, dataUrl); - setImagePreview(dataUrl); - setTextPreview(null); - return; - } - - if (doc.type === 'epub') { - const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal }); - if (cached.type !== 'epub') return; - const data = cached.data; - if (cancelled) return; - const cover = await extractEpubCoverToDataUrl(data, targetWidth); - if (cancelled) return; - if (cover) { - imagePreviewCache.set(previewKey, cover); - setImagePreview(cover); + if (doc.type === 'pdf' || doc.type === 'epub') { + const persistedUrl = await getPersistedDocumentPreviewUrl( + doc.id, + Number(doc.lastModified), + previewKey, + ); + if (!cancelled && persistedUrl) { + setImagePreview(persistedUrl); setTextPreview(null); + return; + } + + let attempt = 0; + while (!cancelled && attempt < 12) { + const status = await getDocumentPreviewStatus(doc.id, { signal: controller.signal }); + if (cancelled) return; + + if (status.kind === 'ready') { + const primedUrl = await primeDocumentPreviewCache( + doc.id, + Number(doc.lastModified), + previewKey, + { signal: controller.signal }, + ).catch(() => null); + if (cancelled) return; + + if (primedUrl) { + setImagePreview(primedUrl); + setTextPreview(null); + return; + } + + const fallbackUrl = status.fallbackUrl || documentPreviewFallbackUrl(doc.id); + setInMemoryDocumentPreviewUrl(previewKey, fallbackUrl); + setImagePreview(fallbackUrl); + setTextPreview(null); + return; + } + + if (status.status === 'failed') { + return; + } + + const waitMs = Math.max( + 400, + Math.min(6000, Number.isFinite(status.retryAfterMs) ? status.retryAfterMs : 1500), + ); + await new Promise((resolve) => { + const timer = setTimeout(resolve, waitMs); + controller.signal.addEventListener( + 'abort', + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); + attempt += 1; } return; } @@ -145,7 +173,11 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { cancelled = true; controller.abort(); }; - }, [cacheMeta, doc.id, doc.type, isVisible, previewKey]); + }, [doc.id, doc.lastModified, doc.type, isVisible, previewKey]); + + useEffect(() => { + setIsImageReady(false); + }, [imagePreview]); const gradientClass = isPDF ? 'from-red-500/80 via-red-400/60 to-red-600/80' @@ -176,15 +208,40 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { > {imagePreview ? ( <> +
+ {!isImageReady ? ( +
+ + + {typeLabel} + +
+ ) : null} {/* eslint-disable-next-line @next/next/no-img-element */} {`${doc.name} { + setIsImageReady(true); + }} + onError={() => { + if (!imagePreview) return; + setIsImageReady(false); + const fallback = documentPreviewFallbackUrl(doc.id); + if (imagePreview === fallback) return; + setInMemoryDocumentPreviewUrl(previewKey, fallback); + setImagePreview(fallback); + void primeDocumentPreviewCache( + doc.id, + Number(doc.lastModified), + previewKey, + ).catch(() => {}); + }} /> -
+ {isImageReady ?
: null} ) : textPreview ? ( <> diff --git a/src/db/schema.ts b/src/db/schema.ts index a89f820..726969b 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -14,3 +14,4 @@ export const verification = usePostgres ? postgresSchema.verification : sqliteSc export const userTtsChars = usePostgres ? postgresSchema.userTtsChars : sqliteSchema.userTtsChars; export const userPreferences = usePostgres ? postgresSchema.userPreferences : sqliteSchema.userPreferences; export const userDocumentProgress = usePostgres ? postgresSchema.userDocumentProgress : sqliteSchema.userDocumentProgress; +export const documentPreviews = usePostgres ? postgresSchema.documentPreviews : sqliteSchema.documentPreviews; diff --git a/src/db/schema_postgres.ts b/src/db/schema_postgres.ts index d514176..2bc955e 100644 --- a/src/db/schema_postgres.ts +++ b/src/db/schema_postgres.ts @@ -118,3 +118,26 @@ export const userDocumentProgress = pgTable('user_document_progress', { pk: primaryKey({ columns: [table.userId, table.documentId] }), userUpdatedIdx: index('idx_user_document_progress_user_id_updated_at').on(table.userId, table.updatedAt), })); + +export const documentPreviews = pgTable('document_previews', { + documentId: text('document_id').notNull(), + namespace: text('namespace').notNull().default(''), + variant: text('variant').notNull().default('card-240-jpeg'), + status: text('status').notNull().default('queued'), + sourceLastModifiedMs: bigint('source_last_modified_ms', { mode: 'number' }).notNull(), + objectKey: text('object_key').notNull(), + contentType: text('content_type').notNull().default('image/jpeg'), + width: integer('width').notNull().default(240), + height: integer('height'), + byteSize: bigint('byte_size', { mode: 'number' }), + eTag: text('etag'), + leaseOwner: text('lease_owner'), + leaseUntilMs: bigint('lease_until_ms', { mode: 'number' }).notNull().default(0), + attemptCount: integer('attempt_count').notNull().default(0), + lastError: text('last_error'), + createdAtMs: bigint('created_at_ms', { mode: 'number' }).notNull().default(0), + updatedAtMs: bigint('updated_at_ms', { mode: 'number' }).notNull().default(0), +}, (table) => ({ + pk: primaryKey({ columns: [table.documentId, table.namespace, table.variant] }), + statusLeaseIdx: index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs), +})); diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index c806f86..b82386c 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -119,3 +119,26 @@ export const userDocumentProgress = sqliteTable('user_document_progress', { pk: primaryKey({ columns: [table.userId, table.documentId] }), userUpdatedIdx: index('idx_user_document_progress_user_id_updated_at').on(table.userId, table.updatedAt), })); + +export const documentPreviews = sqliteTable('document_previews', { + documentId: text('document_id').notNull(), + namespace: text('namespace').notNull().default(''), + variant: text('variant').notNull().default('card-240-jpeg'), + status: text('status').notNull().default('queued'), + sourceLastModifiedMs: integer('source_last_modified_ms').notNull(), + objectKey: text('object_key').notNull(), + contentType: text('content_type').notNull().default('image/jpeg'), + width: integer('width').notNull().default(240), + height: integer('height'), + byteSize: integer('byte_size'), + eTag: text('etag'), + leaseOwner: text('lease_owner'), + leaseUntilMs: integer('lease_until_ms').notNull().default(0), + attemptCount: integer('attempt_count').notNull().default(0), + lastError: text('last_error'), + createdAtMs: integer('created_at_ms').notNull().default(0), + updatedAtMs: integer('updated_at_ms').notNull().default(0), +}, (table) => ({ + pk: primaryKey({ columns: [table.documentId, table.namespace, table.variant] }), + statusLeaseIdx: index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs), +})); diff --git a/src/lib/client-documents.ts b/src/lib/client-documents.ts index 194524b..3bdb954 100644 --- a/src/lib/client-documents.ts +++ b/src/lib/client-documents.ts @@ -202,17 +202,39 @@ export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user' } export async function downloadDocumentContent(id: string, options?: { signal?: AbortSignal }): Promise { - const res = await fetch(`/api/documents/blob?id=${encodeURIComponent(id)}`, { signal: options?.signal }); - if (!res.ok) { - const contentType = res.headers.get('content-type') || ''; - if (contentType.includes('application/json')) { - const data = (await res.json().catch(() => null)) as { error?: string } | null; - throw new Error(data?.error || `Failed to download document (status ${res.status})`); - } - throw new Error(`Failed to download document (status ${res.status})`); - } + const fallbackUrl = `/api/documents/blob/get/fallback?id=${encodeURIComponent(id)}`; - return res.arrayBuffer(); + const fetchFallback = async (): Promise => { + const res = await fetch(fallbackUrl, { signal: options?.signal }); + if (!res.ok) { + const contentType = res.headers.get('content-type') || ''; + if (contentType.includes('application/json')) { + const data = (await res.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || `Failed to download document (status ${res.status})`); + } + throw new Error(`Failed to download document (status ${res.status})`); + } + return res.arrayBuffer(); + }; + + try { + const directRes = await fetch(`/api/documents/blob/get/presign?id=${encodeURIComponent(id)}`, { + signal: options?.signal, + cache: 'no-store', + }); + if (!directRes.ok) { + const contentType = directRes.headers.get('content-type') || ''; + if (contentType.includes('application/json')) { + const data = (await directRes.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || `Failed to download document (status ${directRes.status})`); + } + throw new Error(`Failed to download document (status ${directRes.status})`); + } + return directRes.arrayBuffer(); + } catch (error) { + if (options?.signal?.aborted) throw error; + return fetchFallback(); + } } export async function getDocumentContentSnippet( @@ -235,6 +257,86 @@ export async function getDocumentContentSnippet( return data?.snippet || ''; } +export type DocumentPreviewPending = { + kind: 'pending'; + status: 'queued' | 'processing' | 'failed'; + retryAfterMs: number; + fallbackUrl: string; + presignUrl: string; + directUrl?: string; +}; + +export type DocumentPreviewReady = { + kind: 'ready'; + fallbackUrl: string; + presignUrl: string; + directUrl?: string; +}; + +export type DocumentPreviewStatus = DocumentPreviewPending | DocumentPreviewReady; + +function documentPreviewEnsureUrl(id: string): string { + return `/api/documents/blob/preview/ensure?id=${encodeURIComponent(id)}`; +} + +export function documentPreviewPresignUrl(id: string): string { + return `/api/documents/blob/preview/presign?id=${encodeURIComponent(id)}`; +} + +export function documentPreviewFallbackUrl(id: string): string { + return `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`; +} + +export async function getDocumentPreviewStatus( + id: string, + options?: { signal?: AbortSignal }, +): Promise { + const res = await fetch(documentPreviewEnsureUrl(id), { + signal: options?.signal, + cache: 'no-store', + }); + + if (res.status === 202) { + const data = (await res.json().catch(() => null)) as { + status?: 'queued' | 'processing' | 'failed'; + retryAfterMs?: number; + fallbackUrl?: string; + presignUrl?: string; + directUrl?: string; + } | null; + return { + kind: 'pending', + status: data?.status ?? 'queued', + retryAfterMs: Number.isFinite(data?.retryAfterMs) ? Number(data?.retryAfterMs) : 1500, + fallbackUrl: data?.fallbackUrl || documentPreviewFallbackUrl(id), + presignUrl: data?.presignUrl || documentPreviewPresignUrl(id), + directUrl: data?.directUrl, + }; + } + + if (res.ok) { + const data = (await res.json().catch(() => null)) as { + fallbackUrl?: string; + presignUrl?: string; + directUrl?: string; + } | null; + return { + kind: 'ready', + fallbackUrl: data?.fallbackUrl || documentPreviewFallbackUrl(id), + presignUrl: data?.presignUrl || documentPreviewPresignUrl(id), + directUrl: data?.directUrl, + }; + } + + const contentType = res.headers.get('content-type') || ''; + if (contentType.includes('application/json')) { + const data = (await res.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || `Failed to load preview status (status ${res.status})`); + } + + throw new Error(`Failed to load preview status (status ${res.status})`); +} + export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise { const form = new FormData(); form.append('file', file); diff --git a/src/lib/dexie.ts b/src/lib/dexie.ts index 5392b18..d48c0dd 100644 --- a/src/lib/dexie.ts +++ b/src/lib/dexie.ts @@ -14,7 +14,7 @@ import { cacheStoredDocumentFromBytes } from '@/lib/document-cache'; const DB_NAME = 'openreader-db'; // Managed via Dexie (version bumped from the original manual IndexedDB) -const DB_VERSION = 6; +const DB_VERSION = 8; const PDF_TABLE = 'pdf-documents' as const; const EPUB_TABLE = 'epub-documents' as const; @@ -23,6 +23,20 @@ const CONFIG_TABLE = 'config' as const; const APP_CONFIG_TABLE = 'app-config' as const; const LAST_LOCATION_TABLE = 'last-locations' as const; const DOCUMENT_ID_MAP_TABLE = 'document-id-map' as const; +const PREVIEW_CACHE_TABLE = 'document-preview-cache' as const; +const MIB = 1024 * 1024; +const DOCUMENT_CACHE_MAX_BYTES = 1024 * MIB; // 1 GiB +const PREVIEW_CACHE_MAX_BYTES = 128 * MIB; // 128 MiB + +interface DocumentCacheMeta { + cacheCreatedAt?: number; + cacheAccessedAt?: number; + cacheByteSize?: number; +} + +type PDFCacheRow = PDFDocument & DocumentCacheMeta; +type EPUBCacheRow = EPUBDocument & DocumentCacheMeta; +type HTMLCacheRow = HTMLDocument & DocumentCacheMeta; export interface LastLocationRow { docId: string; @@ -35,19 +49,29 @@ export interface DocumentIdMapRow { createdAt: number; } +export interface DocumentPreviewCacheRow { + docId: string; + lastModified: number; + contentType: string; + data: ArrayBuffer; + cachedAt: number; + byteSize?: number; +} + export interface ConfigRow { key: string; value: string; } type OpenReaderDB = Dexie & { - [PDF_TABLE]: EntityTable; - [EPUB_TABLE]: EntityTable; - [HTML_TABLE]: EntityTable; + [PDF_TABLE]: EntityTable; + [EPUB_TABLE]: EntityTable; + [HTML_TABLE]: EntityTable; [CONFIG_TABLE]: EntityTable; [APP_CONFIG_TABLE]: EntityTable; [LAST_LOCATION_TABLE]: EntityTable; [DOCUMENT_ID_MAP_TABLE]: EntityTable; + [PREVIEW_CACHE_TABLE]: EntityTable; }; export const db = new Dexie(DB_NAME) as OpenReaderDB; @@ -192,15 +216,15 @@ function buildAppConfigFromRaw(raw: RawConfigMap): AppConfigRow { return config; } -// Version 6: add document-id-map table; keep v5 upgrade to migrate scattered config keys -// and drop the legacy config table in a single upgrade step. +// Version 8: add local cache metadata/indexes so document + preview caches can be bounded via LRU pruning. db.version(DB_VERSION).stores({ - [PDF_TABLE]: 'id, type, name, lastModified, size, folderId', - [EPUB_TABLE]: 'id, type, name, lastModified, size, folderId', - [HTML_TABLE]: 'id, type, name, lastModified, size, folderId', + [PDF_TABLE]: 'id, type, name, lastModified, size, folderId, cacheAccessedAt', + [EPUB_TABLE]: 'id, type, name, lastModified, size, folderId, cacheAccessedAt', + [HTML_TABLE]: 'id, type, name, lastModified, size, folderId, cacheAccessedAt', [APP_CONFIG_TABLE]: 'id', [LAST_LOCATION_TABLE]: 'docId', [DOCUMENT_ID_MAP_TABLE]: 'oldId, id, createdAt', + [PREVIEW_CACHE_TABLE]: 'docId, lastModified, cachedAt, byteSize', // `null` here means: drop the old 'config' table after upgrade runs, // but Dexie still lets us read it inside the upgrade transaction. [CONFIG_TABLE]: null, @@ -231,6 +255,7 @@ db.version(DB_VERSION).stores({ }); let dbOpenPromise: Promise | null = null; +const cacheTextEncoder = new TextEncoder(); export async function initDB(): Promise { if (dbOpenPromise) { @@ -246,6 +271,9 @@ export async function initDB(): Promise { emitDexieStatus('stalled', { ms: Date.now() - startedAt }); }, 4000); await db.open(); + await Promise.all([pruneDocumentCacheIfNeededInternal(), prunePreviewCacheIfNeededInternal()]).catch((error) => { + console.warn('Dexie cache prune on open failed:', error); + }); clearTimeout(stallTimer); console.log('Dexie database opened successfully'); emitDexieStatus('opened'); @@ -265,6 +293,139 @@ async function withDB(operation: () => Promise): Promise { return operation(); } +type DocumentCacheTableName = typeof PDF_TABLE | typeof EPUB_TABLE | typeof HTML_TABLE; + +type DocumentCacheEntryRef = { + table: DocumentCacheTableName; + id: string; + byteSize: number; + accessedAt: number; +}; + +function toPositiveInt(value: unknown, fallback: number = 0): number { + const n = Number(value); + if (!Number.isFinite(n) || n < 0) return fallback; + return Math.max(0, Math.floor(n)); +} + +function byteSizeForPdfRow(row: PDFCacheRow): number { + const cached = toPositiveInt(row.cacheByteSize, 0); + if (cached > 0) return cached; + return row.data instanceof ArrayBuffer ? row.data.byteLength : toPositiveInt(row.size, 0); +} + +function byteSizeForEpubRow(row: EPUBCacheRow): number { + const cached = toPositiveInt(row.cacheByteSize, 0); + if (cached > 0) return cached; + return row.data instanceof ArrayBuffer ? row.data.byteLength : toPositiveInt(row.size, 0); +} + +function byteSizeForHtmlRow(row: HTMLCacheRow): number { + const cached = toPositiveInt(row.cacheByteSize, 0); + if (cached > 0) return cached; + if (typeof row.data === 'string') return cacheTextEncoder.encode(row.data).byteLength; + return toPositiveInt(row.size, 0); +} + +function accessedAtForRow(row: DocumentCacheMeta & { lastModified?: number }): number { + return toPositiveInt(row.cacheAccessedAt, toPositiveInt(row.lastModified, Date.now())); +} + +async function deleteDocumentCacheEntryInternal(table: DocumentCacheTableName, id: string): Promise { + if (table === PDF_TABLE) { + await db[PDF_TABLE].delete(id); + return; + } + if (table === EPUB_TABLE) { + await db[EPUB_TABLE].delete(id); + return; + } + await db[HTML_TABLE].delete(id); +} + +async function pruneDocumentCacheIfNeededInternal(): Promise { + const budget = DOCUMENT_CACHE_MAX_BYTES; + if (!Number.isFinite(budget) || budget <= 0) return; + + const [pdfRows, epubRows, htmlRows] = await Promise.all([ + db[PDF_TABLE].toArray(), + db[EPUB_TABLE].toArray(), + db[HTML_TABLE].toArray(), + ]); + + const entries: DocumentCacheEntryRef[] = []; + let totalBytes = 0; + + for (const row of pdfRows) { + const byteSize = byteSizeForPdfRow(row); + totalBytes += byteSize; + entries.push({ + table: PDF_TABLE, + id: row.id, + byteSize, + accessedAt: accessedAtForRow(row), + }); + } + for (const row of epubRows) { + const byteSize = byteSizeForEpubRow(row); + totalBytes += byteSize; + entries.push({ + table: EPUB_TABLE, + id: row.id, + byteSize, + accessedAt: accessedAtForRow(row), + }); + } + for (const row of htmlRows) { + const byteSize = byteSizeForHtmlRow(row); + totalBytes += byteSize; + entries.push({ + table: HTML_TABLE, + id: row.id, + byteSize, + accessedAt: accessedAtForRow(row), + }); + } + + if (totalBytes <= budget) return; + + entries.sort((a, b) => a.accessedAt - b.accessedAt); + for (const entry of entries) { + if (totalBytes <= budget) break; + await deleteDocumentCacheEntryInternal(entry.table, entry.id); + totalBytes -= entry.byteSize; + } +} + +async function prunePreviewCacheIfNeededInternal(): Promise { + const budget = PREVIEW_CACHE_MAX_BYTES; + if (!Number.isFinite(budget) || budget <= 0) return; + + const rows = await db[PREVIEW_CACHE_TABLE].toArray(); + let totalBytes = 0; + const entries = rows.map((row) => { + const byteSize = toPositiveInt( + row.byteSize, + row.data instanceof ArrayBuffer ? row.data.byteLength : 0, + ); + totalBytes += byteSize; + return { + docId: row.docId, + byteSize, + accessedAt: toPositiveInt(row.cachedAt, 0), + }; + }); + + if (totalBytes <= budget) return; + + entries.sort((a, b) => a.accessedAt - b.accessedAt); + for (const entry of entries) { + if (totalBytes <= budget) break; + await db[PREVIEW_CACHE_TABLE].delete(entry.docId); + totalBytes -= entry.byteSize; + } +} + function isSha256HexId(value: string): boolean { return /^[a-f0-9]{64}$/i.test(value); } @@ -325,6 +486,7 @@ async function applyDocumentIdMapping(oldId: string, newId: string): Promise { await recordDocumentIdMapping(oldId, nextId); @@ -395,6 +557,15 @@ async function applyDocumentIdMapping(oldId: string, newId: string): Promise { await withDB(async () => { console.log('Adding PDF document via Dexie:', document.name); - await db[PDF_TABLE].put(document); + const now = Date.now(); + await db[PDF_TABLE].put({ + ...document, + cacheCreatedAt: now, + cacheAccessedAt: now, + cacheByteSize: document.data.byteLength, + }); + await pruneDocumentCacheIfNeededInternal(); }); } @@ -465,7 +643,11 @@ export async function getPdfDocument(id: string): Promise { console.log('Fetching PDF document via Dexie:', id); const resolved = await getMappedDocumentId(id); - return db[PDF_TABLE].get(resolved); + const row = await db[PDF_TABLE].get(resolved); + if (row) { + await db[PDF_TABLE].update(resolved, { cacheAccessedAt: Date.now() }); + } + return row; }); } @@ -480,9 +662,10 @@ export async function removePdfDocument(id: string): Promise { await withDB(async () => { console.log('Removing PDF document via Dexie:', id); const resolved = await getMappedDocumentId(id); - await db.transaction('readwrite', db[PDF_TABLE], db[LAST_LOCATION_TABLE], async () => { + await db.transaction('readwrite', db[PDF_TABLE], db[LAST_LOCATION_TABLE], db[PREVIEW_CACHE_TABLE], async () => { await db[PDF_TABLE].delete(resolved); await db[LAST_LOCATION_TABLE].delete(resolved); + await db[PREVIEW_CACHE_TABLE].delete(resolved); }); }); } @@ -508,7 +691,14 @@ export async function addEpubDocument(document: EPUBDocument): Promise { actualSize: document.data.byteLength, }); - await db[EPUB_TABLE].put(document); + const now = Date.now(); + await db[EPUB_TABLE].put({ + ...document, + cacheCreatedAt: now, + cacheAccessedAt: now, + cacheByteSize: document.data.byteLength, + }); + await pruneDocumentCacheIfNeededInternal(); }); } @@ -516,7 +706,11 @@ export async function getEpubDocument(id: string): Promise { console.log('Fetching EPUB document via Dexie:', id); const resolved = await getMappedDocumentId(id); - return db[EPUB_TABLE].get(resolved); + const row = await db[EPUB_TABLE].get(resolved); + if (row) { + await db[EPUB_TABLE].update(resolved, { cacheAccessedAt: Date.now() }); + } + return row; }); } @@ -531,9 +725,10 @@ export async function removeEpubDocument(id: string): Promise { await withDB(async () => { console.log('Removing EPUB document via Dexie:', id); const resolved = await getMappedDocumentId(id); - await db.transaction('readwrite', db[EPUB_TABLE], db[LAST_LOCATION_TABLE], async () => { + await db.transaction('readwrite', db[EPUB_TABLE], db[LAST_LOCATION_TABLE], db[PREVIEW_CACHE_TABLE], async () => { await db[EPUB_TABLE].delete(resolved); await db[LAST_LOCATION_TABLE].delete(resolved); + await db[PREVIEW_CACHE_TABLE].delete(resolved); }); }); } @@ -550,7 +745,14 @@ export async function clearEpubDocuments(): Promise { export async function addHtmlDocument(document: HTMLDocument): Promise { await withDB(async () => { console.log('Adding HTML document via Dexie:', document.name); - await db[HTML_TABLE].put(document); + const now = Date.now(); + await db[HTML_TABLE].put({ + ...document, + cacheCreatedAt: now, + cacheAccessedAt: now, + cacheByteSize: cacheTextEncoder.encode(document.data).byteLength, + }); + await pruneDocumentCacheIfNeededInternal(); }); } @@ -558,7 +760,11 @@ export async function getHtmlDocument(id: string): Promise { console.log('Fetching HTML document via Dexie:', id); const resolved = await getMappedDocumentId(id); - return db[HTML_TABLE].get(resolved); + const row = await db[HTML_TABLE].get(resolved); + if (row) { + await db[HTML_TABLE].update(resolved, { cacheAccessedAt: Date.now() }); + } + return row; }); } @@ -573,7 +779,10 @@ export async function removeHtmlDocument(id: string): Promise { await withDB(async () => { console.log('Removing HTML document via Dexie:', id); const resolved = await getMappedDocumentId(id); - await db[HTML_TABLE].delete(resolved); + await db.transaction('readwrite', db[HTML_TABLE], db[PREVIEW_CACHE_TABLE], async () => { + await db[HTML_TABLE].delete(resolved); + await db[PREVIEW_CACHE_TABLE].delete(resolved); + }); }); } @@ -648,6 +857,45 @@ export async function setFirstVisit(value: boolean): Promise { await updateAppConfig({ firstVisit: value }); } +// Document preview cache helpers + +export async function getDocumentPreviewCache(docId: string): Promise { + return withDB(async () => { + const resolved = await getMappedDocumentId(docId); + const row = await db[PREVIEW_CACHE_TABLE].get(resolved); + if (row) { + await db[PREVIEW_CACHE_TABLE].update(resolved, { cachedAt: Date.now() }); + } + return row; + }); +} + +export async function putDocumentPreviewCache(row: DocumentPreviewCacheRow): Promise { + await withDB(async () => { + const resolved = await getMappedDocumentId(row.docId); + await db[PREVIEW_CACHE_TABLE].put({ + ...row, + docId: resolved, + cachedAt: Date.now(), + byteSize: toPositiveInt(row.byteSize, row.data.byteLength), + }); + await prunePreviewCacheIfNeededInternal(); + }); +} + +export async function removeDocumentPreviewCache(docId: string): Promise { + await withDB(async () => { + const resolved = await getMappedDocumentId(docId); + await db[PREVIEW_CACHE_TABLE].delete(resolved); + }); +} + +export async function clearDocumentPreviewCache(): Promise { + await withDB(async () => { + await db[PREVIEW_CACHE_TABLE].clear(); + }); +} + // Sync helpers (server round-trip) export async function syncDocumentsToServer( diff --git a/src/lib/document-preview-cache.ts b/src/lib/document-preview-cache.ts new file mode 100644 index 0000000..ead268d --- /dev/null +++ b/src/lib/document-preview-cache.ts @@ -0,0 +1,125 @@ +import { + clearDocumentPreviewCache as clearPersistedDocumentPreviewCache, + getDocumentPreviewCache, + putDocumentPreviewCache, + removeDocumentPreviewCache, +} from '@/lib/dexie'; +import { documentPreviewFallbackUrl, documentPreviewPresignUrl } from '@/lib/client-documents'; + +const inMemoryPreviewUrlCache = new Map(); +const inFlightPreviewPrime = new Map>(); + +function revokeIfBlobUrl(url: string | null | undefined): void { + if (!url) return; + if (!url.startsWith('blob:')) return; + try { + URL.revokeObjectURL(url); + } catch { + // ignore + } +} + +export function getInMemoryDocumentPreviewUrl(cacheKey: string): string | null { + return inMemoryPreviewUrlCache.get(cacheKey) || null; +} + +export function setInMemoryDocumentPreviewUrl(cacheKey: string, url: string): void { + const prev = inMemoryPreviewUrlCache.get(cacheKey); + if (prev && prev !== url) { + revokeIfBlobUrl(prev); + } + inMemoryPreviewUrlCache.set(cacheKey, url); +} + +export function clearInMemoryDocumentPreviewCache(): void { + for (const value of inMemoryPreviewUrlCache.values()) { + revokeIfBlobUrl(value); + } + inMemoryPreviewUrlCache.clear(); +} + +export async function getPersistedDocumentPreviewUrl( + docId: string, + lastModified: number, + cacheKey: string, +): Promise { + const row = await getDocumentPreviewCache(docId); + if (!row) return null; + + if (Number(row.lastModified) !== Number(lastModified)) { + await removeDocumentPreviewCache(docId).catch(() => {}); + return null; + } + + const contentType = row.contentType || 'image/jpeg'; + const bytes = row.data; + if (!(bytes instanceof ArrayBuffer) || bytes.byteLength === 0) { + await removeDocumentPreviewCache(docId).catch(() => {}); + return null; + } + + const url = URL.createObjectURL(new Blob([bytes], { type: contentType })); + setInMemoryDocumentPreviewUrl(cacheKey, url); + return url; +} + +export async function primeDocumentPreviewCache( + docId: string, + lastModified: number, + cacheKey: string, + options?: { signal?: AbortSignal }, +): Promise { + const primeKey = `${cacheKey}:${Number(lastModified)}`; + const existingPrime = inFlightPreviewPrime.get(primeKey); + if (existingPrime) { + return existingPrime; + } + + const promise = (async (): Promise => { + const existing = await getPersistedDocumentPreviewUrl(docId, lastModified, cacheKey); + if (existing) return existing; + + const fetchOptions = { + signal: options?.signal, + cache: 'no-store' as const, + }; + + // Prefer presign path for priming so healthy direct object access avoids proxy load. + let res = await fetch(documentPreviewPresignUrl(docId), fetchOptions).catch(() => null); + if (!res || !res.ok) { + res = await fetch(documentPreviewFallbackUrl(docId), fetchOptions).catch(() => null); + } + if (!res || !res.ok) return null; + + const blob = await res.blob(); + const bytes = await blob.arrayBuffer(); + if (bytes.byteLength === 0) return null; + + const contentType = blob.type || 'image/jpeg'; + await putDocumentPreviewCache({ + docId, + lastModified: Number(lastModified), + contentType, + data: bytes, + cachedAt: Date.now(), + }); + + const url = URL.createObjectURL(new Blob([bytes], { type: contentType })); + setInMemoryDocumentPreviewUrl(cacheKey, url); + return url; + })(); + + inFlightPreviewPrime.set(primeKey, promise); + try { + return await promise; + } finally { + if (inFlightPreviewPrime.get(primeKey) === promise) { + inFlightPreviewPrime.delete(primeKey); + } + } +} + +export async function clearAllDocumentPreviewCaches(): Promise { + clearInMemoryDocumentPreviewCache(); + await clearPersistedDocumentPreviewCache(); +} diff --git a/src/lib/server/document-previews-blobstore.ts b/src/lib/server/document-previews-blobstore.ts new file mode 100644 index 0000000..8fce80c --- /dev/null +++ b/src/lib/server/document-previews-blobstore.ts @@ -0,0 +1,148 @@ +import { + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { deleteDocumentPrefix, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { getS3Client, getS3Config } from '@/lib/server/s3'; + +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; +const DEFAULT_NAMESPACE_SEGMENT = '_default'; + +export const DOCUMENT_PREVIEW_VARIANT = 'card-240-jpeg'; +export const DOCUMENT_PREVIEW_FILE_NAME = 'card-240.jpg'; +export const DOCUMENT_PREVIEW_CONTENT_TYPE = 'image/jpeg'; +export const DOCUMENT_PREVIEW_WIDTH = 240; + +function sanitizeNamespace(namespace: string | null): string | null { + if (!namespace) return null; + if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null; + return namespace; +} + +function namespaceSegment(namespace: string | null): string { + const safe = sanitizeNamespace(namespace); + return safe ?? DEFAULT_NAMESPACE_SEGMENT; +} + +function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream { + return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function'; +} + +async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) { + chunks.push(chunk); + } else if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk)); + } else { + chunks.push(Buffer.from(chunk as Uint8Array)); + } + } + return Buffer.concat(chunks); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + + if (isNodeReadableStream(body)) { + return streamToBuffer(body); + } + + throw new Error('Unsupported S3 response body type'); +} + +export function documentPreviewPrefix(documentId: string, namespace: string | null): string { + if (!isValidDocumentId(documentId)) { + throw new Error(`Invalid document id: ${documentId}`); + } + + const cfg = getS3Config(); + const ns = namespaceSegment(namespace); + return `${cfg.prefix}/document_previews_v1/ns/${ns}/${documentId}/`; +} + +export function documentPreviewKey(documentId: string, namespace: string | null): string { + return `${documentPreviewPrefix(documentId, namespace)}${DOCUMENT_PREVIEW_FILE_NAME}`; +} + +export async function headDocumentPreview( + documentId: string, + namespace: string | null, +): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentPreviewKey(documentId, namespace); + const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key })); + return { + contentLength: Number(res.ContentLength ?? 0), + contentType: res.ContentType ?? null, + eTag: res.ETag ?? null, + }; +} + +export async function getDocumentPreviewBuffer(documentId: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentPreviewKey(documentId, namespace); + const res = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: key })); + return bodyToBuffer(res.Body); +} + +export async function putDocumentPreviewBuffer( + documentId: string, + bytes: Buffer, + namespace: string | null, + options?: { ifNoneMatch?: boolean }, +): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentPreviewKey(documentId, namespace); + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: bytes, + ContentType: DOCUMENT_PREVIEW_CONTENT_TYPE, + ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), + }), + ); +} + +export async function presignDocumentPreviewGet( + documentId: string, + namespace: string | null, + options?: { expiresInSeconds?: number }, +): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentPreviewKey(documentId, namespace); + return getSignedUrl( + client, + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + ResponseContentType: DOCUMENT_PREVIEW_CONTENT_TYPE, + }), + { expiresIn: Math.max(30, Math.min(options?.expiresInSeconds ?? 300, 3600)) }, + ); +} + +export async function deleteDocumentPreviewArtifacts(documentId: string, namespace: string | null): Promise { + return deleteDocumentPrefix(documentPreviewPrefix(documentId, namespace)); +} + +export { isMissingBlobError }; diff --git a/src/lib/server/document-previews-render.ts b/src/lib/server/document-previews-render.ts new file mode 100644 index 0000000..87415c5 --- /dev/null +++ b/src/lib/server/document-previews-render.ts @@ -0,0 +1,258 @@ +import path from 'path'; +import { DOMMatrix, Path2D, createCanvas, loadImage } from '@napi-rs/canvas'; +import JSZip from 'jszip'; +import { XMLParser } from 'fast-xml-parser'; + +export type RenderedDocumentPreview = { + bytes: Buffer; + width: number; + height: number; +}; + +type CanvasAndContext = { + canvas: unknown; + context: unknown; +}; + +type NodeCanvasFactory = { + create: (width: number, height: number) => CanvasAndContext; + reset: (target: CanvasAndContext, width: number, height: number) => void; + destroy: (target: CanvasAndContext) => void; +}; + +function ensureNodeCanvasGlobals(): void { + const g = globalThis as Record; + if (typeof g.DOMMatrix === 'undefined') { + g.DOMMatrix = DOMMatrix as unknown; + } + if (typeof g.Path2D === 'undefined') { + g.Path2D = Path2D as unknown; + } +} + +function normalizeTargetWidth(targetWidth: number): number { + if (!Number.isFinite(targetWidth) || targetWidth <= 0) return 240; + return Math.max(64, Math.min(2048, Math.round(targetWidth))); +} + +function asArray(value: T | T[] | undefined): T[] { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +} + +function findBestEpubCoverPath(opfPath: string, opfXml: string): string | null { + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + removeNSPrefix: true, + }); + const parsed = parser.parse(opfXml) as Record; + const pkg = (parsed.package ?? parsed['opf:package']) as Record | undefined; + if (!pkg) return null; + + const metadata = pkg.metadata as Record | undefined; + const manifest = pkg.manifest as Record | undefined; + const items = asArray((manifest?.item as Record | Array> | undefined)) + .filter((item) => typeof item === 'object' && item !== null); + + const coverMetaId = asArray((metadata?.meta as Record | Array> | undefined)) + .find((meta) => { + const name = String(meta?.['@_name'] ?? '').trim().toLowerCase(); + return name === 'cover'; + })?.['@_content']; + + const byCoverProperty = items.find((item) => + String(item['@_properties'] ?? '') + .split(/\s+/) + .map((value) => value.trim().toLowerCase()) + .includes('cover-image'), + ); + const byMetaRef = coverMetaId + ? items.find((item) => String(item['@_id'] ?? '') === String(coverMetaId)) + : null; + const byNameHint = items.find((item) => String(item['@_id'] ?? '').toLowerCase().includes('cover')); + const byImageType = items.find((item) => String(item['@_media-type'] ?? '').toLowerCase().startsWith('image/')); + + const selected = byCoverProperty ?? byMetaRef ?? byNameHint ?? byImageType; + if (!selected) return null; + + const href = String(selected['@_href'] ?? '').trim(); + if (!href) return null; + + const opfDir = path.posix.dirname(opfPath); + return path.posix.normalize(path.posix.join(opfDir, href)); +} + +async function renderImageBytesToJpeg(imageBytes: Buffer, targetWidth: number): Promise { + const bitmap = await loadImage(imageBytes); + const width = bitmap.width; + const height = bitmap.height; + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + throw new Error('Invalid source image dimensions'); + } + + const target = normalizeTargetWidth(targetWidth); + const scale = target / width; + const outWidth = Math.max(1, Math.round(width * scale)); + const outHeight = Math.max(1, Math.round(height * scale)); + const canvas = createCanvas(outWidth, outHeight); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, outWidth, outHeight); + ctx.drawImage(bitmap, 0, 0, outWidth, outHeight); + return { + bytes: canvas.toBuffer('image/jpeg', 82), + width: outWidth, + height: outHeight, + }; +} + +export async function renderEpubCoverToJpeg(sourceBytes: Buffer, targetWidth: number): Promise { + const zip = await JSZip.loadAsync(sourceBytes); + const containerFile = zip.file('META-INF/container.xml'); + if (!containerFile) { + throw new Error('EPUB container.xml not found'); + } + + const containerXml = await containerFile.async('string'); + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + removeNSPrefix: true, + }); + const containerParsed = parser.parse(containerXml) as Record; + const rootfilesNode = (containerParsed.container as Record | undefined)?.rootfiles as + | Record + | undefined; + const rootfiles = asArray(rootfilesNode?.rootfile as Record | Array> | undefined); + const rootfilePath = String(rootfiles[0]?.['@_full-path'] ?? '').trim(); + if (!rootfilePath) { + throw new Error('EPUB OPF rootfile path missing'); + } + + const opfFile = zip.file(rootfilePath) ?? zip.file(decodeURI(rootfilePath)); + if (!opfFile) { + throw new Error(`EPUB OPF not found at ${rootfilePath}`); + } + const opfXml = await opfFile.async('string'); + const coverPath = findBestEpubCoverPath(rootfilePath, opfXml); + if (!coverPath) { + throw new Error('EPUB cover image not found'); + } + + const coverFile = zip.file(coverPath) ?? zip.file(decodeURI(coverPath)); + if (!coverFile) { + throw new Error(`EPUB cover file missing at ${coverPath}`); + } + + const coverBytes = await coverFile.async('nodebuffer'); + return renderImageBytesToJpeg(coverBytes, targetWidth); +} + +export async function renderPdfFirstPageToJpeg(sourceBytes: Buffer, targetWidth: number): Promise { + ensureNodeCanvasGlobals(); + + type PdfViewport = { + width: number; + height: number; + }; + type PdfPage = { + getViewport: (params: { scale: number }) => PdfViewport; + render: (params: { + canvasContext: unknown; + viewport: PdfViewport; + intent: 'display'; + canvasFactory?: NodeCanvasFactory; + }) => { promise: Promise }; + }; + + // pdfjs-dist legacy build works in Node and avoids relying on DOM workers. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore - pdfjs-dist legacy build path has no dedicated TypeScript declaration. + const pdfjs = (await import('pdfjs-dist/legacy/build/pdf.mjs')) as { + getDocument: (options: Record) => { + promise: Promise<{ getPage: (n: number) => Promise; destroy: () => Promise }>; + destroy: () => Promise; + }; + GlobalWorkerOptions?: { workerSrc?: string; workerPort?: unknown }; + }; + + if (pdfjs.GlobalWorkerOptions) { + pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs'; + pdfjs.GlobalWorkerOptions.workerPort = null; + } + + const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts'); + const standardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`; + + const nodeCanvasFactory: NodeCanvasFactory = { + create: (width, height) => { + const canvas = createCanvas(width, height); + const context = canvas.getContext('2d'); + return { canvas, context }; + }, + reset: (target, width, height) => { + const canvas = target.canvas as { width: number; height: number }; + canvas.width = width; + canvas.height = height; + }, + destroy: (target) => { + const canvas = target.canvas as { width: number; height: number }; + canvas.width = 0; + canvas.height = 0; + }, + }; + + class PdfNodeCanvasFactory { + create(width: number, height: number): CanvasAndContext { + return nodeCanvasFactory.create(width, height); + } + reset(target: CanvasAndContext, width: number, height: number): void { + nodeCanvasFactory.reset(target, width, height); + } + destroy(target: CanvasAndContext): void { + nodeCanvasFactory.destroy(target); + } + } + + const loadingTask = pdfjs.getDocument({ + data: new Uint8Array(sourceBytes), + useWorkerFetch: false, + standardFontDataUrl, + CanvasFactory: PdfNodeCanvasFactory, + isEvalSupported: false, + }); + const pdf = await loadingTask.promise; + + try { + const page = await pdf.getPage(1); + const viewport = page.getViewport({ scale: 1 }); + const target = normalizeTargetWidth(targetWidth); + const scale = target / viewport.width; + const scaledViewport = page.getViewport({ scale }); + + const outWidth = Math.max(1, Math.floor(scaledViewport.width)); + const outHeight = Math.max(1, Math.floor(scaledViewport.height)); + const canvas = createCanvas(outWidth, outHeight); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, outWidth, outHeight); + + const renderTask = page.render({ + canvasContext: ctx, + viewport: scaledViewport, + intent: 'display', + canvasFactory: nodeCanvasFactory, + }); + await renderTask.promise; + + return { + bytes: canvas.toBuffer('image/jpeg', 82), + width: outWidth, + height: outHeight, + }; + } finally { + await pdf.destroy().catch(() => undefined); + await loadingTask.destroy().catch(() => undefined); + } +} diff --git a/src/lib/server/document-previews.ts b/src/lib/server/document-previews.ts new file mode 100644 index 0000000..3089de2 --- /dev/null +++ b/src/lib/server/document-previews.ts @@ -0,0 +1,427 @@ +import { randomUUID } from 'crypto'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { and, eq, inArray, lt, or, sql } from 'drizzle-orm'; +import { db } from '@/db'; +import { documentPreviews } from '@/db/schema'; +import { + DOCUMENT_PREVIEW_CONTENT_TYPE, + DOCUMENT_PREVIEW_VARIANT, + DOCUMENT_PREVIEW_WIDTH, + deleteDocumentPreviewArtifacts, + documentPreviewKey, + headDocumentPreview, + isMissingBlobError, + putDocumentPreviewBuffer, +} from '@/lib/server/document-previews-blobstore'; +import { getDocumentBlob } from '@/lib/server/documents-blobstore'; +import { renderEpubCoverToJpeg, renderPdfFirstPageToJpeg } from '@/lib/server/document-previews-render'; + +const LEASE_MS = 45_000; +const RETRY_AFTER_MS = 1_500; +const FAILED_RETRY_AFTER_MS = 15_000; + +type PreviewStatus = 'queued' | 'processing' | 'ready' | 'failed'; + +type PreviewRow = { + documentId: string; + namespace: string; + variant: string; + status: PreviewStatus; + sourceLastModifiedMs: number; + objectKey: string; + contentType: string; + width: number; + height: number | null; + byteSize: number | null; + eTag: string | null; + leaseOwner: string | null; + leaseUntilMs: number; + attemptCount: number; + lastError: string | null; + createdAtMs: number; + updatedAtMs: number; +}; + +export type PreviewableDocumentType = 'pdf' | 'epub'; + +export type PreviewSourceDocument = { + id: string; + type: string; + lastModified: number; +}; + +export type EnsureDocumentPreviewResult = + | { + state: 'ready'; + status: 'ready'; + contentType: string; + width: number; + height: number | null; + byteSize: number | null; + eTag: string | null; + } + | { + state: 'pending'; + status: Exclude; + retryAfterMs: number; + lastError: string | null; + }; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function safeDb(): any { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return db as any; +} + +function nowMs(): number { + return Date.now(); +} + +function rowsAffected(result: unknown): number { + if (!result || typeof result !== 'object') return 0; + const rec = result as Record; + if (typeof rec.rowCount === 'number') return rec.rowCount; + if (typeof rec.changes === 'number') return rec.changes; + return 0; +} + +function toNamespaceKey(namespace: string | null): string { + return namespace?.trim() || ''; +} + +function previewObjectKey(documentId: string, namespace: string | null): string { + return documentPreviewKey(documentId, namespace); +} + +function asPreviewStatus(status: string | null | undefined): PreviewStatus { + if (status === 'processing' || status === 'ready' || status === 'failed') return status; + return 'queued'; +} + +function toPreviewRow(raw: Record): PreviewRow { + return { + documentId: String(raw.documentId ?? ''), + namespace: String(raw.namespace ?? ''), + variant: String(raw.variant ?? ''), + status: asPreviewStatus(String(raw.status ?? 'queued')), + sourceLastModifiedMs: Number(raw.sourceLastModifiedMs ?? 0), + objectKey: String(raw.objectKey ?? ''), + contentType: String(raw.contentType ?? DOCUMENT_PREVIEW_CONTENT_TYPE), + width: Number(raw.width ?? DOCUMENT_PREVIEW_WIDTH), + height: raw.height == null ? null : Number(raw.height), + byteSize: raw.byteSize == null ? null : Number(raw.byteSize), + eTag: raw.eTag == null ? null : String(raw.eTag), + leaseOwner: raw.leaseOwner == null ? null : String(raw.leaseOwner), + leaseUntilMs: Number(raw.leaseUntilMs ?? 0), + attemptCount: Number(raw.attemptCount ?? 0), + lastError: raw.lastError == null ? null : String(raw.lastError), + createdAtMs: Number(raw.createdAtMs ?? 0), + updatedAtMs: Number(raw.updatedAtMs ?? 0), + }; +} + +export function isPreviewableDocumentType(type: string): type is PreviewableDocumentType { + return type === 'pdf' || type === 'epub'; +} + +async function getPreviewRow(documentId: string, namespaceKey: string): Promise { + const rows = (await safeDb() + .select() + .from(documentPreviews) + .where( + and( + eq(documentPreviews.documentId, documentId), + eq(documentPreviews.namespace, namespaceKey), + eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT), + ), + )) as Array>; + const row = rows[0]; + return row ? toPreviewRow(row) : null; +} + +async function ensurePreviewRowExists(doc: PreviewSourceDocument, namespaceKey: string, namespace: string | null): Promise { + const now = nowMs(); + await safeDb() + .insert(documentPreviews) + .values({ + documentId: doc.id, + namespace: namespaceKey, + variant: DOCUMENT_PREVIEW_VARIANT, + status: 'queued', + sourceLastModifiedMs: doc.lastModified, + objectKey: previewObjectKey(doc.id, namespace), + contentType: DOCUMENT_PREVIEW_CONTENT_TYPE, + width: DOCUMENT_PREVIEW_WIDTH, + leaseUntilMs: 0, + attemptCount: 0, + createdAtMs: now, + updatedAtMs: now, + }) + .onConflictDoNothing(); +} + +async function markPreviewRowQueued( + doc: PreviewSourceDocument, + namespaceKey: string, + namespace: string | null, +): Promise { + const now = nowMs(); + await safeDb() + .update(documentPreviews) + .set({ + status: 'queued', + sourceLastModifiedMs: doc.lastModified, + objectKey: previewObjectKey(doc.id, namespace), + contentType: DOCUMENT_PREVIEW_CONTENT_TYPE, + width: DOCUMENT_PREVIEW_WIDTH, + height: null, + byteSize: null, + eTag: null, + leaseOwner: null, + leaseUntilMs: 0, + lastError: null, + updatedAtMs: now, + }) + .where( + and( + eq(documentPreviews.documentId, doc.id), + eq(documentPreviews.namespace, namespaceKey), + eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT), + ), + ); +} + +function needsRequeue(row: PreviewRow, doc: PreviewSourceDocument, namespace: string | null): boolean { + if (row.sourceLastModifiedMs !== doc.lastModified) return true; + if (row.objectKey !== previewObjectKey(doc.id, namespace)) return true; + return false; +} + +async function isReadyBlobMissing(docId: string, namespace: string | null): Promise { + try { + await headDocumentPreview(docId, namespace); + return false; + } catch (error) { + if (isMissingBlobError(error)) return true; + throw error; + } +} + +async function tryClaimPreviewLease(docId: string, namespaceKey: string, owner: string): Promise { + const now = nowMs(); + const result = await safeDb() + .update(documentPreviews) + .set({ + status: 'processing', + leaseOwner: owner, + leaseUntilMs: now + LEASE_MS, + attemptCount: sql`${documentPreviews.attemptCount} + 1`, + lastError: null, + updatedAtMs: now, + }) + .where( + and( + eq(documentPreviews.documentId, docId), + eq(documentPreviews.namespace, namespaceKey), + eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT), + or( + inArray(documentPreviews.status, ['queued', 'failed']), + and( + eq(documentPreviews.status, 'processing'), + lt(documentPreviews.leaseUntilMs, now), + ), + ), + ), + ); + return rowsAffected(result) > 0; +} + +async function markPreviewReady( + doc: PreviewSourceDocument, + namespaceKey: string, + content: { width: number; height: number | null; byteSize: number; eTag: string | null }, +): Promise { + const now = nowMs(); + await safeDb() + .update(documentPreviews) + .set({ + status: 'ready', + sourceLastModifiedMs: doc.lastModified, + contentType: DOCUMENT_PREVIEW_CONTENT_TYPE, + width: content.width, + height: content.height, + byteSize: content.byteSize, + eTag: content.eTag, + leaseOwner: null, + leaseUntilMs: 0, + lastError: null, + updatedAtMs: now, + }) + .where( + and( + eq(documentPreviews.documentId, doc.id), + eq(documentPreviews.namespace, namespaceKey), + eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT), + ), + ); +} + +async function markPreviewFailed(docId: string, namespaceKey: string, error: unknown): Promise { + const now = nowMs(); + const message = error instanceof Error ? error.message : String(error ?? 'Preview generation failed'); + await safeDb() + .update(documentPreviews) + .set({ + status: 'failed', + leaseOwner: null, + leaseUntilMs: 0, + lastError: message.slice(0, 1000), + updatedAtMs: now, + }) + .where( + and( + eq(documentPreviews.documentId, docId), + eq(documentPreviews.namespace, namespaceKey), + eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT), + ), + ); +} + +async function generateAndStorePreview(doc: PreviewSourceDocument, namespace: string | null): Promise { + let workDir: string | null = null; + try { + const sourceBytes = await getDocumentBlob(doc.id, namespace); + workDir = await mkdtemp(join(tmpdir(), 'openreader-preview-')); + const sourcePath = join(workDir, 'source'); + await writeFile(sourcePath, sourceBytes); + + let rendered; + if (doc.type === 'pdf') { + rendered = await renderPdfFirstPageToJpeg(sourceBytes, DOCUMENT_PREVIEW_WIDTH); + } else if (doc.type === 'epub') { + rendered = await renderEpubCoverToJpeg(sourceBytes, DOCUMENT_PREVIEW_WIDTH); + } else { + throw new Error(`Unsupported preview type: ${doc.type}`); + } + + await putDocumentPreviewBuffer(doc.id, rendered.bytes, namespace); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +function pendingResult(status: PreviewStatus, lastError: string | null): EnsureDocumentPreviewResult { + return { + state: 'pending', + status: status === 'ready' ? 'processing' : status, + retryAfterMs: status === 'failed' ? FAILED_RETRY_AFTER_MS : RETRY_AFTER_MS, + lastError, + }; +} + +export async function enqueueDocumentPreview(doc: PreviewSourceDocument, namespace: string | null): Promise { + if (!isPreviewableDocumentType(doc.type)) return; + const namespaceKey = toNamespaceKey(namespace); + await ensurePreviewRowExists(doc, namespaceKey, namespace); + const row = await getPreviewRow(doc.id, namespaceKey); + if (!row || needsRequeue(row, doc, namespace) || row.status === 'failed') { + await markPreviewRowQueued(doc, namespaceKey, namespace); + } +} + +export async function ensureDocumentPreview(doc: PreviewSourceDocument, namespace: string | null): Promise { + if (!isPreviewableDocumentType(doc.type)) { + return pendingResult('failed', `Unsupported preview type: ${doc.type}`); + } + + const namespaceKey = toNamespaceKey(namespace); + await ensurePreviewRowExists(doc, namespaceKey, namespace); + + let row = await getPreviewRow(doc.id, namespaceKey); + if (!row) { + return pendingResult('queued', null); + } + + if (needsRequeue(row, doc, namespace)) { + await markPreviewRowQueued(doc, namespaceKey, namespace); + row = await getPreviewRow(doc.id, namespaceKey); + if (!row) return pendingResult('queued', null); + } + + if (row.status === 'ready') { + const missing = await isReadyBlobMissing(doc.id, namespace); + if (!missing) { + return { + state: 'ready', + status: 'ready', + contentType: row.contentType || DOCUMENT_PREVIEW_CONTENT_TYPE, + width: row.width || DOCUMENT_PREVIEW_WIDTH, + height: row.height, + byteSize: row.byteSize, + eTag: row.eTag, + }; + } + await markPreviewRowQueued(doc, namespaceKey, namespace); + row = await getPreviewRow(doc.id, namespaceKey); + if (!row) return pendingResult('queued', null); + } + + const now = nowMs(); + if (row.status === 'processing' && row.leaseUntilMs > now) { + return pendingResult('processing', row.lastError); + } + + const owner = `req-${randomUUID()}`; + const claimed = await tryClaimPreviewLease(doc.id, namespaceKey, owner); + if (claimed) { + try { + await generateAndStorePreview(doc, namespace); + const head = await headDocumentPreview(doc.id, namespace); + await markPreviewReady(doc, namespaceKey, { + width: DOCUMENT_PREVIEW_WIDTH, + height: null, + byteSize: head.contentLength, + eTag: head.eTag, + }); + } catch (error) { + await markPreviewFailed(doc.id, namespaceKey, error); + } + } + + row = await getPreviewRow(doc.id, namespaceKey); + if (!row) return pendingResult('queued', null); + + if (row.status === 'ready') { + const missing = await isReadyBlobMissing(doc.id, namespace); + if (!missing) { + return { + state: 'ready', + status: 'ready', + contentType: row.contentType || DOCUMENT_PREVIEW_CONTENT_TYPE, + width: row.width || DOCUMENT_PREVIEW_WIDTH, + height: row.height, + byteSize: row.byteSize, + eTag: row.eTag, + }; + } + await markPreviewRowQueued(doc, namespaceKey, namespace); + return pendingResult('queued', null); + } + + return pendingResult(row.status, row.lastError); +} + +export async function deleteDocumentPreviewRows(documentId: string, namespace: string | null): Promise { + const namespaceKey = toNamespaceKey(namespace); + await safeDb() + .delete(documentPreviews) + .where(and(eq(documentPreviews.documentId, documentId), eq(documentPreviews.namespace, namespaceKey))); +} + +export async function cleanupDocumentPreviewArtifacts(documentId: string, namespace: string | null): Promise { + await deleteDocumentPreviewArtifacts(documentId, namespace); +} diff --git a/src/lib/server/documents-blobstore.ts b/src/lib/server/documents-blobstore.ts index dad8092..dedfe29 100644 --- a/src/lib/server/documents-blobstore.ts +++ b/src/lib/server/documents-blobstore.ts @@ -146,6 +146,24 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro return bodyToBuffer(res.Body); } +export async function presignGet( + id: string, + namespace: string | null, + options?: { expiresInSeconds?: number }, +): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + return getSignedUrl( + client, + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + }), + { expiresIn: Math.max(30, Math.min(options?.expiresInSeconds ?? 300, 3600)) }, + ); +} + export async function putDocumentBlob( id: string, body: Buffer, diff --git a/tests/unit/document-previews-render.spec.ts b/tests/unit/document-previews-render.spec.ts new file mode 100644 index 0000000..4e815d8 --- /dev/null +++ b/tests/unit/document-previews-render.spec.ts @@ -0,0 +1,29 @@ +import { test, expect } from '@playwright/test'; +import path from 'path'; +import { readFile } from 'fs/promises'; +import { + renderEpubCoverToJpeg, + renderPdfFirstPageToJpeg, +} from '../../src/lib/server/document-previews-render'; + +test.describe('document-previews-render', () => { + test('renders first PDF page to JPEG preview', async () => { + const pdfPath = path.join(process.cwd(), 'tests/files/sample.pdf'); + const bytes = await readFile(pdfPath); + const rendered = await renderPdfFirstPageToJpeg(bytes, 240); + + expect(rendered.bytes.byteLength).toBeGreaterThan(1024); + expect(rendered.width).toBe(240); + expect(rendered.height).toBeGreaterThan(0); + }); + + test('extracts EPUB cover and renders to JPEG preview', async () => { + const epubPath = path.join(process.cwd(), 'tests/files/sample.epub'); + const bytes = await readFile(epubPath); + const rendered = await renderEpubCoverToJpeg(bytes, 240); + + expect(rendered.bytes.byteLength).toBeGreaterThan(1024); + expect(rendered.width).toBe(240); + expect(rendered.height).toBeGreaterThan(0); + }); +});