danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(previews): implement document preview generation and caching

Browse source 
- Add document preview caching logic with in-memory and persisted storage.
- Implement S3 blobstore functions for managing document previews.
- Create rendering functions for PDF and EPUB cover images to JPEG format.
- Introduce database schema and functions for managing document preview metadata.
- Add unit tests for rendering PDF and EPUB previews.
This commit is contained in:
Richard R 2026-02-12 16:05:03 -07:00
parent 82f82a990e
commit 9f25e05cce
37 changed files with 3005 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs-site/docs/configure/database.md
View file

@ -16,6 +16,7 @@ This page covers database mode selection for OpenReader WebUI.
- TTS character usage counters (`user_tts_chars`) for daily rate limiting (when enabled).
- User settings preferences (`user_preferences`) when auth is enabled.
- User reading progress (`user_document_progress`) when auth is enabled.
- Document preview job/asset metadata (`document_previews`) for server-side PDF/EPUB thumbnails.
## Related variables

7
docs-site/docs/configure/object-blob-storage.md
View file

@ -28,6 +28,13 @@ Storage variables are documented in [Environment Variables](../reference/environ
- Primary path: browser uploads to presigned URL from `/api/documents/blob/upload/presign`.
- Fallback path: `/api/documents/blob/upload/fallback` when direct upload fails/unreachable.
- Read/download path: blob/content serving route `/api/documents/blob` (not the upload fallback route).
- Preview path: `/api/documents/blob/preview` (returns `202` while a preview is generating; serves/redirects when ready).
## Document previews
- PDF/EPUB previews are generated server-side and stored in object storage under `document_previews_v1`.
- Preview generation is triggered on upload registration and also backfills on first preview request for older docs.
- Preview artifacts are temporary-cache friendly and can be regenerated from the source document blob.
## FS / Volume Mounts

43
drizzle/postgres/0000_perfect_risque.sql → drizzle/postgres/0000_solid_colossus.sql
View file

@ -38,6 +38,27 @@ CREATE TABLE "audiobooks" (
CONSTRAINT "audiobooks_id_user_id_pk" PRIMARY KEY("id","user_id")
);
--> statement-breakpoint
CREATE TABLE "document_previews" (
"document_id" text NOT NULL,
"namespace" text DEFAULT '' NOT NULL,
"variant" text DEFAULT 'card-240-jpeg' NOT NULL,
"status" text DEFAULT 'queued' NOT NULL,
"source_last_modified_ms" bigint NOT NULL,
"object_key" text NOT NULL,
"content_type" text DEFAULT 'image/jpeg' NOT NULL,
"width" integer DEFAULT 240 NOT NULL,
"height" integer,
"byte_size" bigint,
"etag" text,
"lease_owner" text,
"lease_until_ms" bigint DEFAULT 0 NOT NULL,
"attempt_count" integer DEFAULT 0 NOT NULL,
"last_error" text,
"created_at_ms" bigint DEFAULT 0 NOT NULL,
"updated_at_ms" bigint DEFAULT 0 NOT NULL,
CONSTRAINT "document_previews_document_id_namespace_variant_pk" PRIMARY KEY("document_id","namespace","variant")
);
--> statement-breakpoint
CREATE TABLE "documents" (
"id" text NOT NULL,
"user_id" text NOT NULL,
@ -74,6 +95,26 @@ CREATE TABLE "user" (
CONSTRAINT "user_email_unique" UNIQUE("email")
);
--> statement-breakpoint
CREATE TABLE "user_document_progress" (
"user_id" text NOT NULL,
"document_id" text NOT NULL,
"reader_type" text NOT NULL,
"location" text NOT NULL,
"progress" real,
"client_updated_at_ms" bigint DEFAULT 0 NOT NULL,
"created_at" timestamp DEFAULT now(),
"updated_at" timestamp DEFAULT now(),
CONSTRAINT "user_document_progress_user_id_document_id_pk" PRIMARY KEY("user_id","document_id")
);
--> statement-breakpoint
CREATE TABLE "user_preferences" (
"user_id" text PRIMARY KEY NOT NULL,
"data_json" jsonb DEFAULT '{}'::jsonb NOT NULL,
"client_updated_at_ms" bigint DEFAULT 0 NOT NULL,
"created_at" timestamp DEFAULT now(),
"updated_at" timestamp DEFAULT now()
);
--> statement-breakpoint
CREATE TABLE "user_tts_chars" (
"user_id" text NOT NULL,
"date" date NOT NULL,
@ -92,6 +133,8 @@ CREATE TABLE "verification" (
--> statement-breakpoint
ALTER TABLE "account" ADD CONSTRAINT "account_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "session" ADD CONSTRAINT "session_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "idx_document_previews_status_lease" ON "document_previews" USING btree ("status","lease_until_ms");--> statement-breakpoint
CREATE INDEX "idx_documents_user_id" ON "documents" USING btree ("user_id");--> statement-breakpoint
CREATE INDEX "idx_documents_user_id_last_modified" ON "documents" USING btree ("user_id","last_modified");--> statement-breakpoint
CREATE INDEX "idx_user_document_progress_user_id_updated_at" ON "user_document_progress" USING btree ("user_id","updated_at");--> statement-breakpoint
CREATE INDEX "idx_user_tts_chars_date" ON "user_tts_chars" USING btree ("date");

21
drizzle/postgres/0001_user_state_minimal.sql
View file

@ -1,21 +0,0 @@
CREATE TABLE "user_document_progress" (
"user_id" text NOT NULL,
"document_id" text NOT NULL,
"reader_type" text NOT NULL,
"location" text NOT NULL,
"progress" real,
"client_updated_at_ms" bigint DEFAULT 0 NOT NULL,
"created_at" timestamp DEFAULT now(),
"updated_at" timestamp DEFAULT now(),
CONSTRAINT "user_document_progress_user_id_document_id_pk" PRIMARY KEY("user_id","document_id")
);
--> statement-breakpoint
CREATE TABLE "user_preferences" (
"user_id" text PRIMARY KEY NOT NULL,
"data_json" jsonb DEFAULT '{}'::jsonb NOT NULL,
"client_updated_at_ms" bigint DEFAULT 0 NOT NULL,
"created_at" timestamp DEFAULT now(),
"updated_at" timestamp DEFAULT now()
);
--> statement-breakpoint
CREATE INDEX "idx_user_document_progress_user_id_updated_at" ON "user_document_progress" USING btree ("user_id","updated_at");

298
drizzle/postgres/meta/0000_snapshot.json
View file

@ -1,5 +1,5 @@
{
"id": "03395f16-5c34-409a-9dc1-69329d67bcd9",
"id": "161f4dc9-0c06-4d77-87e5-3496ffad1b97",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
@ -250,6 +250,161 @@
"checkConstraints": {},
"isRLSEnabled": false
},
"public.document_previews": {
"name": "document_previews",
"schema": "",
"columns": {
"document_id": {
"name": "document_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"namespace": {
"name": "namespace",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "''"
},
"variant": {
"name": "variant",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'card-240-jpeg'"
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'queued'"
},
"source_last_modified_ms": {
"name": "source_last_modified_ms",
"type": "bigint",
"primaryKey": false,
"notNull": true
},
"object_key": {
"name": "object_key",
"type": "text",
"primaryKey": false,
"notNull": true
},
"content_type": {
"name": "content_type",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'image/jpeg'"
},
"width": {
"name": "width",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 240
},
"height": {
"name": "height",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"byte_size": {
"name": "byte_size",
"type": "bigint",
"primaryKey": false,
"notNull": false
},
"etag": {
"name": "etag",
"type": "text",
"primaryKey": false,
"notNull": false
},
"lease_owner": {
"name": "lease_owner",
"type": "text",
"primaryKey": false,
"notNull": false
},
"lease_until_ms": {
"name": "lease_until_ms",
"type": "bigint",
"primaryKey": false,
"notNull": true,
"default": 0
},
"attempt_count": {
"name": "attempt_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 0
},
"last_error": {
"name": "last_error",
"type": "text",
"primaryKey": false,
"notNull": false
},
"created_at_ms": {
"name": "created_at_ms",
"type": "bigint",
"primaryKey": false,
"notNull": true,
"default": 0
},
"updated_at_ms": {
"name": "updated_at_ms",
"type": "bigint",
"primaryKey": false,
"notNull": true,
"default": 0
}
},
"indexes": {
"idx_document_previews_status_lease": {
"name": "idx_document_previews_status_lease",
"columns": [
{
"expression": "status",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "lease_until_ms",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {
"document_previews_document_id_namespace_variant_pk": {
"name": "document_previews_document_id_namespace_variant_pk",
"columns": [
"document_id",
"namespace",
"variant"
]
}
},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.documents": {
"name": "documents",
"schema": "",
@ -509,6 +664,147 @@
"checkConstraints": {},
"isRLSEnabled": false
},
"public.user_document_progress": {
"name": "user_document_progress",
"schema": "",
"columns": {
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"document_id": {
"name": "document_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"reader_type": {
"name": "reader_type",
"type": "text",
"primaryKey": false,
"notNull": true
},
"location": {
"name": "location",
"type": "text",
"primaryKey": false,
"notNull": true
},
"progress": {
"name": "progress",
"type": "real",
"primaryKey": false,
"notNull": false
},
"client_updated_at_ms": {
"name": "client_updated_at_ms",
"type": "bigint",
"primaryKey": false,
"notNull": true,
"default": 0
},
"created_at": {
"name": "created_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
},
"updated_at": {
"name": "updated_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {
"idx_user_document_progress_user_id_updated_at": {
"name": "idx_user_document_progress_user_id_updated_at",
"columns": [
{
"expression": "user_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "updated_at",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {
"user_document_progress_user_id_document_id_pk": {
"name": "user_document_progress_user_id_document_id_pk",
"columns": [
"user_id",
"document_id"
]
}
},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.user_preferences": {
"name": "user_preferences",
"schema": "",
"columns": {
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"data_json": {
"name": "data_json",
"type": "jsonb",
"primaryKey": false,
"notNull": true,
"default": "'{}'::jsonb"
},
"client_updated_at_ms": {
"name": "client_updated_at_ms",
"type": "bigint",
"primaryKey": false,
"notNull": true,
"default": 0
},
"created_at": {
"name": "created_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
},
"updated_at": {
"name": "updated_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.user_tts_chars": {
"name": "user_tts_chars",
"schema": "",

13
drizzle/postgres/meta/_journal.json
View file

@ -5,16 +5,9 @@
{
"idx": 0,
"version": "7",
"when": 1770191341206,
"tag": "0000_perfect_risque",
"breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1770843600000,
"tag": "0001_user_state_minimal",
"when": 1770846273728,
"tag": "0000_solid_colossus",
"breakpoints": true
}
]
}
}

43
drizzle/sqlite/0000_lively_korvac.sql → drizzle/sqlite/0000_cool_gwen_stacy.sql
View file

@ -39,6 +39,28 @@ CREATE TABLE `audiobooks` (
PRIMARY KEY(`id`, `user_id`)
);
--> statement-breakpoint
CREATE TABLE `document_previews` (
`document_id` text NOT NULL,
`namespace` text DEFAULT '' NOT NULL,
`variant` text DEFAULT 'card-240-jpeg' NOT NULL,
`status` text DEFAULT 'queued' NOT NULL,
`source_last_modified_ms` integer NOT NULL,
`object_key` text NOT NULL,
`content_type` text DEFAULT 'image/jpeg' NOT NULL,
`width` integer DEFAULT 240 NOT NULL,
`height` integer,
`byte_size` integer,
`etag` text,
`lease_owner` text,
`lease_until_ms` integer DEFAULT 0 NOT NULL,
`attempt_count` integer DEFAULT 0 NOT NULL,
`last_error` text,
`created_at_ms` integer DEFAULT 0 NOT NULL,
`updated_at_ms` integer DEFAULT 0 NOT NULL,
PRIMARY KEY(`document_id`, `namespace`, `variant`)
);
--> statement-breakpoint
CREATE INDEX `idx_document_previews_status_lease` ON `document_previews` (`status`,`lease_until_ms`);--> statement-breakpoint
CREATE TABLE `documents` (
`id` text NOT NULL,
`user_id` text NOT NULL,
@ -78,6 +100,27 @@ CREATE TABLE `user` (
);
--> statement-breakpoint
CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint
CREATE TABLE `user_document_progress` (
`user_id` text NOT NULL,
`document_id` text NOT NULL,
`reader_type` text NOT NULL,
`location` text NOT NULL,
`progress` real,
`client_updated_at_ms` integer DEFAULT 0 NOT NULL,
`created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000),
`updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000),
PRIMARY KEY(`user_id`, `document_id`)
);
--> statement-breakpoint
CREATE INDEX `idx_user_document_progress_user_id_updated_at` ON `user_document_progress` (`user_id`,`updated_at`);--> statement-breakpoint
CREATE TABLE `user_preferences` (
`user_id` text PRIMARY KEY NOT NULL,
`data_json` text DEFAULT '{}' NOT NULL,
`client_updated_at_ms` integer DEFAULT 0 NOT NULL,
`created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000),
`updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000)
);
--> statement-breakpoint
CREATE TABLE `user_tts_chars` (
`user_id` text NOT NULL,
`date` text NOT NULL,

21
drizzle/sqlite/0001_user_state_minimal.sql
View file

@ -1,21 +0,0 @@
CREATE TABLE `user_document_progress` (
`user_id` text NOT NULL,
`document_id` text NOT NULL,
`reader_type` text NOT NULL,
`location` text NOT NULL,
`progress` real,
`client_updated_at_ms` integer DEFAULT 0 NOT NULL,
`created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000),
`updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000),
PRIMARY KEY(`user_id`, `document_id`)
);
--> statement-breakpoint
CREATE TABLE `user_preferences` (
`user_id` text PRIMARY KEY NOT NULL,
`data_json` text DEFAULT '{}' NOT NULL,
`client_updated_at_ms` integer DEFAULT 0 NOT NULL,
`created_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000),
`updated_at` integer DEFAULT (cast(strftime('%s','now') as int) * 1000)
);
--> statement-breakpoint
CREATE INDEX `idx_user_document_progress_user_id_updated_at` ON `user_document_progress` (`user_id`,`updated_at`);

293
drizzle/sqlite/meta/0000_snapshot.json
View file

@ -1,7 +1,7 @@
{
"version": "6",
"dialect": "sqlite",
"id": "9c9e61fc-1f90-463a-99dc-232397f316e7",
"id": "eb884adf-c9c7-494b-b0a1-3601f63bcbbe",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"account": {
@ -270,6 +270,162 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"document_previews": {
"name": "document_previews",
"columns": {
"document_id": {
"name": "document_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"namespace": {
"name": "namespace",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "''"
},
"variant": {
"name": "variant",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'card-240-jpeg'"
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'queued'"
},
"source_last_modified_ms": {
"name": "source_last_modified_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"object_key": {
"name": "object_key",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"content_type": {
"name": "content_type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'image/jpeg'"
},
"width": {
"name": "width",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 240
},
"height": {
"name": "height",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"byte_size": {
"name": "byte_size",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"etag": {
"name": "etag",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"lease_owner": {
"name": "lease_owner",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"lease_until_ms": {
"name": "lease_until_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"attempt_count": {
"name": "attempt_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"last_error": {
"name": "last_error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at_ms": {
"name": "created_at_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"updated_at_ms": {
"name": "updated_at_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
}
},
"indexes": {
"idx_document_previews_status_lease": {
"name": "idx_document_previews_status_lease",
"columns": [
"status",
"lease_until_ms"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {
"document_previews_document_id_namespace_variant_pk": {
"columns": [
"document_id",
"namespace",
"variant"
],
"name": "document_previews_document_id_namespace_variant_pk"
}
},
"uniqueConstraints": {},
"checkConstraints": {}
},
"documents": {
"name": "documents",
"columns": {
@ -523,6 +679,141 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"user_document_progress": {
"name": "user_document_progress",
"columns": {
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"document_id": {
"name": "document_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"reader_type": {
"name": "reader_type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"location": {
"name": "location",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"progress": {
"name": "progress",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"client_updated_at_ms": {
"name": "client_updated_at_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(cast(strftime('%s','now') as int) * 1000)"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(cast(strftime('%s','now') as int) * 1000)"
}
},
"indexes": {
"idx_user_document_progress_user_id_updated_at": {
"name": "idx_user_document_progress_user_id_updated_at",
"columns": [
"user_id",
"updated_at"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {
"user_document_progress_user_id_document_id_pk": {
"columns": [
"user_id",
"document_id"
],
"name": "user_document_progress_user_id_document_id_pk"
}
},
"uniqueConstraints": {},
"checkConstraints": {}
},
"user_preferences": {
"name": "user_preferences",
"columns": {
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"data_json": {
"name": "data_json",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{}'"
},
"client_updated_at_ms": {
"name": "client_updated_at_ms",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(cast(strftime('%s','now') as int) * 1000)"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "(cast(strftime('%s','now') as int) * 1000)"
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"user_tts_chars": {
"name": "user_tts_chars",
"columns": {

13
drizzle/sqlite/meta/_journal.json
View file

@ -5,16 +5,9 @@
{
"idx": 0,
"version": "6",
"when": 1770191340954,
"tag": "0000_lively_korvac",
"breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1770843600000,
"tag": "0001_user_state_minimal",
"when": 1770846273460,
"tag": "0000_cool_gwen_stacy",
"breakpoints": true
}
]
}
}

4
next.config.ts
View file

@ -3,10 +3,10 @@ import type { NextConfig } from "next";
const nextConfig: NextConfig = {
turbopack: {
resolveAlias: {
canvas: './empty-module.ts',
canvas: '@napi-rs/canvas',
},
},
serverExternalPackages: ["better-sqlite3", "ffmpeg-static", "ffprobe-static"],
serverExternalPackages: ["@napi-rs/canvas", "better-sqlite3", "ffmpeg-static", "ffprobe-static"],
outputFileTracingIncludes: {
'/api/audiobook': [
'./node_modules/ffmpeg-static/ffmpeg',

6
package.json
View file

@ -25,6 +25,7 @@
"@aws-sdk/client-s3": "^3.987.0",
"@aws-sdk/s3-request-presigner": "^3.987.0",
"@headlessui/react": "^2.2.9",
"@napi-rs/canvas": "^0.1.91",
"@vercel/analytics": "^1.6.1",
"better-auth": "^1.4.18",
"better-sqlite3": "^12.6.2",
@ -37,9 +38,11 @@
"drizzle-kit": "^0.31.9",
"drizzle-orm": "^0.45.1",
"epubjs": "^0.3.93",
"fast-xml-parser": "^5.3.5",
"ffmpeg-static": "^5.3.0",
"ffprobe-static": "^3.1.0",
"howler": "^2.2.4",
"jszip": "^3.10.1",
"lru-cache": "^11.2.6",
"next": "^15.5.12",
"openai": "^6.21.0",
@ -63,11 +66,11 @@
"@tailwindcss/typography": "^0.5.19",
"@types/better-sqlite3": "^7.6.13",
"@types/ffprobe-static": "^2.0.3",
"@types/howler": "^2.2.12",
"@types/node": "^20.19.33",
"@types/pg": "^8.16.0",
"@types/react": "^19.2.13",
"@types/react-dom": "^19.2.3",
"@types/howler": "^2.2.12",
"@types/uuid": "^10.0.0",
"eslint": "^9.39.2",
"eslint-config-next": "^15.5.12",
@ -77,6 +80,7 @@
},
"pnpm": {
"onlyBuiltDependencies": [
"@napi-rs/canvas",
"better-sqlite3",
"ffmpeg-static"
],

134
pnpm-lock.yaml
View file

@ -22,6 +22,9 @@ importers:
'@headlessui/react':
specifier: ^2.2.9
version: 2.2.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4)
'@napi-rs/canvas':
specifier: ^0.1.91
version: 0.1.91
'@vercel/analytics':
specifier: ^1.6.1
version: 1.6.1(next@15.5.12(@playwright/test@1.58.2)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)
@ -58,6 +61,9 @@ importers:
epubjs:
specifier: ^0.3.93
version: 0.3.93
fast-xml-parser:
specifier: ^5.3.5
version: 5.3.5
ffmpeg-static:
specifier: ^5.3.0
version: 5.3.0
@ -67,6 +73,9 @@ importers:
howler:
specifier: ^2.2.4
version: 2.2.4
jszip:
specifier: ^3.10.1
version: 3.10.1
lru-cache:
specifier: ^11.2.6
version: 11.2.6
@ -915,6 +924,76 @@ packages:
'@jridgewell/trace-mapping@0.3.31':
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
'@napi-rs/canvas-android-arm64@0.1.91':
resolution: {integrity: sha512-SLLzXXgSnfct4zy/BVAfweZQkYkPJsNsJ2e5DOE8DFEHC6PufyUrwb12yqeu2So2IOIDpWJJaDAxKY/xpy6MYQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [android]
'@napi-rs/canvas-darwin-arm64@0.1.91':
resolution: {integrity: sha512-bzdbCjIjw3iRuVFL+uxdSoMra/l09ydGNX9gsBxO/zg+5nlppscIpj6gg+nL6VNG85zwUarDleIrUJ+FWHvmuA==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [darwin]
'@napi-rs/canvas-darwin-x64@0.1.91':
resolution: {integrity: sha512-q3qpkpw0IsG9fAS/dmcGIhCVoNxj8ojbexZKWwz3HwxlEWsLncEQRl4arnxrwbpLc2nTNTyj4WwDn7QR5NDAaA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
'@napi-rs/canvas-linux-arm-gnueabihf@0.1.91':
resolution: {integrity: sha512-Io3g8wJZVhK8G+Fpg1363BE90pIPqg+ZbeehYNxPWDSzbgwU3xV0l8r/JBzODwC7XHi1RpFEk+xyUTMa2POj6w==}
engines: {node: '>= 10'}
cpu: [arm]
os: [linux]
'@napi-rs/canvas-linux-arm64-gnu@0.1.91':
resolution: {integrity: sha512-HBnto+0rxx1bQSl8bCWA9PyBKtlk2z/AI32r3cu4kcNO+M/5SD4b0v1MWBWZyqMQyxFjWgy3ECyDjDKMC6tY1A==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@napi-rs/canvas-linux-arm64-musl@0.1.91':
resolution: {integrity: sha512-/eJtVe2Xw9A86I4kwXpxxoNagdGclu12/NSMsfoL8q05QmeRCbfjhg1PJS7ENAuAvaiUiALGrbVfeY1KU1gztQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@napi-rs/canvas-linux-riscv64-gnu@0.1.91':
resolution: {integrity: sha512-floNK9wQuRWevUhhXRcuis7h0zirdytVxPgkonWO+kQlbvxV7gEUHGUFQyq4n55UHYFwgck1SAfJ1HuXv/+ppQ==}
engines: {node: '>= 10'}
cpu: [riscv64]
os: [linux]
'@napi-rs/canvas-linux-x64-gnu@0.1.91':
resolution: {integrity: sha512-c3YDqBdf7KETuZy2AxsHFMsBBX1dWT43yFfWUq+j1IELdgesWtxf/6N7csi3VPf6VA3PmnT9EhMyb+M1wfGtqw==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@napi-rs/canvas-linux-x64-musl@0.1.91':
resolution: {integrity: sha512-RpZ3RPIwgEcNBHSHSX98adm+4VP8SMT5FN6250s5jQbWpX/XNUX5aLMfAVJS/YnDjS1QlsCgQxFOPU0aCCWgag==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@napi-rs/canvas-win32-arm64-msvc@0.1.91':
resolution: {integrity: sha512-gF8MBp4X134AgVurxqlCdDA2qO0WaDdi9o6Sd5rWRVXRhWhYQ6wkdEzXNLIrmmros0Tsp2J0hQzx4ej/9O8trQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [win32]
'@napi-rs/canvas-win32-x64-msvc@0.1.91':
resolution: {integrity: sha512-++gtW9EV/neKI8TshD8WFxzBYALSPag2kFRahIJV+LYsyt5kBn21b1dBhEUDHf7O+wiZmuFCeUa7QKGHnYRZBA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
'@napi-rs/canvas@0.1.91':
resolution: {integrity: sha512-eeIe1GoB74P1B0Nkw6pV8BCQ3hfCfvyYr4BntzlCsnFXzVJiPMDnLeIx3gVB0xQMblHYnjK/0nCLvirEhOjr5g==}
engines: {node: '>= 10'}
'@napi-rs/wasm-runtime@0.2.12':
resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==}
@ -2307,6 +2386,10 @@ packages:
resolution: {integrity: sha512-EFd6afGmXlCx8H8WTZHhAoDaWaGyuIBoZJ2mknrNxug+aZKjkp0a0dlars9Izl+jF+7Gu1/5f/2h68cQpe0IiA==}
hasBin: true
fast-xml-parser@5.3.5:
resolution: {integrity: sha512-JeaA2Vm9ffQKp9VjvfzObuMCjUYAp5WDYhRYL5LrBPY/jUDlUtOvDfot0vKSkB9tuX885BDHjtw4fZadD95wnA==}
hasBin: true
fastq@1.20.1:
resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==}
@ -4675,6 +4758,53 @@ snapshots:
'@jridgewell/resolve-uri': 3.1.2
'@jridgewell/sourcemap-codec': 1.5.5
'@napi-rs/canvas-android-arm64@0.1.91':
optional: true
'@napi-rs/canvas-darwin-arm64@0.1.91':
optional: true
'@napi-rs/canvas-darwin-x64@0.1.91':
optional: true
'@napi-rs/canvas-linux-arm-gnueabihf@0.1.91':
optional: true
'@napi-rs/canvas-linux-arm64-gnu@0.1.91':
optional: true
'@napi-rs/canvas-linux-arm64-musl@0.1.91':
optional: true
'@napi-rs/canvas-linux-riscv64-gnu@0.1.91':
optional: true
'@napi-rs/canvas-linux-x64-gnu@0.1.91':
optional: true
'@napi-rs/canvas-linux-x64-musl@0.1.91':
optional: true
'@napi-rs/canvas-win32-arm64-msvc@0.1.91':
optional: true
'@napi-rs/canvas-win32-x64-msvc@0.1.91':
optional: true
'@napi-rs/canvas@0.1.91':
optionalDependencies:
'@napi-rs/canvas-android-arm64': 0.1.91
'@napi-rs/canvas-darwin-arm64': 0.1.91
'@napi-rs/canvas-darwin-x64': 0.1.91
'@napi-rs/canvas-linux-arm-gnueabihf': 0.1.91
'@napi-rs/canvas-linux-arm64-gnu': 0.1.91
'@napi-rs/canvas-linux-arm64-musl': 0.1.91
'@napi-rs/canvas-linux-riscv64-gnu': 0.1.91
'@napi-rs/canvas-linux-x64-gnu': 0.1.91
'@napi-rs/canvas-linux-x64-musl': 0.1.91
'@napi-rs/canvas-win32-arm64-msvc': 0.1.91
'@napi-rs/canvas-win32-x64-msvc': 0.1.91
'@napi-rs/wasm-runtime@0.2.12':
dependencies:
'@emnapi/core': 1.8.1
@ -6246,6 +6376,10 @@ snapshots:
dependencies:
strnum: 2.1.2
fast-xml-parser@5.3.5:
dependencies:
strnum: 2.1.2
fastq@1.20.1:
dependencies:
reusify: 1.1.0

85
src/app/api/documents/blob/get/fallback/route.ts Normal file
View file

@ -0,0 +1,85 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { contentTypeForName } from '@/lib/server/library';
import { getDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
function streamBuffer(buffer: Buffer): ReadableStream<Uint8Array> {
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buffer));
controller.close();
},
});
}
export async function GET(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
const rows = (await db
.select({ id: documents.id, userId: documents.userId, name: documents.name })
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
name: string;
}>;
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
if (!doc) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
const filename = doc.name || `${id}.bin`;
const responseType = contentTypeForName(filename);
try {
const content = await getDocumentBlob(id, testNamespace);
return new NextResponse(streamBuffer(content), {
headers: {
'Content-Type': responseType,
'Cache-Control': 'no-store',
},
});
} catch (error) {
if (isMissingBlobError(error)) {
await db
.delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
throw error;
}
} catch (error) {
console.error('Error loading document content fallback:', error);
return NextResponse.json({ error: 'Failed to load document content' }, { status: 500 });
}
}

67
src/app/api/documents/blob/get/presign/route.ts Normal file
View file

@ -0,0 +1,67 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { isValidDocumentId, presignGet } from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
export async function GET(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
const rows = (await db
.select({ id: documents.id, userId: documents.userId })
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
}>;
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
if (!doc) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
const fallbackUrl = `/api/documents/blob/get/fallback?id=${encodeURIComponent(doc.id)}`;
const directUrl = await presignGet(doc.id, testNamespace).catch(() => null);
if (!directUrl) {
return NextResponse.redirect(fallbackUrl, {
status: 307,
headers: { 'Cache-Control': 'no-store' },
});
}
return NextResponse.redirect(directUrl, {
status: 307,
headers: { 'Cache-Control': 'no-store' },
});
} catch (error) {
console.error('Error creating document download signature:', error);
return NextResponse.json({ error: 'Failed to prepare document download' }, { status: 500 });
}
}

105
src/app/api/documents/blob/preview/ensure/route.ts Normal file
View file

@ -0,0 +1,105 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { isValidDocumentId } from '@/lib/server/documents-blobstore';
import { presignDocumentPreviewGet } from '@/lib/server/document-previews-blobstore';
import { ensureDocumentPreview, isPreviewableDocumentType } from '@/lib/server/document-previews';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
export async function GET(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
const rows = (await db
.select({
id: documents.id,
userId: documents.userId,
type: documents.type,
lastModified: documents.lastModified,
})
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
type: string;
lastModified: number;
}>;
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
if (!doc) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
if (!isPreviewableDocumentType(doc.type)) {
return NextResponse.json({ error: `Preview not supported for type ${doc.type}` }, { status: 415 });
}
const presignUrl = `/api/documents/blob/preview/presign?id=${encodeURIComponent(id)}`;
const fallbackUrl = `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`;
const preview = await ensureDocumentPreview(
{
id: doc.id,
type: doc.type,
lastModified: Number(doc.lastModified),
},
testNamespace,
);
if (preview.state !== 'ready') {
return NextResponse.json(
{
status: preview.status,
retryAfterMs: preview.retryAfterMs,
presignUrl,
fallbackUrl,
},
{
status: 202,
headers: { 'Cache-Control': 'no-store' },
},
);
}
const directUrl = await presignDocumentPreviewGet(doc.id, testNamespace).catch(() => null);
return NextResponse.json(
{
status: 'ready',
presignUrl,
fallbackUrl,
...(directUrl ? { directUrl } : {}),
},
{
headers: { 'Cache-Control': 'no-store' },
},
);
} catch (error) {
console.error('Error ensuring document preview:', error);
return NextResponse.json({ error: 'Failed to ensure document preview' }, { status: 500 });
}
}

143
src/app/api/documents/blob/preview/fallback/route.ts Normal file
View file

@ -0,0 +1,143 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { isValidDocumentId } from '@/lib/server/documents-blobstore';
import {
getDocumentPreviewBuffer,
isMissingBlobError,
} from '@/lib/server/document-previews-blobstore';
import {
ensureDocumentPreview,
enqueueDocumentPreview,
isPreviewableDocumentType,
} from '@/lib/server/document-previews';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
function streamBuffer(buffer: Buffer): ReadableStream<Uint8Array> {
return new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(buffer));
controller.close();
},
});
}
export async function GET(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
const presignUrl = `/api/documents/blob/preview/presign?id=${encodeURIComponent(id)}`;
const fallbackUrl = `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`;
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
const rows = (await db
.select({
id: documents.id,
userId: documents.userId,
type: documents.type,
lastModified: documents.lastModified,
})
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
type: string;
lastModified: number;
}>;
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
if (!doc) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
if (!isPreviewableDocumentType(doc.type)) {
return NextResponse.json({ error: `Preview not supported for type ${doc.type}` }, { status: 415 });
}
const preview = await ensureDocumentPreview(
{
id: doc.id,
type: doc.type,
lastModified: Number(doc.lastModified),
},
testNamespace,
);
if (preview.state !== 'ready') {
return NextResponse.json(
{
status: preview.status,
retryAfterMs: preview.retryAfterMs,
presignUrl,
fallbackUrl,
},
{
status: 202,
headers: { 'Cache-Control': 'no-store' },
},
);
}
try {
const content = await getDocumentPreviewBuffer(doc.id, testNamespace);
return new NextResponse(streamBuffer(content), {
headers: {
'Content-Type': preview.contentType,
'Cache-Control': 'no-store',
'Content-Length': String(content.byteLength),
},
});
} catch (error) {
if (isMissingBlobError(error)) {
await enqueueDocumentPreview(
{
id: doc.id,
type: doc.type,
lastModified: Number(doc.lastModified),
},
testNamespace,
).catch(() => {});
return NextResponse.json(
{
status: 'queued',
retryAfterMs: 1500,
presignUrl,
fallbackUrl,
},
{
status: 202,
headers: { 'Cache-Control': 'no-store' },
},
);
}
throw error;
}
} catch (error) {
console.error('Error loading document preview fallback:', error);
return NextResponse.json({ error: 'Failed to load document preview' }, { status: 500 });
}
}

103
src/app/api/documents/blob/preview/presign/route.ts Normal file
View file

@ -0,0 +1,103 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { isValidDocumentId } from '@/lib/server/documents-blobstore';
import { presignDocumentPreviewGet } from '@/lib/server/document-previews-blobstore';
import { ensureDocumentPreview, isPreviewableDocumentType } from '@/lib/server/document-previews';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
export async function GET(req: NextRequest) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
const rows = (await db
.select({
id: documents.id,
userId: documents.userId,
type: documents.type,
lastModified: documents.lastModified,
})
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
type: string;
lastModified: number;
}>;
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
if (!doc) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
if (!isPreviewableDocumentType(doc.type)) {
return NextResponse.json({ error: `Preview not supported for type ${doc.type}` }, { status: 415 });
}
const fallbackUrl = `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`;
const preview = await ensureDocumentPreview(
{
id: doc.id,
type: doc.type,
lastModified: Number(doc.lastModified),
},
testNamespace,
);
if (preview.state !== 'ready') {
return NextResponse.json(
{
status: preview.status,
retryAfterMs: preview.retryAfterMs,
fallbackUrl,
},
{
status: 202,
headers: { 'Cache-Control': 'no-store' },
},
);
}
const directUrl = await presignDocumentPreviewGet(doc.id, testNamespace).catch(() => null);
if (!directUrl) {
return NextResponse.redirect(fallbackUrl, {
status: 307,
headers: { 'Cache-Control': 'no-store' },
});
}
return NextResponse.redirect(directUrl, {
status: 307,
headers: { 'Cache-Control': 'no-store' },
});
} catch (error) {
console.error('Error creating document preview signature:', error);
return NextResponse.json({ error: 'Failed to prepare document preview' }, { status: 500 });
}
}

19
src/app/api/documents/blob/route.ts
View file

@ -5,7 +5,13 @@ import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { contentTypeForName } from '@/lib/server/library';
import { extractRawTextSnippet } from '@/lib/text-snippets';
import { getDocumentBlob, getDocumentRange, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
import {
getDocumentBlob,
getDocumentRange,
isMissingBlobError,
isValidDocumentId,
presignGet,
} from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
@ -38,6 +44,7 @@ export async function GET(req: NextRequest) {
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
const format = (url.searchParams.get('format') || '').toLowerCase().trim();
const prefersProxy = (url.searchParams.get('proxy') || '').trim() === '1';
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
@ -83,6 +90,16 @@ export async function GET(req: NextRequest) {
}
try {
if (!prefersProxy) {
const directUrl = await presignGet(id, testNamespace).catch(() => null);
if (directUrl) {
return NextResponse.redirect(directUrl, {
status: 307,
headers: { 'Cache-Control': 'no-store' },
});
}
}
const content = await getDocumentBlob(id, testNamespace);
const stream = new ReadableStream<Uint8Array>({
start(controller) {

12
src/app/api/documents/docx-to-pdf/upload/route.ts
View file

@ -9,6 +9,7 @@ import { requireAuthContext } from '@/lib/server/auth';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { safeDocumentName } from '@/lib/server/documents-utils';
import { enqueueDocumentPreview } from '@/lib/server/document-previews';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
import { putDocumentBlob } from '@/lib/server/documents-blobstore';
@ -147,6 +148,17 @@ export async function POST(req: NextRequest) {
},
});
await enqueueDocumentPreview(
{
id,
type: 'pdf',
lastModified,
},
testNamespace,
).catch((error) => {
console.error(`Failed to enqueue preview for converted DOCX ${id}:`, error);
});
return NextResponse.json({
stored: {
id,

23
src/app/api/documents/route.ts
View file

@ -4,6 +4,11 @@ import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth';
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents-utils';
import {
cleanupDocumentPreviewArtifacts,
deleteDocumentPreviewRows,
enqueueDocumentPreview,
} from '@/lib/server/document-previews';
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
@ -124,6 +129,17 @@ export async function POST(req: NextRequest) {
lastModified: doc.lastModified,
scope: storageUserId === unclaimedUserId ? 'unclaimed' : 'user',
});
await enqueueDocumentPreview(
{
id: doc.id,
type: doc.type,
lastModified: doc.lastModified,
},
testNamespace,
).catch((error) => {
console.error(`Failed to enqueue preview for document ${doc.id}:`, error);
});
}
return NextResponse.json({ success: true, stored });
@ -269,6 +285,13 @@ export async function DELETE(req: NextRequest) {
throw error;
}
}
await cleanupDocumentPreviewArtifacts(id, testNamespace).catch((error) => {
console.error(`Failed to cleanup preview artifacts for document ${id}:`, error);
});
await deleteDocumentPreviewRows(id, testNamespace).catch((error) => {
console.error(`Failed to cleanup preview rows for document ${id}:`, error);
});
}
return NextResponse.json({ success: true, deleted: deletedRows.length });

30
src/app/epub/[id]/page.tsx
View file

@ -2,7 +2,7 @@
import { useParams, useRouter } from "next/navigation";
import Link from 'next/link';
import { useCallback, useEffect, useState } from 'react';
import { useCallback, useEffect, useRef, useState } from 'react';
import { useEPUB } from '@/contexts/EPUBContext';
import { DocumentSkeleton } from '@/components/DocumentSkeleton';
import { EPUBViewer } from '@/components/views/EPUBViewer';
@ -34,11 +34,20 @@ export default function EPUBPage() {
const [containerHeight, setContainerHeight] = useState<string>('auto');
const [padPct, setPadPct] = useState<number>(100); // 0..100 (100 = full width, 0 = max padding)
const [maxPadPx, setMaxPadPx] = useState<number>(0);
const inFlightDocIdRef = useRef<string | null>(null);
const loadedDocIdRef = useRef<string | null>(null);
useEffect(() => {
setIsLoading(true);
setError(null);
inFlightDocIdRef.current = null;
loadedDocIdRef.current = null;
}, [id]);
const loadDocument = useCallback(async () => {
console.log('Loading new epub (from page.tsx)');
stop(); // Reset TTS when loading new document
let didRedirect = false;
let startedLoad = false;
try {
if (!id) {
setError('Document not found');
@ -50,12 +59,27 @@ export default function EPUBPage() {
router.replace(`/epub/${resolved}`);
return;
}
if (loadedDocIdRef.current === resolved) {
return;
}
if (inFlightDocIdRef.current === resolved) {
return;
}
startedLoad = true;
inFlightDocIdRef.current = resolved;
stop(); // Reset TTS when loading new document
await setCurrentDocument(resolved);
loadedDocIdRef.current = resolved;
} catch (err) {
console.error('Error loading document:', err);
setError('Failed to load document');
} finally {
if (!didRedirect) {
if (startedLoad) {
inFlightDocIdRef.current = null;
}
if (!didRedirect && startedLoad) {
setIsLoading(false);
}
}

30
src/app/html/[id]/page.tsx
View file

@ -2,7 +2,7 @@
import { useParams, useRouter } from "next/navigation";
import Link from 'next/link';
import { useCallback, useEffect, useState } from 'react';
import { useCallback, useEffect, useRef, useState } from 'react';
import { useHTML } from '@/contexts/HTMLContext';
import { DocumentSkeleton } from '@/components/DocumentSkeleton';
import { HTMLViewer } from '@/components/views/HTMLViewer';
@ -28,12 +28,21 @@ export default function HTMLPage() {
const [containerHeight, setContainerHeight] = useState<string>('auto');
const [padPct, setPadPct] = useState<number>(100); // 0..100 (100 = full width)
const [maxPadPx, setMaxPadPx] = useState<number>(0);
const inFlightDocIdRef = useRef<string | null>(null);
const loadedDocIdRef = useRef<string | null>(null);
useEffect(() => {
setIsLoading(true);
setError(null);
inFlightDocIdRef.current = null;
loadedDocIdRef.current = null;
}, [id]);
const loadDocument = useCallback(async () => {
if (!isLoading) return;
console.log('Loading new HTML document (from page.tsx)');
stop();
let didRedirect = false;
let startedLoad = false;
try {
if (!id) {
setError('Document not found');
@ -45,12 +54,27 @@ export default function HTMLPage() {
router.replace(`/html/${resolved}`);
return;
}
if (loadedDocIdRef.current === resolved) {
return;
}
if (inFlightDocIdRef.current === resolved) {
return;
}
startedLoad = true;
inFlightDocIdRef.current = resolved;
stop();
await setCurrentDocument(resolved);
loadedDocIdRef.current = resolved;
} catch (err) {
console.error('Error loading document:', err);
setError('Failed to load document');
} finally {
if (!didRedirect) {
if (startedLoad) {
inFlightDocIdRef.current = null;
}
if (!didRedirect && startedLoad) {
setIsLoading(false);
}
}

30
src/app/pdf/[id]/page.tsx
View file

@ -4,7 +4,7 @@ import dynamic from 'next/dynamic';
import { usePDF } from '@/contexts/PDFContext';
import { useParams, useRouter } from 'next/navigation';
import Link from 'next/link';
import { useCallback, useEffect, useState } from 'react';
import { useCallback, useEffect, useRef, useState } from 'react';
import { DocumentSkeleton } from '@/components/DocumentSkeleton';
import { useTTS } from '@/contexts/TTSContext';
import { DocumentSettings } from '@/components/DocumentSettings';
@ -42,12 +42,21 @@ export default function PDFViewerPage() {
const [isSettingsOpen, setIsSettingsOpen] = useState(false);
const [isAudiobookModalOpen, setIsAudiobookModalOpen] = useState(false);
const [containerHeight, setContainerHeight] = useState<string>('auto');
const inFlightDocIdRef = useRef<string | null>(null);
const loadedDocIdRef = useRef<string | null>(null);
useEffect(() => {
setIsLoading(true);
setError(null);
inFlightDocIdRef.current = null;
loadedDocIdRef.current = null;
}, [id]);
const loadDocument = useCallback(async () => {
if (!isLoading) return; // Prevent calls when not loading new doc
console.log('Loading new document (from page.tsx)');
stop(); // Reset TTS when loading new document
let didRedirect = false;
let startedLoad = false;
try {
if (!id) {
setError('Document not found');
@ -59,12 +68,27 @@ export default function PDFViewerPage() {
router.replace(`/pdf/${resolved}`);
return;
}
if (loadedDocIdRef.current === resolved) {
return;
}
if (inFlightDocIdRef.current === resolved) {
return;
}
startedLoad = true;
inFlightDocIdRef.current = resolved;
stop(); // Reset TTS when loading new document
await setCurrentDocument(resolved);
loadedDocIdRef.current = resolved;
} catch (err) {
console.error('Error loading document:', err);
setError('Failed to load document');
} finally {
if (!didRedirect) {
if (startedLoad) {
inFlightDocIdRef.current = null;
}
if (!didRedirect && startedLoad) {
setIsLoading(false);
}
}

7
src/components/SettingsModal.tsx
View file

@ -38,6 +38,7 @@ import { useRouter } from 'next/navigation';
import { showPrivacyModal } from '@/components/PrivacyModal';
import { deleteDocuments, mimeTypeForDoc, uploadDocuments } from '@/lib/client-documents';
import { cacheStoredDocumentFromBytes, clearDocumentCache } from '@/lib/document-cache';
import { clearAllDocumentPreviewCaches, clearInMemoryDocumentPreviewCache } from '@/lib/document-preview-cache';
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
@ -188,6 +189,7 @@ export function SettingsModal({ className = '' }: { className?: string }) {
const handleRefresh = async () => {
try {
clearInMemoryDocumentPreviewCache();
await refreshDocuments();
} catch (error) {
console.error('Failed to refresh documents:', error);
@ -196,7 +198,10 @@ export function SettingsModal({ className = '' }: { className?: string }) {
const handleClearCache = async () => {
try {
await clearDocumentCache();
await Promise.all([
clearDocumentCache(),
clearAllDocumentPreviewCaches(),
]);
} catch (error) {
console.error('Failed to clear cache:', error);
}

147
src/components/doclist/DocumentPreview.tsx
View file

@ -2,11 +2,16 @@ import { DocumentListDocument } from '@/types/documents';
import { PDFIcon, EPUBIcon, FileIcon } from '@/components/icons/Icons';
import { useEffect, useMemo, useRef, useState } from 'react';
import {
extractEpubCoverToDataUrl,
renderPdfFirstPageToDataUrl,
} from '@/lib/documentPreview';
import { getDocumentContentSnippet } from '@/lib/client-documents';
import { ensureCachedDocument } from '@/lib/document-cache';
documentPreviewFallbackUrl,
getDocumentContentSnippet,
getDocumentPreviewStatus,
} from '@/lib/client-documents';
import {
getInMemoryDocumentPreviewUrl,
getPersistedDocumentPreviewUrl,
primeDocumentPreviewCache,
setInMemoryDocumentPreviewUrl,
} from '@/lib/document-preview-cache';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
@ -14,7 +19,6 @@ interface DocumentPreviewProps {
doc: DocumentListDocument;
}
const imagePreviewCache = new Map<string, string>();
const textPreviewCache = new Map<string, string>();
export function DocumentPreview({ doc }: DocumentPreviewProps) {
@ -33,20 +37,11 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
const containerRef = useRef<HTMLDivElement | null>(null);
const [isVisible, setIsVisible] = useState(false);
const [imagePreview, setImagePreview] = useState<string | null>(null);
const [isImageReady, setIsImageReady] = useState(false);
const [textPreview, setTextPreview] = useState<string | null>(null);
const [isGenerating, setIsGenerating] = useState(false);
const previewKey = useMemo(() => `${doc.type}:${doc.id}`, [doc.id, doc.type]);
const cacheMeta = useMemo(
() => ({
id: doc.id,
name: doc.name,
type: doc.type,
size: doc.size,
lastModified: doc.lastModified,
}),
[doc.id, doc.lastModified, doc.name, doc.size, doc.type],
);
useEffect(() => {
const el = containerRef.current;
@ -69,7 +64,7 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
useEffect(() => {
if (!isVisible) return;
const cachedImage = imagePreviewCache.get(previewKey);
const cachedImage = getInMemoryDocumentPreviewUrl(previewKey);
if (cachedImage) {
setImagePreview(cachedImage);
setTextPreview(null);
@ -89,32 +84,65 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
const run = async () => {
setIsGenerating(true);
try {
const targetWidth = 240;
if (doc.type === 'pdf') {
const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal });
if (cached.type !== 'pdf') return;
const data = cached.data;
if (cancelled) return;
const dataUrl = await renderPdfFirstPageToDataUrl(data, targetWidth);
if (cancelled) return;
imagePreviewCache.set(previewKey, dataUrl);
setImagePreview(dataUrl);
setTextPreview(null);
return;
}
if (doc.type === 'epub') {
const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal });
if (cached.type !== 'epub') return;
const data = cached.data;
if (cancelled) return;
const cover = await extractEpubCoverToDataUrl(data, targetWidth);
if (cancelled) return;
if (cover) {
imagePreviewCache.set(previewKey, cover);
setImagePreview(cover);
if (doc.type === 'pdf' || doc.type === 'epub') {
const persistedUrl = await getPersistedDocumentPreviewUrl(
doc.id,
Number(doc.lastModified),
previewKey,
);
if (!cancelled && persistedUrl) {
setImagePreview(persistedUrl);
setTextPreview(null);
return;
}
let attempt = 0;
while (!cancelled && attempt < 12) {
const status = await getDocumentPreviewStatus(doc.id, { signal: controller.signal });
if (cancelled) return;
if (status.kind === 'ready') {
const primedUrl = await primeDocumentPreviewCache(
doc.id,
Number(doc.lastModified),
previewKey,
{ signal: controller.signal },
).catch(() => null);
if (cancelled) return;
if (primedUrl) {
setImagePreview(primedUrl);
setTextPreview(null);
return;
}
const fallbackUrl = status.fallbackUrl || documentPreviewFallbackUrl(doc.id);
setInMemoryDocumentPreviewUrl(previewKey, fallbackUrl);
setImagePreview(fallbackUrl);
setTextPreview(null);
return;
}
if (status.status === 'failed') {
return;
}
const waitMs = Math.max(
400,
Math.min(6000, Number.isFinite(status.retryAfterMs) ? status.retryAfterMs : 1500),
);
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, waitMs);
controller.signal.addEventListener(
'abort',
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
attempt += 1;
}
return;
}
@ -145,7 +173,11 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
cancelled = true;
controller.abort();
};
}, [cacheMeta, doc.id, doc.type, isVisible, previewKey]);
}, [doc.id, doc.lastModified, doc.type, isVisible, previewKey]);
useEffect(() => {
setIsImageReady(false);
}, [imagePreview]);
const gradientClass = isPDF
? 'from-red-500/80 via-red-400/60 to-red-600/80'
@ -176,15 +208,40 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
>
{imagePreview ? (
<>
<div className={`absolute inset-0 bg-gradient-to-br ${gradientClass}`} />
{!isImageReady ? (
<div className="relative z-10 flex flex-col items-center justify-center h-full gap-2 px-2 text-white">
<Icon className="w-10 h-10 sm:w-12 sm:h-12 drop-shadow-md" />
<span className="text-[10px] sm:text-[11px] tracking-wide uppercase font-semibold opacity-90">
{typeLabel}
</span>
</div>
) : null}
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={imagePreview}
alt={`${doc.name} preview`}
className="absolute inset-0 h-full w-full object-cover"
className={`absolute inset-0 h-full w-full object-cover transition-opacity duration-150 ${isImageReady ? 'opacity-100' : 'opacity-0'}`}
draggable={false}
loading="lazy"
onLoad={() => {
setIsImageReady(true);
}}
onError={() => {
if (!imagePreview) return;
setIsImageReady(false);
const fallback = documentPreviewFallbackUrl(doc.id);
if (imagePreview === fallback) return;
setInMemoryDocumentPreviewUrl(previewKey, fallback);
setImagePreview(fallback);
void primeDocumentPreviewCache(
doc.id,
Number(doc.lastModified),
previewKey,
).catch(() => {});
}}
/>
<div className="absolute inset-0 bg-gradient-to-t from-black/35 via-black/0 to-black/15" />
{isImageReady ? <div className="absolute inset-0 bg-gradient-to-t from-black/35 via-black/0 to-black/15" /> : null}
</>
) : textPreview ? (
<>

1
src/db/schema.ts
View file

@ -14,3 +14,4 @@ export const verification = usePostgres ? postgresSchema.verification : sqliteSc
export const userTtsChars = usePostgres ? postgresSchema.userTtsChars : sqliteSchema.userTtsChars;
export const userPreferences = usePostgres ? postgresSchema.userPreferences : sqliteSchema.userPreferences;
export const userDocumentProgress = usePostgres ? postgresSchema.userDocumentProgress : sqliteSchema.userDocumentProgress;
export const documentPreviews = usePostgres ? postgresSchema.documentPreviews : sqliteSchema.documentPreviews;

23
src/db/schema_postgres.ts
View file

@ -118,3 +118,26 @@ export const userDocumentProgress = pgTable('user_document_progress', {
pk: primaryKey({ columns: [table.userId, table.documentId] }),
userUpdatedIdx: index('idx_user_document_progress_user_id_updated_at').on(table.userId, table.updatedAt),
}));
export const documentPreviews = pgTable('document_previews', {
documentId: text('document_id').notNull(),
namespace: text('namespace').notNull().default(''),
variant: text('variant').notNull().default('card-240-jpeg'),
status: text('status').notNull().default('queued'),
sourceLastModifiedMs: bigint('source_last_modified_ms', { mode: 'number' }).notNull(),
objectKey: text('object_key').notNull(),
contentType: text('content_type').notNull().default('image/jpeg'),
width: integer('width').notNull().default(240),
height: integer('height'),
byteSize: bigint('byte_size', { mode: 'number' }),
eTag: text('etag'),
leaseOwner: text('lease_owner'),
leaseUntilMs: bigint('lease_until_ms', { mode: 'number' }).notNull().default(0),
attemptCount: integer('attempt_count').notNull().default(0),
lastError: text('last_error'),
createdAtMs: bigint('created_at_ms', { mode: 'number' }).notNull().default(0),
updatedAtMs: bigint('updated_at_ms', { mode: 'number' }).notNull().default(0),
}, (table) => ({
pk: primaryKey({ columns: [table.documentId, table.namespace, table.variant] }),
statusLeaseIdx: index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs),
}));

23
src/db/schema_sqlite.ts
View file

@ -119,3 +119,26 @@ export const userDocumentProgress = sqliteTable('user_document_progress', {
pk: primaryKey({ columns: [table.userId, table.documentId] }),
userUpdatedIdx: index('idx_user_document_progress_user_id_updated_at').on(table.userId, table.updatedAt),
}));
export const documentPreviews = sqliteTable('document_previews', {
documentId: text('document_id').notNull(),
namespace: text('namespace').notNull().default(''),
variant: text('variant').notNull().default('card-240-jpeg'),
status: text('status').notNull().default('queued'),
sourceLastModifiedMs: integer('source_last_modified_ms').notNull(),
objectKey: text('object_key').notNull(),
contentType: text('content_type').notNull().default('image/jpeg'),
width: integer('width').notNull().default(240),
height: integer('height'),
byteSize: integer('byte_size'),
eTag: text('etag'),
leaseOwner: text('lease_owner'),
leaseUntilMs: integer('lease_until_ms').notNull().default(0),
attemptCount: integer('attempt_count').notNull().default(0),
lastError: text('last_error'),
createdAtMs: integer('created_at_ms').notNull().default(0),
updatedAtMs: integer('updated_at_ms').notNull().default(0),
}, (table) => ({
pk: primaryKey({ columns: [table.documentId, table.namespace, table.variant] }),
statusLeaseIdx: index('idx_document_previews_status_lease').on(table.status, table.leaseUntilMs),
}));

122
src/lib/client-documents.ts
View file

@ -202,17 +202,39 @@ export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user'
}
export async function downloadDocumentContent(id: string, options?: { signal?: AbortSignal }): Promise<ArrayBuffer> {
const res = await fetch(`/api/documents/blob?id=${encodeURIComponent(id)}`, { signal: options?.signal });
if (!res.ok) {
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || `Failed to download document (status ${res.status})`);
}
throw new Error(`Failed to download document (status ${res.status})`);
}
const fallbackUrl = `/api/documents/blob/get/fallback?id=${encodeURIComponent(id)}`;
return res.arrayBuffer();
const fetchFallback = async (): Promise<ArrayBuffer> => {
const res = await fetch(fallbackUrl, { signal: options?.signal });
if (!res.ok) {
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || `Failed to download document (status ${res.status})`);
}
throw new Error(`Failed to download document (status ${res.status})`);
}
return res.arrayBuffer();
};
try {
const directRes = await fetch(`/api/documents/blob/get/presign?id=${encodeURIComponent(id)}`, {
signal: options?.signal,
cache: 'no-store',
});
if (!directRes.ok) {
const contentType = directRes.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
const data = (await directRes.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || `Failed to download document (status ${directRes.status})`);
}
throw new Error(`Failed to download document (status ${directRes.status})`);
}
return directRes.arrayBuffer();
} catch (error) {
if (options?.signal?.aborted) throw error;
return fetchFallback();
}
}
export async function getDocumentContentSnippet(
@ -235,6 +257,86 @@ export async function getDocumentContentSnippet(
return data?.snippet || '';
}
export type DocumentPreviewPending = {
kind: 'pending';
status: 'queued' | 'processing' | 'failed';
retryAfterMs: number;
fallbackUrl: string;
presignUrl: string;
directUrl?: string;
};
export type DocumentPreviewReady = {
kind: 'ready';
fallbackUrl: string;
presignUrl: string;
directUrl?: string;
};
export type DocumentPreviewStatus = DocumentPreviewPending | DocumentPreviewReady;
function documentPreviewEnsureUrl(id: string): string {
return `/api/documents/blob/preview/ensure?id=${encodeURIComponent(id)}`;
}
export function documentPreviewPresignUrl(id: string): string {
return `/api/documents/blob/preview/presign?id=${encodeURIComponent(id)}`;
}
export function documentPreviewFallbackUrl(id: string): string {
return `/api/documents/blob/preview/fallback?id=${encodeURIComponent(id)}`;
}
export async function getDocumentPreviewStatus(
id: string,
options?: { signal?: AbortSignal },
): Promise<DocumentPreviewStatus> {
const res = await fetch(documentPreviewEnsureUrl(id), {
signal: options?.signal,
cache: 'no-store',
});
if (res.status === 202) {
const data = (await res.json().catch(() => null)) as {
status?: 'queued' | 'processing' | 'failed';
retryAfterMs?: number;
fallbackUrl?: string;
presignUrl?: string;
directUrl?: string;
} | null;
return {
kind: 'pending',
status: data?.status ?? 'queued',
retryAfterMs: Number.isFinite(data?.retryAfterMs) ? Number(data?.retryAfterMs) : 1500,
fallbackUrl: data?.fallbackUrl || documentPreviewFallbackUrl(id),
presignUrl: data?.presignUrl || documentPreviewPresignUrl(id),
directUrl: data?.directUrl,
};
}
if (res.ok) {
const data = (await res.json().catch(() => null)) as {
fallbackUrl?: string;
presignUrl?: string;
directUrl?: string;
} | null;
return {
kind: 'ready',
fallbackUrl: data?.fallbackUrl || documentPreviewFallbackUrl(id),
presignUrl: data?.presignUrl || documentPreviewPresignUrl(id),
directUrl: data?.directUrl,
};
}
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || `Failed to load preview status (status ${res.status})`);
}
throw new Error(`Failed to load preview status (status ${res.status})`);
}
export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise<BaseDocument> {
const form = new FormData();
form.append('file', file);

284
src/lib/dexie.ts
View file

@ -14,7 +14,7 @@ import { cacheStoredDocumentFromBytes } from '@/lib/document-cache';
const DB_NAME = 'openreader-db';
// Managed via Dexie (version bumped from the original manual IndexedDB)
const DB_VERSION = 6;
const DB_VERSION = 8;
const PDF_TABLE = 'pdf-documents' as const;
const EPUB_TABLE = 'epub-documents' as const;
@ -23,6 +23,20 @@ const CONFIG_TABLE = 'config' as const;
const APP_CONFIG_TABLE = 'app-config' as const;
const LAST_LOCATION_TABLE = 'last-locations' as const;
const DOCUMENT_ID_MAP_TABLE = 'document-id-map' as const;
const PREVIEW_CACHE_TABLE = 'document-preview-cache' as const;
const MIB = 1024 * 1024;
const DOCUMENT_CACHE_MAX_BYTES = 1024 * MIB; // 1 GiB
const PREVIEW_CACHE_MAX_BYTES = 128 * MIB; // 128 MiB
interface DocumentCacheMeta {
cacheCreatedAt?: number;
cacheAccessedAt?: number;
cacheByteSize?: number;
}
type PDFCacheRow = PDFDocument & DocumentCacheMeta;
type EPUBCacheRow = EPUBDocument & DocumentCacheMeta;
type HTMLCacheRow = HTMLDocument & DocumentCacheMeta;
export interface LastLocationRow {
docId: string;
@ -35,19 +49,29 @@ export interface DocumentIdMapRow {
createdAt: number;
}
export interface DocumentPreviewCacheRow {
docId: string;
lastModified: number;
contentType: string;
data: ArrayBuffer;
cachedAt: number;
byteSize?: number;
}
export interface ConfigRow {
key: string;
value: string;
}
type OpenReaderDB = Dexie & {
[PDF_TABLE]: EntityTable<PDFDocument, 'id'>;
[EPUB_TABLE]: EntityTable<EPUBDocument, 'id'>;
[HTML_TABLE]: EntityTable<HTMLDocument, 'id'>;
[PDF_TABLE]: EntityTable<PDFCacheRow, 'id'>;
[EPUB_TABLE]: EntityTable<EPUBCacheRow, 'id'>;
[HTML_TABLE]: EntityTable<HTMLCacheRow, 'id'>;
[CONFIG_TABLE]: EntityTable<ConfigRow, 'key'>;
[APP_CONFIG_TABLE]: EntityTable<AppConfigRow, 'id'>;
[LAST_LOCATION_TABLE]: EntityTable<LastLocationRow, 'docId'>;
[DOCUMENT_ID_MAP_TABLE]: EntityTable<DocumentIdMapRow, 'oldId'>;
[PREVIEW_CACHE_TABLE]: EntityTable<DocumentPreviewCacheRow, 'docId'>;
};
export const db = new Dexie(DB_NAME) as OpenReaderDB;
@ -192,15 +216,15 @@ function buildAppConfigFromRaw(raw: RawConfigMap): AppConfigRow {
return config;
}
// Version 6: add document-id-map table; keep v5 upgrade to migrate scattered config keys
// and drop the legacy config table in a single upgrade step.
// Version 8: add local cache metadata/indexes so document + preview caches can be bounded via LRU pruning.
db.version(DB_VERSION).stores({
[PDF_TABLE]: 'id, type, name, lastModified, size, folderId',
[EPUB_TABLE]: 'id, type, name, lastModified, size, folderId',
[HTML_TABLE]: 'id, type, name, lastModified, size, folderId',
[PDF_TABLE]: 'id, type, name, lastModified, size, folderId, cacheAccessedAt',
[EPUB_TABLE]: 'id, type, name, lastModified, size, folderId, cacheAccessedAt',
[HTML_TABLE]: 'id, type, name, lastModified, size, folderId, cacheAccessedAt',
[APP_CONFIG_TABLE]: 'id',
[LAST_LOCATION_TABLE]: 'docId',
[DOCUMENT_ID_MAP_TABLE]: 'oldId, id, createdAt',
[PREVIEW_CACHE_TABLE]: 'docId, lastModified, cachedAt, byteSize',
// `null` here means: drop the old 'config' table after upgrade runs,
// but Dexie still lets us read it inside the upgrade transaction.
[CONFIG_TABLE]: null,
@ -231,6 +255,7 @@ db.version(DB_VERSION).stores({
});
let dbOpenPromise: Promise<void> | null = null;
const cacheTextEncoder = new TextEncoder();
export async function initDB(): Promise<void> {
if (dbOpenPromise) {
@ -246,6 +271,9 @@ export async function initDB(): Promise<void> {
emitDexieStatus('stalled', { ms: Date.now() - startedAt });
}, 4000);
await db.open();
await Promise.all([pruneDocumentCacheIfNeededInternal(), prunePreviewCacheIfNeededInternal()]).catch((error) => {
console.warn('Dexie cache prune on open failed:', error);
});
clearTimeout(stallTimer);
console.log('Dexie database opened successfully');
emitDexieStatus('opened');
@ -265,6 +293,139 @@ async function withDB<T>(operation: () => Promise<T>): Promise<T> {
return operation();
}
type DocumentCacheTableName = typeof PDF_TABLE | typeof EPUB_TABLE | typeof HTML_TABLE;
type DocumentCacheEntryRef = {
table: DocumentCacheTableName;
id: string;
byteSize: number;
accessedAt: number;
};
function toPositiveInt(value: unknown, fallback: number = 0): number {
const n = Number(value);
if (!Number.isFinite(n) || n < 0) return fallback;
return Math.max(0, Math.floor(n));
}
function byteSizeForPdfRow(row: PDFCacheRow): number {
const cached = toPositiveInt(row.cacheByteSize, 0);
if (cached > 0) return cached;
return row.data instanceof ArrayBuffer ? row.data.byteLength : toPositiveInt(row.size, 0);
}
function byteSizeForEpubRow(row: EPUBCacheRow): number {
const cached = toPositiveInt(row.cacheByteSize, 0);
if (cached > 0) return cached;
return row.data instanceof ArrayBuffer ? row.data.byteLength : toPositiveInt(row.size, 0);
}
function byteSizeForHtmlRow(row: HTMLCacheRow): number {
const cached = toPositiveInt(row.cacheByteSize, 0);
if (cached > 0) return cached;
if (typeof row.data === 'string') return cacheTextEncoder.encode(row.data).byteLength;
return toPositiveInt(row.size, 0);
}
function accessedAtForRow(row: DocumentCacheMeta & { lastModified?: number }): number {
return toPositiveInt(row.cacheAccessedAt, toPositiveInt(row.lastModified, Date.now()));
}
async function deleteDocumentCacheEntryInternal(table: DocumentCacheTableName, id: string): Promise<void> {
if (table === PDF_TABLE) {
await db[PDF_TABLE].delete(id);
return;
}
if (table === EPUB_TABLE) {
await db[EPUB_TABLE].delete(id);
return;
}
await db[HTML_TABLE].delete(id);
}
async function pruneDocumentCacheIfNeededInternal(): Promise<void> {
const budget = DOCUMENT_CACHE_MAX_BYTES;
if (!Number.isFinite(budget) || budget <= 0) return;
const [pdfRows, epubRows, htmlRows] = await Promise.all([
db[PDF_TABLE].toArray(),
db[EPUB_TABLE].toArray(),
db[HTML_TABLE].toArray(),
]);
const entries: DocumentCacheEntryRef[] = [];
let totalBytes = 0;
for (const row of pdfRows) {
const byteSize = byteSizeForPdfRow(row);
totalBytes += byteSize;
entries.push({
table: PDF_TABLE,
id: row.id,
byteSize,
accessedAt: accessedAtForRow(row),
});
}
for (const row of epubRows) {
const byteSize = byteSizeForEpubRow(row);
totalBytes += byteSize;
entries.push({
table: EPUB_TABLE,
id: row.id,
byteSize,
accessedAt: accessedAtForRow(row),
});
}
for (const row of htmlRows) {
const byteSize = byteSizeForHtmlRow(row);
totalBytes += byteSize;
entries.push({
table: HTML_TABLE,
id: row.id,
byteSize,
accessedAt: accessedAtForRow(row),
});
}
if (totalBytes <= budget) return;
entries.sort((a, b) => a.accessedAt - b.accessedAt);
for (const entry of entries) {
if (totalBytes <= budget) break;
await deleteDocumentCacheEntryInternal(entry.table, entry.id);
totalBytes -= entry.byteSize;
}
}
async function prunePreviewCacheIfNeededInternal(): Promise<void> {
const budget = PREVIEW_CACHE_MAX_BYTES;
if (!Number.isFinite(budget) || budget <= 0) return;
const rows = await db[PREVIEW_CACHE_TABLE].toArray();
let totalBytes = 0;
const entries = rows.map((row) => {
const byteSize = toPositiveInt(
row.byteSize,
row.data instanceof ArrayBuffer ? row.data.byteLength : 0,
);
totalBytes += byteSize;
return {
docId: row.docId,
byteSize,
accessedAt: toPositiveInt(row.cachedAt, 0),
};
});
if (totalBytes <= budget) return;
entries.sort((a, b) => a.accessedAt - b.accessedAt);
for (const entry of entries) {
if (totalBytes <= budget) break;
await db[PREVIEW_CACHE_TABLE].delete(entry.docId);
totalBytes -= entry.byteSize;
}
}
function isSha256HexId(value: string): boolean {
return /^[a-f0-9]{64}$/i.test(value);
}
@ -325,6 +486,7 @@ async function applyDocumentIdMapping(oldId: string, newId: string): Promise<voi
db[LAST_LOCATION_TABLE],
db[APP_CONFIG_TABLE],
db[DOCUMENT_ID_MAP_TABLE],
db[PREVIEW_CACHE_TABLE],
],
async () => {
await recordDocumentIdMapping(oldId, nextId);
@ -395,6 +557,15 @@ async function applyDocumentIdMapping(oldId: string, newId: string): Promise<voi
await db[LAST_LOCATION_TABLE].delete(oldId);
}
const preview = await db[PREVIEW_CACHE_TABLE].get(oldId);
if (preview) {
const existing = await db[PREVIEW_CACHE_TABLE].get(nextId);
if (!existing || Number(existing.cachedAt ?? 0) < Number(preview.cachedAt ?? 0)) {
await db[PREVIEW_CACHE_TABLE].put({ ...preview, docId: nextId });
}
await db[PREVIEW_CACHE_TABLE].delete(oldId);
}
const appConfig = await db[APP_CONFIG_TABLE].get('singleton');
if (appConfig?.documentListState) {
const mapped = rewriteDocumentListStateDocIds(appConfig.documentListState, new Map([[oldId, nextId]]));
@ -457,7 +628,14 @@ export async function getDocumentIdMappings(): Promise<Array<{ oldId: string; id
export async function addPdfDocument(document: PDFDocument): Promise<void> {
await withDB(async () => {
console.log('Adding PDF document via Dexie:', document.name);
await db[PDF_TABLE].put(document);
const now = Date.now();
await db[PDF_TABLE].put({
...document,
cacheCreatedAt: now,
cacheAccessedAt: now,
cacheByteSize: document.data.byteLength,
});
await pruneDocumentCacheIfNeededInternal();
});
}
@ -465,7 +643,11 @@ export async function getPdfDocument(id: string): Promise<PDFDocument | undefine
return withDB(async () => {
console.log('Fetching PDF document via Dexie:', id);
const resolved = await getMappedDocumentId(id);
return db[PDF_TABLE].get(resolved);
const row = await db[PDF_TABLE].get(resolved);
if (row) {
await db[PDF_TABLE].update(resolved, { cacheAccessedAt: Date.now() });
}
return row;
});
}
@ -480,9 +662,10 @@ export async function removePdfDocument(id: string): Promise<void> {
await withDB(async () => {
console.log('Removing PDF document via Dexie:', id);
const resolved = await getMappedDocumentId(id);
await db.transaction('readwrite', db[PDF_TABLE], db[LAST_LOCATION_TABLE], async () => {
await db.transaction('readwrite', db[PDF_TABLE], db[LAST_LOCATION_TABLE], db[PREVIEW_CACHE_TABLE], async () => {
await db[PDF_TABLE].delete(resolved);
await db[LAST_LOCATION_TABLE].delete(resolved);
await db[PREVIEW_CACHE_TABLE].delete(resolved);
});
});
}
@ -508,7 +691,14 @@ export async function addEpubDocument(document: EPUBDocument): Promise<void> {
actualSize: document.data.byteLength,
});
await db[EPUB_TABLE].put(document);
const now = Date.now();
await db[EPUB_TABLE].put({
...document,
cacheCreatedAt: now,
cacheAccessedAt: now,
cacheByteSize: document.data.byteLength,
});
await pruneDocumentCacheIfNeededInternal();
});
}
@ -516,7 +706,11 @@ export async function getEpubDocument(id: string): Promise<EPUBDocument | undefi
return withDB(async () => {
console.log('Fetching EPUB document via Dexie:', id);
const resolved = await getMappedDocumentId(id);
return db[EPUB_TABLE].get(resolved);
const row = await db[EPUB_TABLE].get(resolved);
if (row) {
await db[EPUB_TABLE].update(resolved, { cacheAccessedAt: Date.now() });
}
return row;
});
}
@ -531,9 +725,10 @@ export async function removeEpubDocument(id: string): Promise<void> {
await withDB(async () => {
console.log('Removing EPUB document via Dexie:', id);
const resolved = await getMappedDocumentId(id);
await db.transaction('readwrite', db[EPUB_TABLE], db[LAST_LOCATION_TABLE], async () => {
await db.transaction('readwrite', db[EPUB_TABLE], db[LAST_LOCATION_TABLE], db[PREVIEW_CACHE_TABLE], async () => {
await db[EPUB_TABLE].delete(resolved);
await db[LAST_LOCATION_TABLE].delete(resolved);
await db[PREVIEW_CACHE_TABLE].delete(resolved);
});
});
}
@ -550,7 +745,14 @@ export async function clearEpubDocuments(): Promise<void> {
export async function addHtmlDocument(document: HTMLDocument): Promise<void> {
await withDB(async () => {
console.log('Adding HTML document via Dexie:', document.name);
await db[HTML_TABLE].put(document);
const now = Date.now();
await db[HTML_TABLE].put({
...document,
cacheCreatedAt: now,
cacheAccessedAt: now,
cacheByteSize: cacheTextEncoder.encode(document.data).byteLength,
});
await pruneDocumentCacheIfNeededInternal();
});
}
@ -558,7 +760,11 @@ export async function getHtmlDocument(id: string): Promise<HTMLDocument | undefi
return withDB(async () => {
console.log('Fetching HTML document via Dexie:', id);
const resolved = await getMappedDocumentId(id);
return db[HTML_TABLE].get(resolved);
const row = await db[HTML_TABLE].get(resolved);
if (row) {
await db[HTML_TABLE].update(resolved, { cacheAccessedAt: Date.now() });
}
return row;
});
}
@ -573,7 +779,10 @@ export async function removeHtmlDocument(id: string): Promise<void> {
await withDB(async () => {
console.log('Removing HTML document via Dexie:', id);
const resolved = await getMappedDocumentId(id);
await db[HTML_TABLE].delete(resolved);
await db.transaction('readwrite', db[HTML_TABLE], db[PREVIEW_CACHE_TABLE], async () => {
await db[HTML_TABLE].delete(resolved);
await db[PREVIEW_CACHE_TABLE].delete(resolved);
});
});
}
@ -648,6 +857,45 @@ export async function setFirstVisit(value: boolean): Promise<void> {
await updateAppConfig({ firstVisit: value });
}
// Document preview cache helpers
export async function getDocumentPreviewCache(docId: string): Promise<DocumentPreviewCacheRow | undefined> {
return withDB(async () => {
const resolved = await getMappedDocumentId(docId);
const row = await db[PREVIEW_CACHE_TABLE].get(resolved);
if (row) {
await db[PREVIEW_CACHE_TABLE].update(resolved, { cachedAt: Date.now() });
}
return row;
});
}
export async function putDocumentPreviewCache(row: DocumentPreviewCacheRow): Promise<void> {
await withDB(async () => {
const resolved = await getMappedDocumentId(row.docId);
await db[PREVIEW_CACHE_TABLE].put({
...row,
docId: resolved,
cachedAt: Date.now(),
byteSize: toPositiveInt(row.byteSize, row.data.byteLength),
});
await prunePreviewCacheIfNeededInternal();
});
}
export async function removeDocumentPreviewCache(docId: string): Promise<void> {
await withDB(async () => {
const resolved = await getMappedDocumentId(docId);
await db[PREVIEW_CACHE_TABLE].delete(resolved);
});
}
export async function clearDocumentPreviewCache(): Promise<void> {
await withDB(async () => {
await db[PREVIEW_CACHE_TABLE].clear();
});
}
// Sync helpers (server round-trip)
export async function syncDocumentsToServer(

125
src/lib/document-preview-cache.ts Normal file
View file

@ -0,0 +1,125 @@
import {
clearDocumentPreviewCache as clearPersistedDocumentPreviewCache,
getDocumentPreviewCache,
putDocumentPreviewCache,
removeDocumentPreviewCache,
} from '@/lib/dexie';
import { documentPreviewFallbackUrl, documentPreviewPresignUrl } from '@/lib/client-documents';
const inMemoryPreviewUrlCache = new Map<string, string>();
const inFlightPreviewPrime = new Map<string, Promise<string | null>>();
function revokeIfBlobUrl(url: string | null | undefined): void {
if (!url) return;
if (!url.startsWith('blob:')) return;
try {
URL.revokeObjectURL(url);
} catch {
// ignore
}
}
export function getInMemoryDocumentPreviewUrl(cacheKey: string): string | null {
return inMemoryPreviewUrlCache.get(cacheKey) || null;
}
export function setInMemoryDocumentPreviewUrl(cacheKey: string, url: string): void {
const prev = inMemoryPreviewUrlCache.get(cacheKey);
if (prev && prev !== url) {
revokeIfBlobUrl(prev);
}
inMemoryPreviewUrlCache.set(cacheKey, url);
}
export function clearInMemoryDocumentPreviewCache(): void {
for (const value of inMemoryPreviewUrlCache.values()) {
revokeIfBlobUrl(value);
}
inMemoryPreviewUrlCache.clear();
}
export async function getPersistedDocumentPreviewUrl(
docId: string,
lastModified: number,
cacheKey: string,
): Promise<string | null> {
const row = await getDocumentPreviewCache(docId);
if (!row) return null;
if (Number(row.lastModified) !== Number(lastModified)) {
await removeDocumentPreviewCache(docId).catch(() => {});
return null;
}
const contentType = row.contentType || 'image/jpeg';
const bytes = row.data;
if (!(bytes instanceof ArrayBuffer) || bytes.byteLength === 0) {
await removeDocumentPreviewCache(docId).catch(() => {});
return null;
}
const url = URL.createObjectURL(new Blob([bytes], { type: contentType }));
setInMemoryDocumentPreviewUrl(cacheKey, url);
return url;
}
export async function primeDocumentPreviewCache(
docId: string,
lastModified: number,
cacheKey: string,
options?: { signal?: AbortSignal },
): Promise<string | null> {
const primeKey = `${cacheKey}:${Number(lastModified)}`;
const existingPrime = inFlightPreviewPrime.get(primeKey);
if (existingPrime) {
return existingPrime;
}
const promise = (async (): Promise<string | null> => {
const existing = await getPersistedDocumentPreviewUrl(docId, lastModified, cacheKey);
if (existing) return existing;
const fetchOptions = {
signal: options?.signal,
cache: 'no-store' as const,
};
// Prefer presign path for priming so healthy direct object access avoids proxy load.
let res = await fetch(documentPreviewPresignUrl(docId), fetchOptions).catch(() => null);
if (!res || !res.ok) {
res = await fetch(documentPreviewFallbackUrl(docId), fetchOptions).catch(() => null);
}
if (!res || !res.ok) return null;
const blob = await res.blob();
const bytes = await blob.arrayBuffer();
if (bytes.byteLength === 0) return null;
const contentType = blob.type || 'image/jpeg';
await putDocumentPreviewCache({
docId,
lastModified: Number(lastModified),
contentType,
data: bytes,
cachedAt: Date.now(),
});
const url = URL.createObjectURL(new Blob([bytes], { type: contentType }));
setInMemoryDocumentPreviewUrl(cacheKey, url);
return url;
})();
inFlightPreviewPrime.set(primeKey, promise);
try {
return await promise;
} finally {
if (inFlightPreviewPrime.get(primeKey) === promise) {
inFlightPreviewPrime.delete(primeKey);
}
}
}
export async function clearAllDocumentPreviewCaches(): Promise<void> {
clearInMemoryDocumentPreviewCache();
await clearPersistedDocumentPreviewCache();
}

148
src/lib/server/document-previews-blobstore.ts Normal file
View file

@ -0,0 +1,148 @@
import {
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { deleteDocumentPrefix, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
import { getS3Client, getS3Config } from '@/lib/server/s3';
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
const DEFAULT_NAMESPACE_SEGMENT = '_default';
export const DOCUMENT_PREVIEW_VARIANT = 'card-240-jpeg';
export const DOCUMENT_PREVIEW_FILE_NAME = 'card-240.jpg';
export const DOCUMENT_PREVIEW_CONTENT_TYPE = 'image/jpeg';
export const DOCUMENT_PREVIEW_WIDTH = 240;
function sanitizeNamespace(namespace: string | null): string | null {
if (!namespace) return null;
if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null;
return namespace;
}
function namespaceSegment(namespace: string | null): string {
const safe = sanitizeNamespace(namespace);
return safe ?? DEFAULT_NAMESPACE_SEGMENT;
}
function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream {
return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function';
}
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
if (Buffer.isBuffer(chunk)) {
chunks.push(chunk);
} else if (typeof chunk === 'string') {
chunks.push(Buffer.from(chunk));
} else {
chunks.push(Buffer.from(chunk as Uint8Array));
}
}
return Buffer.concat(chunks);
}
async function bodyToBuffer(body: unknown): Promise<Buffer> {
if (!body) return Buffer.alloc(0);
if (body instanceof Uint8Array) return Buffer.from(body);
if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
if (body instanceof ArrayBuffer) return Buffer.from(body);
if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) {
const maybe = body as { transformToByteArray?: () => Promise<Uint8Array> };
if (typeof maybe.transformToByteArray === 'function') {
return Buffer.from(await maybe.transformToByteArray());
}
}
if (isNodeReadableStream(body)) {
return streamToBuffer(body);
}
throw new Error('Unsupported S3 response body type');
}
export function documentPreviewPrefix(documentId: string, namespace: string | null): string {
if (!isValidDocumentId(documentId)) {
throw new Error(`Invalid document id: ${documentId}`);
}
const cfg = getS3Config();
const ns = namespaceSegment(namespace);
return `${cfg.prefix}/document_previews_v1/ns/${ns}/${documentId}/`;
}
export function documentPreviewKey(documentId: string, namespace: string | null): string {
return `${documentPreviewPrefix(documentId, namespace)}${DOCUMENT_PREVIEW_FILE_NAME}`;
}
export async function headDocumentPreview(
documentId: string,
namespace: string | null,
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentPreviewKey(documentId, namespace);
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
return {
contentLength: Number(res.ContentLength ?? 0),
contentType: res.ContentType ?? null,
eTag: res.ETag ?? null,
};
}
export async function getDocumentPreviewBuffer(documentId: string, namespace: string | null): Promise<Buffer> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentPreviewKey(documentId, namespace);
const res = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: key }));
return bodyToBuffer(res.Body);
}
export async function putDocumentPreviewBuffer(
documentId: string,
bytes: Buffer,
namespace: string | null,
options?: { ifNoneMatch?: boolean },
): Promise<void> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentPreviewKey(documentId, namespace);
await client.send(
new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
Body: bytes,
ContentType: DOCUMENT_PREVIEW_CONTENT_TYPE,
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
}),
);
}
export async function presignDocumentPreviewGet(
documentId: string,
namespace: string | null,
options?: { expiresInSeconds?: number },
): Promise<string> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentPreviewKey(documentId, namespace);
return getSignedUrl(
client,
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
ResponseContentType: DOCUMENT_PREVIEW_CONTENT_TYPE,
}),
{ expiresIn: Math.max(30, Math.min(options?.expiresInSeconds ?? 300, 3600)) },
);
}
export async function deleteDocumentPreviewArtifacts(documentId: string, namespace: string | null): Promise<number> {
return deleteDocumentPrefix(documentPreviewPrefix(documentId, namespace));
}
export { isMissingBlobError };

258
src/lib/server/document-previews-render.ts Normal file
View file

@ -0,0 +1,258 @@
import path from 'path';
import { DOMMatrix, Path2D, createCanvas, loadImage } from '@napi-rs/canvas';
import JSZip from 'jszip';
import { XMLParser } from 'fast-xml-parser';
export type RenderedDocumentPreview = {
bytes: Buffer;
width: number;
height: number;
};
type CanvasAndContext = {
canvas: unknown;
context: unknown;
};
type NodeCanvasFactory = {
create: (width: number, height: number) => CanvasAndContext;
reset: (target: CanvasAndContext, width: number, height: number) => void;
destroy: (target: CanvasAndContext) => void;
};
function ensureNodeCanvasGlobals(): void {
const g = globalThis as Record<string, unknown>;
if (typeof g.DOMMatrix === 'undefined') {
g.DOMMatrix = DOMMatrix as unknown;
}
if (typeof g.Path2D === 'undefined') {
g.Path2D = Path2D as unknown;
}
}
function normalizeTargetWidth(targetWidth: number): number {
if (!Number.isFinite(targetWidth) || targetWidth <= 0) return 240;
return Math.max(64, Math.min(2048, Math.round(targetWidth)));
}
function asArray<T>(value: T | T[] | undefined): T[] {
if (!value) return [];
return Array.isArray(value) ? value : [value];
}
function findBestEpubCoverPath(opfPath: string, opfXml: string): string | null {
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
removeNSPrefix: true,
});
const parsed = parser.parse(opfXml) as Record<string, unknown>;
const pkg = (parsed.package ?? parsed['opf:package']) as Record<string, unknown> | undefined;
if (!pkg) return null;
const metadata = pkg.metadata as Record<string, unknown> | undefined;
const manifest = pkg.manifest as Record<string, unknown> | undefined;
const items = asArray((manifest?.item as Record<string, unknown> | Array<Record<string, unknown>> | undefined))
.filter((item) => typeof item === 'object' && item !== null);
const coverMetaId = asArray((metadata?.meta as Record<string, unknown> | Array<Record<string, unknown>> | undefined))
.find((meta) => {
const name = String(meta?.['@_name'] ?? '').trim().toLowerCase();
return name === 'cover';
})?.['@_content'];
const byCoverProperty = items.find((item) =>
String(item['@_properties'] ?? '')
.split(/\s+/)
.map((value) => value.trim().toLowerCase())
.includes('cover-image'),
);
const byMetaRef = coverMetaId
? items.find((item) => String(item['@_id'] ?? '') === String(coverMetaId))
: null;
const byNameHint = items.find((item) => String(item['@_id'] ?? '').toLowerCase().includes('cover'));
const byImageType = items.find((item) => String(item['@_media-type'] ?? '').toLowerCase().startsWith('image/'));
const selected = byCoverProperty ?? byMetaRef ?? byNameHint ?? byImageType;
if (!selected) return null;
const href = String(selected['@_href'] ?? '').trim();
if (!href) return null;
const opfDir = path.posix.dirname(opfPath);
return path.posix.normalize(path.posix.join(opfDir, href));
}
async function renderImageBytesToJpeg(imageBytes: Buffer, targetWidth: number): Promise<RenderedDocumentPreview> {
const bitmap = await loadImage(imageBytes);
const width = bitmap.width;
const height = bitmap.height;
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
throw new Error('Invalid source image dimensions');
}
const target = normalizeTargetWidth(targetWidth);
const scale = target / width;
const outWidth = Math.max(1, Math.round(width * scale));
const outHeight = Math.max(1, Math.round(height * scale));
const canvas = createCanvas(outWidth, outHeight);
const ctx = canvas.getContext('2d');
ctx.fillStyle = '#ffffff';
ctx.fillRect(0, 0, outWidth, outHeight);
ctx.drawImage(bitmap, 0, 0, outWidth, outHeight);
return {
bytes: canvas.toBuffer('image/jpeg', 82),
width: outWidth,
height: outHeight,
};
}
export async function renderEpubCoverToJpeg(sourceBytes: Buffer, targetWidth: number): Promise<RenderedDocumentPreview> {
const zip = await JSZip.loadAsync(sourceBytes);
const containerFile = zip.file('META-INF/container.xml');
if (!containerFile) {
throw new Error('EPUB container.xml not found');
}
const containerXml = await containerFile.async('string');
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
removeNSPrefix: true,
});
const containerParsed = parser.parse(containerXml) as Record<string, unknown>;
const rootfilesNode = (containerParsed.container as Record<string, unknown> | undefined)?.rootfiles as
| Record<string, unknown>
| undefined;
const rootfiles = asArray(rootfilesNode?.rootfile as Record<string, unknown> | Array<Record<string, unknown>> | undefined);
const rootfilePath = String(rootfiles[0]?.['@_full-path'] ?? '').trim();
if (!rootfilePath) {
throw new Error('EPUB OPF rootfile path missing');
}
const opfFile = zip.file(rootfilePath) ?? zip.file(decodeURI(rootfilePath));
if (!opfFile) {
throw new Error(`EPUB OPF not found at ${rootfilePath}`);
}
const opfXml = await opfFile.async('string');
const coverPath = findBestEpubCoverPath(rootfilePath, opfXml);
if (!coverPath) {
throw new Error('EPUB cover image not found');
}
const coverFile = zip.file(coverPath) ?? zip.file(decodeURI(coverPath));
if (!coverFile) {
throw new Error(`EPUB cover file missing at ${coverPath}`);
}
const coverBytes = await coverFile.async('nodebuffer');
return renderImageBytesToJpeg(coverBytes, targetWidth);
}
export async function renderPdfFirstPageToJpeg(sourceBytes: Buffer, targetWidth: number): Promise<RenderedDocumentPreview> {
ensureNodeCanvasGlobals();
type PdfViewport = {
width: number;
height: number;
};
type PdfPage = {
getViewport: (params: { scale: number }) => PdfViewport;
render: (params: {
canvasContext: unknown;
viewport: PdfViewport;
intent: 'display';
canvasFactory?: NodeCanvasFactory;
}) => { promise: Promise<void> };
};
// pdfjs-dist legacy build works in Node and avoids relying on DOM workers.
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore - pdfjs-dist legacy build path has no dedicated TypeScript declaration.
const pdfjs = (await import('pdfjs-dist/legacy/build/pdf.mjs')) as {
getDocument: (options: Record<string, unknown>) => {
promise: Promise<{ getPage: (n: number) => Promise<PdfPage>; destroy: () => Promise<void> }>;
destroy: () => Promise<void>;
};
GlobalWorkerOptions?: { workerSrc?: string; workerPort?: unknown };
};
if (pdfjs.GlobalWorkerOptions) {
pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs';
pdfjs.GlobalWorkerOptions.workerPort = null;
}
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
const standardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`;
const nodeCanvasFactory: NodeCanvasFactory = {
create: (width, height) => {
const canvas = createCanvas(width, height);
const context = canvas.getContext('2d');
return { canvas, context };
},
reset: (target, width, height) => {
const canvas = target.canvas as { width: number; height: number };
canvas.width = width;
canvas.height = height;
},
destroy: (target) => {
const canvas = target.canvas as { width: number; height: number };
canvas.width = 0;
canvas.height = 0;
},
};
class PdfNodeCanvasFactory {
create(width: number, height: number): CanvasAndContext {
return nodeCanvasFactory.create(width, height);
}
reset(target: CanvasAndContext, width: number, height: number): void {
nodeCanvasFactory.reset(target, width, height);
}
destroy(target: CanvasAndContext): void {
nodeCanvasFactory.destroy(target);
}
}
const loadingTask = pdfjs.getDocument({
data: new Uint8Array(sourceBytes),
useWorkerFetch: false,
standardFontDataUrl,
CanvasFactory: PdfNodeCanvasFactory,
isEvalSupported: false,
});
const pdf = await loadingTask.promise;
try {
const page = await pdf.getPage(1);
const viewport = page.getViewport({ scale: 1 });
const target = normalizeTargetWidth(targetWidth);
const scale = target / viewport.width;
const scaledViewport = page.getViewport({ scale });
const outWidth = Math.max(1, Math.floor(scaledViewport.width));
const outHeight = Math.max(1, Math.floor(scaledViewport.height));
const canvas = createCanvas(outWidth, outHeight);
const ctx = canvas.getContext('2d');
ctx.fillStyle = '#ffffff';
ctx.fillRect(0, 0, outWidth, outHeight);
const renderTask = page.render({
canvasContext: ctx,
viewport: scaledViewport,
intent: 'display',
canvasFactory: nodeCanvasFactory,
});
await renderTask.promise;
return {
bytes: canvas.toBuffer('image/jpeg', 82),
width: outWidth,
height: outHeight,
};
} finally {
await pdf.destroy().catch(() => undefined);
await loadingTask.destroy().catch(() => undefined);
}
}

427
src/lib/server/document-previews.ts Normal file
View file

@ -0,0 +1,427 @@
import { randomUUID } from 'crypto';
import { mkdtemp, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { and, eq, inArray, lt, or, sql } from 'drizzle-orm';
import { db } from '@/db';
import { documentPreviews } from '@/db/schema';
import {
DOCUMENT_PREVIEW_CONTENT_TYPE,
DOCUMENT_PREVIEW_VARIANT,
DOCUMENT_PREVIEW_WIDTH,
deleteDocumentPreviewArtifacts,
documentPreviewKey,
headDocumentPreview,
isMissingBlobError,
putDocumentPreviewBuffer,
} from '@/lib/server/document-previews-blobstore';
import { getDocumentBlob } from '@/lib/server/documents-blobstore';
import { renderEpubCoverToJpeg, renderPdfFirstPageToJpeg } from '@/lib/server/document-previews-render';
const LEASE_MS = 45_000;
const RETRY_AFTER_MS = 1_500;
const FAILED_RETRY_AFTER_MS = 15_000;
type PreviewStatus = 'queued' | 'processing' | 'ready' | 'failed';
type PreviewRow = {
documentId: string;
namespace: string;
variant: string;
status: PreviewStatus;
sourceLastModifiedMs: number;
objectKey: string;
contentType: string;
width: number;
height: number | null;
byteSize: number | null;
eTag: string | null;
leaseOwner: string | null;
leaseUntilMs: number;
attemptCount: number;
lastError: string | null;
createdAtMs: number;
updatedAtMs: number;
};
export type PreviewableDocumentType = 'pdf' | 'epub';
export type PreviewSourceDocument = {
id: string;
type: string;
lastModified: number;
};
export type EnsureDocumentPreviewResult =
| {
state: 'ready';
status: 'ready';
contentType: string;
width: number;
height: number | null;
byteSize: number | null;
eTag: string | null;
}
| {
state: 'pending';
status: Exclude<PreviewStatus, 'ready'>;
retryAfterMs: number;
lastError: string | null;
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function safeDb(): any {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return db as any;
}
function nowMs(): number {
return Date.now();
}
function rowsAffected(result: unknown): number {
if (!result || typeof result !== 'object') return 0;
const rec = result as Record<string, unknown>;
if (typeof rec.rowCount === 'number') return rec.rowCount;
if (typeof rec.changes === 'number') return rec.changes;
return 0;
}
function toNamespaceKey(namespace: string | null): string {
return namespace?.trim() || '';
}
function previewObjectKey(documentId: string, namespace: string | null): string {
return documentPreviewKey(documentId, namespace);
}
function asPreviewStatus(status: string | null | undefined): PreviewStatus {
if (status === 'processing' || status === 'ready' || status === 'failed') return status;
return 'queued';
}
function toPreviewRow(raw: Record<string, unknown>): PreviewRow {
return {
documentId: String(raw.documentId ?? ''),
namespace: String(raw.namespace ?? ''),
variant: String(raw.variant ?? ''),
status: asPreviewStatus(String(raw.status ?? 'queued')),
sourceLastModifiedMs: Number(raw.sourceLastModifiedMs ?? 0),
objectKey: String(raw.objectKey ?? ''),
contentType: String(raw.contentType ?? DOCUMENT_PREVIEW_CONTENT_TYPE),
width: Number(raw.width ?? DOCUMENT_PREVIEW_WIDTH),
height: raw.height == null ? null : Number(raw.height),
byteSize: raw.byteSize == null ? null : Number(raw.byteSize),
eTag: raw.eTag == null ? null : String(raw.eTag),
leaseOwner: raw.leaseOwner == null ? null : String(raw.leaseOwner),
leaseUntilMs: Number(raw.leaseUntilMs ?? 0),
attemptCount: Number(raw.attemptCount ?? 0),
lastError: raw.lastError == null ? null : String(raw.lastError),
createdAtMs: Number(raw.createdAtMs ?? 0),
updatedAtMs: Number(raw.updatedAtMs ?? 0),
};
}
export function isPreviewableDocumentType(type: string): type is PreviewableDocumentType {
return type === 'pdf' || type === 'epub';
}
async function getPreviewRow(documentId: string, namespaceKey: string): Promise<PreviewRow | null> {
const rows = (await safeDb()
.select()
.from(documentPreviews)
.where(
and(
eq(documentPreviews.documentId, documentId),
eq(documentPreviews.namespace, namespaceKey),
eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT),
),
)) as Array<Record<string, unknown>>;
const row = rows[0];
return row ? toPreviewRow(row) : null;
}
async function ensurePreviewRowExists(doc: PreviewSourceDocument, namespaceKey: string, namespace: string | null): Promise<void> {
const now = nowMs();
await safeDb()
.insert(documentPreviews)
.values({
documentId: doc.id,
namespace: namespaceKey,
variant: DOCUMENT_PREVIEW_VARIANT,
status: 'queued',
sourceLastModifiedMs: doc.lastModified,
objectKey: previewObjectKey(doc.id, namespace),
contentType: DOCUMENT_PREVIEW_CONTENT_TYPE,
width: DOCUMENT_PREVIEW_WIDTH,
leaseUntilMs: 0,
attemptCount: 0,
createdAtMs: now,
updatedAtMs: now,
})
.onConflictDoNothing();
}
async function markPreviewRowQueued(
doc: PreviewSourceDocument,
namespaceKey: string,
namespace: string | null,
): Promise<void> {
const now = nowMs();
await safeDb()
.update(documentPreviews)
.set({
status: 'queued',
sourceLastModifiedMs: doc.lastModified,
objectKey: previewObjectKey(doc.id, namespace),
contentType: DOCUMENT_PREVIEW_CONTENT_TYPE,
width: DOCUMENT_PREVIEW_WIDTH,
height: null,
byteSize: null,
eTag: null,
leaseOwner: null,
leaseUntilMs: 0,
lastError: null,
updatedAtMs: now,
})
.where(
and(
eq(documentPreviews.documentId, doc.id),
eq(documentPreviews.namespace, namespaceKey),
eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT),
),
);
}
function needsRequeue(row: PreviewRow, doc: PreviewSourceDocument, namespace: string | null): boolean {
if (row.sourceLastModifiedMs !== doc.lastModified) return true;
if (row.objectKey !== previewObjectKey(doc.id, namespace)) return true;
return false;
}
async function isReadyBlobMissing(docId: string, namespace: string | null): Promise<boolean> {
try {
await headDocumentPreview(docId, namespace);
return false;
} catch (error) {
if (isMissingBlobError(error)) return true;
throw error;
}
}
async function tryClaimPreviewLease(docId: string, namespaceKey: string, owner: string): Promise<boolean> {
const now = nowMs();
const result = await safeDb()
.update(documentPreviews)
.set({
status: 'processing',
leaseOwner: owner,
leaseUntilMs: now + LEASE_MS,
attemptCount: sql`${documentPreviews.attemptCount} + 1`,
lastError: null,
updatedAtMs: now,
})
.where(
and(
eq(documentPreviews.documentId, docId),
eq(documentPreviews.namespace, namespaceKey),
eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT),
or(
inArray(documentPreviews.status, ['queued', 'failed']),
and(
eq(documentPreviews.status, 'processing'),
lt(documentPreviews.leaseUntilMs, now),
),
),
),
);
return rowsAffected(result) > 0;
}
async function markPreviewReady(
doc: PreviewSourceDocument,
namespaceKey: string,
content: { width: number; height: number | null; byteSize: number; eTag: string | null },
): Promise<void> {
const now = nowMs();
await safeDb()
.update(documentPreviews)
.set({
status: 'ready',
sourceLastModifiedMs: doc.lastModified,
contentType: DOCUMENT_PREVIEW_CONTENT_TYPE,
width: content.width,
height: content.height,
byteSize: content.byteSize,
eTag: content.eTag,
leaseOwner: null,
leaseUntilMs: 0,
lastError: null,
updatedAtMs: now,
})
.where(
and(
eq(documentPreviews.documentId, doc.id),
eq(documentPreviews.namespace, namespaceKey),
eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT),
),
);
}
async function markPreviewFailed(docId: string, namespaceKey: string, error: unknown): Promise<void> {
const now = nowMs();
const message = error instanceof Error ? error.message : String(error ?? 'Preview generation failed');
await safeDb()
.update(documentPreviews)
.set({
status: 'failed',
leaseOwner: null,
leaseUntilMs: 0,
lastError: message.slice(0, 1000),
updatedAtMs: now,
})
.where(
and(
eq(documentPreviews.documentId, docId),
eq(documentPreviews.namespace, namespaceKey),
eq(documentPreviews.variant, DOCUMENT_PREVIEW_VARIANT),
),
);
}
async function generateAndStorePreview(doc: PreviewSourceDocument, namespace: string | null): Promise<void> {
let workDir: string | null = null;
try {
const sourceBytes = await getDocumentBlob(doc.id, namespace);
workDir = await mkdtemp(join(tmpdir(), 'openreader-preview-'));
const sourcePath = join(workDir, 'source');
await writeFile(sourcePath, sourceBytes);
let rendered;
if (doc.type === 'pdf') {
rendered = await renderPdfFirstPageToJpeg(sourceBytes, DOCUMENT_PREVIEW_WIDTH);
} else if (doc.type === 'epub') {
rendered = await renderEpubCoverToJpeg(sourceBytes, DOCUMENT_PREVIEW_WIDTH);
} else {
throw new Error(`Unsupported preview type: ${doc.type}`);
}
await putDocumentPreviewBuffer(doc.id, rendered.bytes, namespace);
} finally {
if (workDir) {
await rm(workDir, { recursive: true, force: true }).catch(() => {});
}
}
}
function pendingResult(status: PreviewStatus, lastError: string | null): EnsureDocumentPreviewResult {
return {
state: 'pending',
status: status === 'ready' ? 'processing' : status,
retryAfterMs: status === 'failed' ? FAILED_RETRY_AFTER_MS : RETRY_AFTER_MS,
lastError,
};
}
export async function enqueueDocumentPreview(doc: PreviewSourceDocument, namespace: string | null): Promise<void> {
if (!isPreviewableDocumentType(doc.type)) return;
const namespaceKey = toNamespaceKey(namespace);
await ensurePreviewRowExists(doc, namespaceKey, namespace);
const row = await getPreviewRow(doc.id, namespaceKey);
if (!row || needsRequeue(row, doc, namespace) || row.status === 'failed') {
await markPreviewRowQueued(doc, namespaceKey, namespace);
}
}
export async function ensureDocumentPreview(doc: PreviewSourceDocument, namespace: string | null): Promise<EnsureDocumentPreviewResult> {
if (!isPreviewableDocumentType(doc.type)) {
return pendingResult('failed', `Unsupported preview type: ${doc.type}`);
}
const namespaceKey = toNamespaceKey(namespace);
await ensurePreviewRowExists(doc, namespaceKey, namespace);
let row = await getPreviewRow(doc.id, namespaceKey);
if (!row) {
return pendingResult('queued', null);
}
if (needsRequeue(row, doc, namespace)) {
await markPreviewRowQueued(doc, namespaceKey, namespace);
row = await getPreviewRow(doc.id, namespaceKey);
if (!row) return pendingResult('queued', null);
}
if (row.status === 'ready') {
const missing = await isReadyBlobMissing(doc.id, namespace);
if (!missing) {
return {
state: 'ready',
status: 'ready',
contentType: row.contentType || DOCUMENT_PREVIEW_CONTENT_TYPE,
width: row.width || DOCUMENT_PREVIEW_WIDTH,
height: row.height,
byteSize: row.byteSize,
eTag: row.eTag,
};
}
await markPreviewRowQueued(doc, namespaceKey, namespace);
row = await getPreviewRow(doc.id, namespaceKey);
if (!row) return pendingResult('queued', null);
}
const now = nowMs();
if (row.status === 'processing' && row.leaseUntilMs > now) {
return pendingResult('processing', row.lastError);
}
const owner = `req-${randomUUID()}`;
const claimed = await tryClaimPreviewLease(doc.id, namespaceKey, owner);
if (claimed) {
try {
await generateAndStorePreview(doc, namespace);
const head = await headDocumentPreview(doc.id, namespace);
await markPreviewReady(doc, namespaceKey, {
width: DOCUMENT_PREVIEW_WIDTH,
height: null,
byteSize: head.contentLength,
eTag: head.eTag,
});
} catch (error) {
await markPreviewFailed(doc.id, namespaceKey, error);
}
}
row = await getPreviewRow(doc.id, namespaceKey);
if (!row) return pendingResult('queued', null);
if (row.status === 'ready') {
const missing = await isReadyBlobMissing(doc.id, namespace);
if (!missing) {
return {
state: 'ready',
status: 'ready',
contentType: row.contentType || DOCUMENT_PREVIEW_CONTENT_TYPE,
width: row.width || DOCUMENT_PREVIEW_WIDTH,
height: row.height,
byteSize: row.byteSize,
eTag: row.eTag,
};
}
await markPreviewRowQueued(doc, namespaceKey, namespace);
return pendingResult('queued', null);
}
return pendingResult(row.status, row.lastError);
}
export async function deleteDocumentPreviewRows(documentId: string, namespace: string | null): Promise<void> {
const namespaceKey = toNamespaceKey(namespace);
await safeDb()
.delete(documentPreviews)
.where(and(eq(documentPreviews.documentId, documentId), eq(documentPreviews.namespace, namespaceKey)));
}
export async function cleanupDocumentPreviewArtifacts(documentId: string, namespace: string | null): Promise<void> {
await deleteDocumentPreviewArtifacts(documentId, namespace);
}

18
src/lib/server/documents-blobstore.ts
View file

@ -146,6 +146,24 @@ export async function getDocumentBlob(id: string, namespace: string | null): Pro
return bodyToBuffer(res.Body);
}
export async function presignGet(
id: string,
namespace: string | null,
options?: { expiresInSeconds?: number },
): Promise<string> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
return getSignedUrl(
client,
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
}),
{ expiresIn: Math.max(30, Math.min(options?.expiresInSeconds ?? 300, 3600)) },
);
}
export async function putDocumentBlob(
id: string,
body: Buffer,

29
tests/unit/document-previews-render.spec.ts Normal file
View file

@ -0,0 +1,29 @@
import { test, expect } from '@playwright/test';
import path from 'path';
import { readFile } from 'fs/promises';
import {
renderEpubCoverToJpeg,
renderPdfFirstPageToJpeg,
} from '../../src/lib/server/document-previews-render';
test.describe('document-previews-render', () => {
test('renders first PDF page to JPEG preview', async () => {
const pdfPath = path.join(process.cwd(), 'tests/files/sample.pdf');
const bytes = await readFile(pdfPath);
const rendered = await renderPdfFirstPageToJpeg(bytes, 240);
expect(rendered.bytes.byteLength).toBeGreaterThan(1024);
expect(rendered.width).toBe(240);
expect(rendered.height).toBeGreaterThan(0);
});
test('extracts EPUB cover and renders to JPEG preview', async () => {
const epubPath = path.join(process.cwd(), 'tests/files/sample.epub');
const bytes = await readFile(epubPath);
const rendered = await renderEpubCoverToJpeg(bytes, 240);
expect(rendered.bytes.byteLength).toBeGreaterThan(1024);
expect(rendered.width).toBe(240);
expect(rendered.height).toBeGreaterThan(0);
});
});