feat(storage): implement S3/SeaweedFS blob storage for documents

Replaces local filesystem document storage with S3-compatible object storage.
Adds embedded SeaweedFS 'weed mini' for local development and Docker deployments.
Updates document upload flow to use presigned URLs with a server fallback proxy.
Refactors auth configuration to use BASE_URL and AUTH_SECRET.

BREAKING CHANGE: Renamed BETTER_AUTH_URL to BASE_URL and BETTER_AUTH_SECRET to AUTH_SECRET. Removed legacy document upload/content endpoints. Requires S3 environment variables (auto-configured for embedded SeaweedFS).
This commit is contained in:
Richard R 2026-02-10 12:29:15 -07:00
parent 4a5f3060f2
commit 81d249ed52
38 changed files with 3314 additions and 887 deletions

View file

@ -9,14 +9,30 @@ API_KEY=api_key_optional
# Path to your local whisper.cpp CLI binary for STT timestamp generation
WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli
# Auth (recommended for contributors and public instances)
# (Optional) Auth is only enabled when **both** AUTH_SECRET and BASE_URL are set
BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network)
AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32`
AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional trusted origins (BASE_URL is always trusted)
# (Optional) Sign in w/ GitHub Configuration
GITHUB_CLIENT_ID=
GITHUB_CLIENT_SECRET=
# Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables.
# Defaults to SQLite at docstore/sqlite3.db when not set.
POSTGRES_URL=
# Auth (recommended for contributors and public instances)
# (Optional) Auth is only enabled when **both** BETTER_AUTH_SECRET and BETTER_AUTH_URL are set
BETTER_AUTH_URL=http://localhost:3003 # Your externally facing URL for this app
BETTER_AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32`
# (Optional) Sign in w/ GitHub Configuration
GITHUB_CLIENT_ID=
GITHUB_CLIENT_SECRET=
# Embedded SeaweedFS weed mini config
USE_EMBEDDED_WEED_MINI=
WEED_MINI_DIR=
WEED_MINI_WAIT_SEC=
# S3 storage config (use with embedded weed mini or external S3-compatible storage)
# For embedded weed mini, set explicit keys if you want stable credentials across restarts.
S3_ACCESS_KEY_ID=
S3_SECRET_ACCESS_KEY=
S3_BUCKET=
S3_REGION=
# If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host.
S3_ENDPOINT=
S3_FORCE_PATH_STYLE=
S3_PREFIX=

3
.gitignore vendored
View file

@ -2,6 +2,7 @@
# dependencies
/node_modules
/.pnpm-store
/.pnp
.pnp.*
.yarn/*
@ -52,4 +53,4 @@ node_modules/
/playwright/.cache/
# vscode
.vscode
.vscode

View file

@ -12,6 +12,10 @@ RUN git clone --depth 1 https://github.com/ggml-org/whisper.cpp.git && \
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF $( [ "$TARGETARCH" = "arm64" ] && echo "-DGGML_CPU_ARM_ARCH=armv8-a" || true ) && \
cmake --build build -j
# Stage 1b: extract seaweedfs weed binary (for optional embedded weed mini)
FROM chrislusf/seaweedfs:latest AS seaweedfs-builder
RUN cp "$(command -v weed)" /tmp/weed
# Stage 2: build the Next.js app
FROM node:lts-alpine AS app-builder
@ -42,7 +46,7 @@ FROM node:lts-alpine AS runner
# Add runtime OS dependencies:
# - ffmpeg: required for audiobook export and word-by-word alignment (/api/whisper)
# - libreoffice-writer: required for DOCX → PDF conversion
RUN apk add --no-cache ffmpeg libreoffice-writer
RUN apk add --no-cache ca-certificates ffmpeg libreoffice-writer
# Install pnpm globally for running the app
RUN npm install -g pnpm
@ -56,6 +60,9 @@ COPY --from=app-builder /app ./
# Copy the compiled whisper.cpp build output into the runtime image
# (includes whisper-cli and its shared libraries, e.g. libwhisper.so, libggml.so)
COPY --from=whisper-builder /opt/whisper.cpp/build /opt/whisper.cpp/build
# Copy seaweedfs weed binary for optional embedded local S3.
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
RUN chmod +x /usr/local/bin/weed
# Point the app at the compiled whisper-cli binary and ensure its libs are discoverable
ENV WHISPER_CPP_BIN=/opt/whisper.cpp/build/bin/whisper-cli
@ -65,4 +72,5 @@ ENV LD_LIBRARY_PATH=/opt/whisper.cpp/build
EXPOSE 3003
# Start the application
CMD ["pnpm", "start"]
ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"]
CMD ["pnpm", "start:raw"]

189
README.md
View file

@ -42,63 +42,110 @@ OpenReader WebUI is an open source text to speech document reader web app built
### 1. 🐳 Start the Docker container
Minimal (no persistence, auth disabled unless you set auth env vars):
Minimal (auth disabled, embedded storage is ephemeral, no library import):
```bash
docker run --name openreader-webui \
--restart unless-stopped \
-p 3003:3003 \
-p 8333:8333 \
ghcr.io/richardr1126/openreader-webui:latest
```
Fully featured (persistent storage + server library import + KokoroFastAPI in Docker + optional auth):
Fully featured (persistent storage, embedded SeaweedFS `weed mini` for documents, optional auth):
```bash
docker run --name openreader-webui \
--restart unless-stopped \
-p 3003:3003 \
-p 8333:8333 \
-v openreader_docstore:/app/docstore \
-v /path/to/your/library:/app/docstore/library:ro \
-e API_BASE=http://host.docker.internal:8880/v1 \
-e API_KEY=none \
-e BETTER_AUTH_URL=http://localhost:3003 \
-e BETTER_AUTH_SECRET=<paste_the_output_of_openssl_here> \
-e BASE_URL=http://localhost:3003 \
-e AUTH_SECRET=<paste_the_output_of_openssl_here> \
ghcr.io/richardr1126/openreader-webui:latest
```
You can remove the `/app/docstore/library` mount if you don't need server library import.
You can remove both `BETTER_AUTH_*` env vars to keep auth disabled.
You can remove either `BASE_URL` or `AUTH_SECRET` to keep auth disabled.
> **Notes:**
>
> - `API_BASE` should point to your TTS API server's base URL (if running Kokoro-FastAPI locally in Docker, use `http://host.docker.internal:8880/v1`).
> - `BETTER_AUTH_URL` should be your externally-facing URL for this app (for example `https://reader.example.com` or `http://localhost:3003`).
> - To enable auth, set **both** `BETTER_AUTH_URL` and `BETTER_AUTH_SECRET` generated with `openssl rand -base64 32`.
> - OpenReader always uses a backend DB for server-side metadata. By default it uses SQLite at `/app/docstore/sqlite3.db` (persisted when `/app/docstore` is mounted).
> - If you set `POSTGRES_URL`, the container uses Postgres instead of SQLite and will run migrations against it on startup. Ensure the database is accessible.
Quick notes:
- `API_BASE` should point to your TTS API server's base URL (for local Docker Kokoro, use `http://host.docker.internal:8880/v1`).
- Expose `-p 8333:8333` for direct browser access to embedded SeaweedFS presigned URLs.
- If port `8333` is not exposed, uploads still work via `/api/documents/blob/upload/fallback`.
- To enable auth, set both `BASE_URL` and `AUTH_SECRET`.
<details>
<summary><strong>Docker environment variables</strong> (Click to expand)</summary>
<summary><strong>Docker networking and blob behavior</strong> (Click to expand)</summary>
**Ports**
- `3003` serves the OpenReader app and API routes.
- `8333` serves embedded SeaweedFS S3 for browser direct blob access.
**Upload behavior**
- Primary upload path: browser uploads to presigned URL from `/api/documents/blob/upload/presign`.
- Fallback upload path: `/api/documents/blob/upload/fallback` if direct upload fails.
- Content serving path: `/api/documents/blob` (server-served bytes/snippets).
</details>
<details>
<summary><strong>Which Docker setup should I use?</strong> (Click to expand)</summary>
| Goal | Recommended setup |
| --- | --- |
| Fast local setup, no persistence | Minimal `docker run` example |
| Persistent docs/audiobooks and optional auth | Fully featured `docker run` example with `/app/docstore` volume |
| Browser direct blob uploads/downloads | Publish both `3003` and `8333` |
| Private blob endpoint (no `8333` published) | Keep using app APIs; uploads use fallback proxy when direct presigned upload is unreachable |
</details>
<details>
<summary><strong>Common Docker environment variables</strong> (Click to expand)</summary>
| Variable | Purpose | Example / Notes |
| --- | --- | --- |
| `API_BASE` | Default TTS API base URL (server-side) | `http://host.docker.internal:8880/v1` |
| `API_KEY` | Default TTS API key | `none` or your provider key |
| `BETTER_AUTH_URL` | Enables auth when set with `BETTER_AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` |
| `BETTER_AUTH_SECRET` | Enables auth when set with `BETTER_AUTH_URL` | Generate with `openssl rand -base64 32` |
| `BASE_URL` | Enables auth when set with `AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` |
| `AUTH_SECRET` | Enables auth when set with `BASE_URL` | Generate with `openssl rand -base64 32` |
| `AUTH_TRUSTED_ORIGINS` | Extra allowed auth request origins | Comma-separated list; leave empty to trust only `BASE_URL` |
| `POSTGRES_URL` | Use Postgres for server DB (metadata + auth tables) instead of SQLite | If set, startup migrations target Postgres |
| `GITHUB_CLIENT_ID` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_SECRET` |
| `GITHUB_CLIENT_SECRET` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_ID` |
</details>
<details>
<summary><strong>Blob and embedded storage environment variables</strong> (Click to expand)</summary>
| Variable | Purpose | Example / Notes |
| --- | --- | --- |
| `USE_EMBEDDED_WEED_MINI` | Start SeaweedFS before app startup | default: `true` when unset in shared entrypoint |
| `WEED_MINI_DIR` | Data directory for embedded `weed mini` | default: `docstore/seaweedfs` |
| `WEED_MINI_WAIT_SEC` | Startup wait timeout for embedded `weed mini` | default: `20` |
| `S3_BUCKET` | S3 bucket used for document blobs | default: `openreader-documents` in embedded mode |
| `S3_REGION` | S3 region used by AWS SDK | default: `us-east-1` in embedded mode |
| `S3_ENDPOINT` | Custom endpoint for S3-compatible providers | default: `http://<BASE_URL host>:8333` when `BASE_URL` is set, otherwise detected LAN host |
| `S3_ACCESS_KEY_ID` | S3 access key | auto-generated in embedded mode if unset |
| `S3_SECRET_ACCESS_KEY` | S3 secret key | auto-generated in embedded mode if unset |
| `S3_FORCE_PATH_STYLE` | Force path-style addressing | default: `true` in embedded mode |
| `S3_PREFIX` | Prefix for stored object keys | default: `openreader` |
</details>
<details>
<summary><strong>Docker volume mounts</strong> (Click to expand)</summary>
| Mount | Type | Recommended | Purpose | Example |
| --- | --- | --- | --- | --- |
| `/app/docstore` | Docker named volume | Yes | Persists server-side storage (documents, audiobook exports, settings, SQLite DB if used) | `-v openreader_docstore:/app/docstore` |
| `/app/docstore/library` | Bind mount (host folder) | Optional + `:ro` | Exposes an existing folder of documents for **Server Library Import** | `-v /path/to/your/library:/app/docstore/library:ro` |
| `/app/docstore` | Docker named volume | Yes (if you want persistence) | Persists embedded SeaweedFS blob data (`docstore/seaweedfs`), SQLite metadata DB (`docstore/sqlite3.db` when `POSTGRES_URL` is unset), audiobook export artifacts, and migration/runtime files | `-v openreader_docstore:/app/docstore` |
| `/app/docstore/library` | Bind mount (host folder) | Optional + `:ro` | Read-only source directory for **Server Library Import**; files are imported/copied into browser storage, not modified in place | `-v /path/to/your/library:/app/docstore/library:ro` |
To import from the mounted library: **Settings → Documents → Server Library Import**
@ -117,8 +164,9 @@ OpenReader WebUI is an open source text to speech document reader web app built
### 3. ⬆️ Updating Docker Image
```bash
docker stop openreader-webui && \
docker rm openreader-webui && \
docker stop openreader-webui || true && \
docker rm openreader-webui || true && \
docker image rm ghcr.io/richardr1126/openreader-webui:latest || true && \
docker pull ghcr.io/richardr1126/openreader-webui:latest
```
@ -194,7 +242,15 @@ docker run -d \
```
- A TTS API server (Kokoro-FastAPI, Orpheus-FastAPI, Deepinfra, OpenAI, etc.) running and accessible
Optionally required for different features:
- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) `weed` binary (required)
```bash
brew install seaweedfs
```
> **Note:** Verify install with `weed version`.
#### Optionally required for different features:
- [FFmpeg](https://ffmpeg.org) (required for audiobook m4b creation only)
```bash
@ -248,16 +304,52 @@ Optionally required for different features:
Auth is recommended for contributors and is enabled when **both** values are set:
- Set `BETTER_AUTH_URL` to your local URL (default: `http://localhost:3003`)
- Generate a `BETTER_AUTH_SECRET` and paste it into `.env`:
- Set `BASE_URL` to your local URL (default: `http://localhost:3003`)
- Generate a `AUTH_SECRET` and paste it into `.env`:
```bash
openssl rand -base64 32
```
- (Optional) If you use both localhost and LAN URL, set `AUTH_TRUSTED_ORIGINS` (leave empty to trust only `BASE_URL`):
> Note: To disable auth, remove either `BETTER_AUTH_URL` or `BETTER_AUTH_SECRET`.
```bash
AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://192.168.0.116:3003
```
The embedded weed mini object store is enabled by default for local development, and started through the shared entrypoint. The ACCESS_KEY_ID and SECRET_ACCESS_KEY are auto-generated if unset, but for stable credentials across restarts, you can generate and set them in `.env`:
- (Optional) Generate `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` and paste them into `.env` for stable credentials:
```bash
S3_ACCESS_KEY_ID=<`openssl rand -hex 16`>
S3_SECRET_ACCESS_KEY=<`openssl rand -hex 32`>
```
- (Optional) Connect to external S3-compatible storage instead of embedded `weed mini`:
```bash
USE_EMBEDDED_WEED_MINI=false
# Required
S3_BUCKET=your-bucket
S3_REGION=us-east-1
S3_ACCESS_KEY_ID=your-access-key
S3_SECRET_ACCESS_KEY=your-secret-key
# Optional / provider-specific
S3_ENDPOINT=
S3_FORCE_PATH_STYLE=
S3_PREFIX=openreader
```
Notes:
- For AWS S3: usually leave `S3_ENDPOINT` empty and `S3_FORCE_PATH_STYLE` empty/false.
- For MinIO/SeaweedFS/R2/B2 S3: set `S3_ENDPOINT` to your endpoint URL and set `S3_FORCE_PATH_STYLE=true` when required by that provider.
> Notes:
> - The base URL for the TTS API should be accessible and relative to the Next.js server
>
> Note: The base URL for the TTS API should be accessible and relative to the Next.js server
> - To disable auth, remove either `BASE_URL` or `AUTH_SECRET`.
>
> - If S3 credentials are unset, they are auto-generated per startup.
4. Run DB migrations:
@ -316,6 +408,10 @@ This project would not be possible without standing on the shoulders of these gi
- [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M) model
- [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI)
- [Better Auth](https://www.better-auth.com/)
- [SQLite](https://www.sqlite.org/)
- [PostgreSQL](https://www.postgresql.org/)
- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`)
- [whisper.cpp](https://github.com/ggerganov/whisper.cpp)
- [ffmpeg](https://ffmpeg.org)
- [react-pdf](https://github.com/wojtekmaj/react-pdf) npm package
@ -328,31 +424,26 @@ This project would not be possible without standing on the shoulders of these gi
## Stack
- **Framework:** Next.js (React)
- **Containerization:** Docker
- **Storage:**
- [Dexie.js](https://dexie.org/) IndexedDB wrapper for client-side storage
- **PDF:**
- [react-pdf](https://github.com/wojtekmaj/react-pdf)
- [pdf.js](https://mozilla.github.io/pdf.js/)
- **EPUB:**
- [react-reader](https://github.com/gerhardsletten/react-reader)
- [epubjs](https://github.com/futurepress/epub.js/)
- **Markdown/Text:**
- [react-markdown](https://github.com/remarkjs/react-markdown)
- [remark-gfm](https://github.com/remarkjs/remark-gfm)
- **UI:**
- [Tailwind CSS](https://tailwindcss.com)
- [Headless UI](https://headlessui.com)
- [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin)
- **TTS:** (tested on)
- [Deepinfra API](https://deepinfra.com) (Kokoro-82M, Orpheus-3B, Sesame-1B)
- [Kokoro FastAPI TTS](https://github.com/remsky/Kokoro-FastAPI/tree/v0.0.5post1-stable)
- [Orpheus FastAPI TTS](https://github.com/Lex-au/Orpheus-FastAPI)
- **NLP:**
- [compromise](https://github.com/spencermountain/compromise) NLP library for sentence splitting
- [cmpstr](https://github.com/remsky/cmpstr) String comparison library
- [whisper.cpp](https://github.com/ggerganov/whisper.cpp) for TTS timestamps (word-by-word highlighting)
- **Framework:** [Next.js](https://nextjs.org/) 15 (App Router), [React](https://react.dev/) 19, [TypeScript](https://www.typescriptlang.org/)
- **Containerization / Runtime:** [Docker](https://www.docker.com/) (linux/amd64 + linux/arm64), with a shared entrypoint that can bootstrap embedded SeaweedFS before app startup
- **Next.js Client:**
- **UI:** [Tailwind CSS](https://tailwindcss.com), [Headless UI](https://headlessui.com), [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin)
- **Interactions:** `react-dnd`, `react-dropzone`
- **Authentication:** [Better Auth](https://www.better-auth.com/) client SDK (`better-auth/react`, anonymous client plugin)
- **Local storage/cache:** [Dexie.js](https://dexie.org/) (IndexedDB) for documents, cache, and app settings
- **Document rendering:**
- **PDF:** [react-pdf](https://github.com/wojtekmaj/react-pdf), [pdf.js](https://mozilla.github.io/pdf.js/)
- **EPUB:** [react-reader](https://github.com/gerhardsletten/react-reader), [epubjs](https://github.com/futurepress/epub.js/)
- **Markdown/Text:** [react-markdown](https://github.com/remarkjs/react-markdown), [remark-gfm](https://github.com/remarkjs/remark-gfm)
- **Text preprocessing / matching:** [compromise](https://github.com/spencermountain/compromise), [cmpstr](https://github.com/remsky/cmpstr)
- **Next.js Server:**
- **APIs:** Next.js Route Handlers for document sync, blob/content access, migration flows, audiobook export, and TTS/Whisper proxying
- **Authentication:** [Better Auth](https://www.better-auth.com/) server handlers/adapters for session and auth routing
- **Text preprocessing / NLP utilities:** [compromise](https://github.com/spencermountain/compromise) via shared `lib/nlp` helpers used in server processing paths
- **Metadata database:** [Drizzle ORM](https://orm.drizzle.team/), [SQLite](https://www.sqlite.org/) (`better-sqlite3`) by default, optional [PostgreSQL](https://www.postgresql.org/) (`pg`)
- **Blob/object storage:** embedded [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`) by default, or external S3-compatible storage via AWS SDK v3 (`@aws-sdk/client-s3`, presigned URLs, upload fallback proxy)
- **Audio/processing pipeline:** OpenAI-compatible TTS providers (OpenAI, DeepInfra, Kokoro, Orpheus, custom), [ffmpeg](https://ffmpeg.org/) for audiobook assembly, optional [whisper.cpp](https://github.com/ggerganov/whisper.cpp) for word-level timestamps
- **Tooling / Testing:** ESLint, TypeScript, [Playwright](https://playwright.dev/) end-to-end tests, and Drizzle migrations/generation scripts
## License

View file

@ -1,30 +0,0 @@
services:
kokoro-tts:
container_name: kokoro-tts
image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4
ports:
- "8880:8880"
environment:
# ONNX Optimization Settings for vectorized operations
- ONNX_NUM_THREADS=8 # Maximize core usage for vectorized ops
- ONNX_INTER_OP_THREADS=4 # Higher inter-op for parallel matrix operations
- ONNX_EXECUTION_MODE=parallel
- ONNX_OPTIMIZATION_LEVEL=all
- ONNX_MEMORY_PATTERN=true
- ONNX_ARENA_EXTEND_STRATEGY=kNextPowerOfTwo
- API_LOG_LEVEL=DEBUG
restart: unless-stopped
openreader-webui:
container_name: openreader-webui
image: ghcr.io/richardr1126/openreader-webui:latest
environment:
- API_BASE=http://host.docker.internal:8880/v1
ports:
- "3003:3003"
volumes:
- docstore:/app/docstore
restart: unless-stopped
volumes:
docstore:

View file

@ -3,15 +3,19 @@
"version": "v1.3.0",
"private": true,
"scripts": {
"dev": "next dev --turbopack -p 3003",
"dev": "node scripts/openreader-entrypoint.mjs -- pnpm dev:raw",
"dev:raw": "next dev --turbopack -p 3003",
"build": "next build",
"start": "node drizzle/scripts/migrate.mjs && next start -p 3003",
"start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw",
"start:raw": "next start -p 3003",
"lint": "next lint",
"test": "playwright test",
"migrate": "node drizzle/scripts/migrate.mjs",
"generate": "node drizzle/scripts/generate.mjs"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.985.0",
"@aws-sdk/s3-request-presigner": "^3.985.0",
"@headlessui/react": "^2.2.9",
"@types/howler": "^2.2.12",
"@types/uuid": "^10.0.0",

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,367 @@
#!/usr/bin/env node
import { spawn, spawnSync } from 'node:child_process';
import { randomBytes } from 'node:crypto';
import { once } from 'node:events';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import process from 'node:process';
import { setTimeout as delay } from 'node:timers/promises';
import * as dotenv from 'dotenv';
function loadEnvFiles() {
const cwd = process.cwd();
const envPath = path.join(cwd, '.env');
const envLocalPath = path.join(cwd, '.env.local');
if (fs.existsSync(envPath)) {
dotenv.config({ path: envPath });
}
if (fs.existsSync(envLocalPath)) {
dotenv.config({ path: envLocalPath, override: true });
}
}
function isTrue(value, defaultValue) {
if (value == null || value.trim() === '') return defaultValue;
const normalized = value.trim().toLowerCase();
return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
}
function withDefault(value, fallback) {
return value && value.trim() ? value.trim() : fallback;
}
function isPrivateIPv4(address) {
if (!address) return false;
if (address.startsWith('10.')) return true;
if (address.startsWith('192.168.')) return true;
const m = /^172\.(\d+)\./.exec(address);
if (m) {
const second = Number.parseInt(m[1], 10);
if (second >= 16 && second <= 31) return true;
}
return false;
}
function detectHostForDefaultEndpoint() {
const interfaces = os.networkInterfaces();
const ipv4 = [];
for (const entries of Object.values(interfaces)) {
for (const entry of entries || []) {
if (!entry) continue;
const family = typeof entry.family === 'string' ? entry.family : String(entry.family);
if (family !== 'IPv4') continue;
if (entry.internal) continue;
ipv4.push(entry.address);
}
}
const privateAddr = ipv4.find(isPrivateIPv4);
if (privateAddr) return privateAddr;
if (ipv4[0]) return ipv4[0];
return '127.0.0.1';
}
function parseS3Endpoint(endpoint) {
let url;
try {
url = new URL(endpoint);
} catch {
throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`);
}
if (!url.hostname) {
throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`);
}
const port = Number.parseInt(url.port || '8333', 10);
if (!Number.isFinite(port) || port < 1 || port > 65535) {
throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`);
}
return {
hostname: url.hostname,
port,
normalized: `${url.protocol}//${url.hostname}:${port}`,
};
}
function parseUrlHost(urlValue, fieldName) {
let url;
try {
url = new URL(urlValue);
} catch {
throw new Error(`Invalid ${fieldName}: ${urlValue}`);
}
if (!url.hostname) {
throw new Error(`Invalid ${fieldName} host: ${urlValue}`);
}
return url.hostname;
}
function parseCommandFromArgs(argv) {
const marker = argv.indexOf('--');
if (marker >= 0) return argv.slice(marker + 1);
return argv;
}
function forwardChildStream(stream, target) {
if (!stream) return () => {};
const onData = (chunk) => {
target.write(chunk);
};
stream.on('data', onData);
return () => {
stream.off('data', onData);
};
}
function hasWeedBinary() {
const probe = spawnSync('weed', ['version'], { stdio: 'ignore' });
if (probe.error) return false;
return true;
}
async function waitForEndpoint(url, timeoutSeconds) {
const waitMs = Math.max(1, timeoutSeconds) * 1000;
const deadline = Date.now() + waitMs;
while (Date.now() < deadline) {
try {
const res = await fetch(url, { method: 'GET' });
if (res) return;
} catch {
// retry
}
await delay(1000);
}
throw new Error(`Embedded weed mini did not become ready at ${url} within ${timeoutSeconds}s.`);
}
function spawnMainCommand(command, env) {
const [cmd, ...args] = command;
const child = spawn(cmd, args, {
env,
stdio: 'inherit',
shell: process.platform === 'win32',
});
const exitPromise = new Promise((resolve) => {
child.on('error', (error) => {
console.error('Failed to launch command:', error);
resolve(1);
});
child.on('exit', (code, signal) => {
if (typeof code === 'number') {
resolve(code);
return;
}
if (signal) {
resolve(1);
return;
}
resolve(0);
});
});
return { child, exitPromise };
}
function runDbMigrations(env) {
const migrateScript = path.join(process.cwd(), 'drizzle', 'scripts', 'migrate.mjs');
if (!fs.existsSync(migrateScript)) {
throw new Error(`Could not find migration script at ${migrateScript}`);
}
console.log('Running database migrations...');
const migration = spawnSync(process.execPath, [migrateScript], {
env,
stdio: 'inherit',
});
if (migration.error) {
throw migration.error;
}
if (typeof migration.status === 'number' && migration.status !== 0) {
throw new Error(`Database migrations failed with exit code ${migration.status}.`);
}
}
function sendSignal(child, signal, useProcessGroup) {
if (!child) return false;
if (useProcessGroup && process.platform !== 'win32' && typeof child.pid === 'number' && child.pid > 0) {
try {
process.kill(-child.pid, signal);
return true;
} catch {
return false;
}
}
try {
child.kill(signal);
return true;
} catch {
return false;
}
}
async function terminateChild(child, signal = 'SIGTERM', graceMs = 3000, useProcessGroup = false) {
if (!child) return;
if (child.exitCode != null) return;
if (!sendSignal(child, signal, useProcessGroup)) return;
const exited = await Promise.race([
once(child, 'exit').then(() => true).catch(() => true),
delay(graceMs).then(() => false),
]);
if (exited) return;
if (!sendSignal(child, 'SIGKILL', useProcessGroup)) return;
await Promise.race([
once(child, 'exit').then(() => true).catch(() => true),
delay(1000).then(() => false),
]);
}
async function main() {
loadEnvFiles();
const command = parseCommandFromArgs(process.argv.slice(2));
if (command.length === 0) {
console.error('Usage: node scripts/openreader-entrypoint.mjs -- <command> [args]');
process.exit(2);
}
const embeddedEnvRaw = process.env.USE_EMBEDDED_WEED_MINI;
let useEmbeddedWeed = isTrue(embeddedEnvRaw, true);
if (useEmbeddedWeed && !hasWeedBinary()) {
if (embeddedEnvRaw && isTrue(embeddedEnvRaw, true)) {
console.error('USE_EMBEDDED_WEED_MINI=true but `weed` binary is not available in PATH.');
process.exit(1);
}
useEmbeddedWeed = false;
console.warn('`weed` binary not found; skipping embedded SeaweedFS startup.');
}
const runtimeEnv = { ...process.env };
let weedProc = null;
let weedExitPromise = Promise.resolve();
let appProc = null;
let shutdownPromise = null;
let stopWeedStdoutForward = () => {};
let stopWeedStderrForward = () => {};
let didExit = false;
const exitOnce = (code) => {
if (didExit) return;
didExit = true;
process.exit(code);
};
const shutdown = async (signal = 'SIGTERM') => {
if (shutdownPromise) return shutdownPromise;
shutdownPromise = (async () => {
await Promise.all([
terminateChild(appProc, signal, 4000),
terminateChild(weedProc, 'SIGTERM', 4000),
]);
await weedExitPromise;
stopWeedStdoutForward();
stopWeedStderrForward();
})();
return shutdownPromise;
};
process.once('SIGINT', () => {
void shutdown('SIGINT').finally(() => exitOnce(130));
});
process.once('SIGTERM', () => {
void shutdown('SIGTERM').finally(() => exitOnce(143));
});
try {
const shouldRunDbMigrations = isTrue(runtimeEnv.RUN_DB_MIGRATIONS, true);
if (shouldRunDbMigrations) {
runDbMigrations(runtimeEnv);
}
if (useEmbeddedWeed) {
runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, 'docstore/seaweedfs');
runtimeEnv.WEED_MINI_WAIT_SEC = withDefault(runtimeEnv.WEED_MINI_WAIT_SEC, '20');
runtimeEnv.S3_BUCKET = withDefault(runtimeEnv.S3_BUCKET, 'openreader-documents');
runtimeEnv.S3_REGION = withDefault(runtimeEnv.S3_REGION, 'us-east-1');
const configuredBaseUrl = runtimeEnv.BASE_URL?.trim() || '';
const baseUrlHost = configuredBaseUrl ? parseUrlHost(configuredBaseUrl, 'BASE_URL') : '';
const configuredS3Endpoint = runtimeEnv.S3_ENDPOINT?.trim() || '';
const defaultS3Host = baseUrlHost || detectHostForDefaultEndpoint();
runtimeEnv.S3_ENDPOINT = configuredS3Endpoint || `http://${defaultS3Host}:8333`;
runtimeEnv.S3_FORCE_PATH_STYLE = withDefault(runtimeEnv.S3_FORCE_PATH_STYLE, 'true');
runtimeEnv.S3_PREFIX = withDefault(runtimeEnv.S3_PREFIX, 'openreader');
runtimeEnv.S3_ACCESS_KEY_ID = withDefault(runtimeEnv.S3_ACCESS_KEY_ID, randomBytes(16).toString('hex'));
runtimeEnv.S3_SECRET_ACCESS_KEY = withDefault(runtimeEnv.S3_SECRET_ACCESS_KEY, randomBytes(32).toString('hex'));
runtimeEnv.AWS_ACCESS_KEY_ID = runtimeEnv.S3_ACCESS_KEY_ID;
runtimeEnv.AWS_SECRET_ACCESS_KEY = runtimeEnv.S3_SECRET_ACCESS_KEY;
if (!runtimeEnv.S3_ACCESS_KEY_ID || !runtimeEnv.S3_SECRET_ACCESS_KEY) {
throw new Error('Failed to initialize embedded S3 credentials.');
}
fs.mkdirSync(runtimeEnv.WEED_MINI_DIR, { recursive: true });
console.log('Starting embedded SeaweedFS weed mini...');
const weedArgs = ['mini', `-dir=${runtimeEnv.WEED_MINI_DIR}`];
if (configuredS3Endpoint || baseUrlHost) {
const endpoint = parseS3Endpoint(runtimeEnv.S3_ENDPOINT);
weedArgs.push(`-ip=${endpoint.hostname}`);
weedArgs.push(`-s3.port=${endpoint.port}`);
}
weedProc = spawn('weed', weedArgs, {
env: runtimeEnv,
stdio: ['ignore', 'pipe', 'pipe'],
});
stopWeedStdoutForward = forwardChildStream(weedProc.stdout, process.stdout);
stopWeedStderrForward = forwardChildStream(weedProc.stderr, process.stderr);
weedExitPromise = once(weedProc, 'exit').then(() => undefined).catch(() => undefined);
weedProc.on('exit', (code, signal) => {
if (typeof code === 'number' && code !== 0) {
console.error(`Embedded weed mini exited with code ${code}.`);
return;
}
if (signal) {
console.error(`Embedded weed mini exited due to signal ${signal}.`);
}
});
const endpoint = runtimeEnv.S3_ENDPOINT;
const waitSec = Number.parseInt(runtimeEnv.WEED_MINI_WAIT_SEC || '20', 10);
await waitForEndpoint(endpoint, Number.isFinite(waitSec) ? waitSec : 20);
console.log(`Embedded SeaweedFS is ready at ${endpoint}`);
}
const { child, exitPromise } = spawnMainCommand(command, runtimeEnv);
appProc = child;
const exitCode = await exitPromise;
await shutdown('SIGTERM');
exitOnce(exitCode);
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
await shutdown('SIGTERM');
exitOnce(1);
}
}
await main();

View file

@ -2,34 +2,18 @@ import { NextRequest, NextResponse } from 'next/server';
import { createReadStream, existsSync } from 'fs';
import { readdir, unlink } from 'fs/promises';
import { join } from 'path';
import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
import { findStoredChapterByIndex } from '@/lib/server/audiobook';
import { db } from '@/db';
import { audiobooks, audiobookChapters } from '@/db/schema';
import { and, eq, inArray } from 'drizzle-orm';
import { requireAuthContext } from '@/lib/server/auth';
import { ensureDbIndexed } from '@/lib/server/db-indexing';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { pruneAudiobookChapterIfMissingFile, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
export const dynamic = 'force-dynamic';
/**
* Get the base audiobooks directory, accounting for test namespaces.
* When auth is disabled, returns AUDIOBOOKS_V1_DIR.
* When auth is enabled, returns the user-specific directory.
*/
function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string {
const namespace = getOpenReaderTestNamespace(request.headers);
if (!authEnabled || !userId) {
return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace);
}
const userDir = getUserAudiobookDir(userId);
return applyOpenReaderTestNamespacePath(userDir, namespace);
}
export async function GET(request: NextRequest) {
try {
const bookId = request.nextUrl.searchParams.get('bookId');
@ -77,7 +61,14 @@ export async function GET(request: NextRequest) {
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
}
const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`);
const intermediateDir = join(
getAudiobooksRootDir({
userId: existingBook.userId,
authEnabled,
namespace: testNamespace,
}),
`${bookId}-audiobook`,
);
const dirExists = existsSync(intermediateDir);
if (!dirExists) {
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);
@ -185,7 +176,14 @@ export async function DELETE(request: NextRequest) {
),
);
const intermediateDir = join(getAudiobooksRootDir(request, storageUserId, authEnabled), `${bookId}-audiobook`);
const intermediateDir = join(
getAudiobooksRootDir({
userId: storageUserId,
authEnabled,
namespace: testNamespace,
}),
`${bookId}-audiobook`,
);
const chapterPrefix = `${String(chapterIndex + 1).padStart(4, '0')}__`;
const files = await readdir(intermediateDir).catch(() => []);
for (const file of files) {

View file

@ -4,7 +4,7 @@ import { readFile, writeFile, mkdir, unlink, rm, rename, readdir } from 'fs/prom
import { existsSync, createReadStream } from 'fs';
import { basename, join } from 'path';
import { randomUUID } from 'crypto';
import { AUDIOBOOKS_V1_DIR, ensureAudiobooksV1Ready, isAudiobooksV1Ready, getUserAudiobookDir } from '@/lib/server/docstore';
import { ensureAudiobooksV1Ready, isAudiobooksV1Ready, getAudiobooksRootDir } from '@/lib/server/docstore';
import { encodeChapterFileName, encodeChapterTitleTag, listStoredChapters, ffprobeAudio, escapeFFMetadata } from '@/lib/server/audiobook';
import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
import type { AudiobookGenerationSettings } from '@/types/client';
@ -13,35 +13,11 @@ import { audiobooks, audiobookChapters } from '@/db/schema';
import { eq, and, inArray } from 'drizzle-orm';
import { requireAuthContext } from '@/lib/server/auth';
import { ensureDbIndexed } from '@/lib/server/db-indexing';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
export const dynamic = 'force-dynamic';
/**
* Apply test namespace to a directory path if present in request headers.
*/
function applyTestNamespace(baseDir: string, request: NextRequest): string {
const namespace = getOpenReaderTestNamespace(request.headers);
return applyOpenReaderTestNamespacePath(baseDir, namespace);
}
/**
* Get the base audiobooks directory, accounting for test namespaces.
* When auth is disabled, returns AUDIOBOOKS_V1_DIR (possibly with test namespace).
* When auth is enabled, returns the user-specific directory under AUDIOBOOKS_USERS_DIR.
*/
function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string {
// When auth is disabled, use the flat audiobooks_v1 directory
if (!authEnabled || !userId) {
return applyTestNamespace(AUDIOBOOKS_V1_DIR, request);
}
// When auth is enabled, use user-specific directory
const userDir = getUserAudiobookDir(userId);
return applyTestNamespace(userDir, request);
}
interface ConversionRequest {
chapterTitle: string;
buffer: TTSAudioBytes;
@ -207,7 +183,14 @@ export async function POST(request: NextRequest) {
})
.onConflictDoNothing();
const intermediateDir = join(getAudiobooksRootDir(request, userId, ctxOrRes.authEnabled), `${bookId}-audiobook`);
const intermediateDir = join(
getAudiobooksRootDir({
userId,
authEnabled: ctxOrRes.authEnabled,
namespace: testNamespace,
}),
`${bookId}-audiobook`,
);
// Create intermediate directory
await mkdir(intermediateDir, { recursive: true });
@ -434,7 +417,11 @@ export async function GET(request: NextRequest) {
}
const intermediateDir = join(
getAudiobooksRootDir(request, existingBook.userId, authEnabled),
getAudiobooksRootDir({
userId: existingBook.userId,
authEnabled,
namespace: testNamespace,
}),
`${bookId}-audiobook`,
);
@ -657,7 +644,14 @@ export async function DELETE(request: NextRequest) {
await db.delete(audiobooks).where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId)));
const intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`);
const intermediateDir = join(
getAudiobooksRootDir({
userId,
authEnabled,
namespace: testNamespace,
}),
`${bookId}-audiobook`,
);
// If directory doesn't exist, consider it already reset
if (!existsSync(intermediateDir)) {

View file

@ -1,7 +1,7 @@
import { NextRequest, NextResponse } from 'next/server';
import { existsSync } from 'fs';
import { join } from 'path';
import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
import { listStoredChapters } from '@/lib/server/audiobook';
import type { AudiobookGenerationSettings } from '@/types/client';
import type { TTSAudiobookFormat, TTSAudiobookChapter } from '@/types/tts';
@ -11,27 +11,11 @@ import { db } from '@/db';
import { audiobooks } from '@/db/schema';
import { eq, and, inArray } from 'drizzle-orm';
import { ensureDbIndexed } from '@/lib/server/db-indexing';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { pruneAudiobookChaptersNotOnDisk, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
export const dynamic = 'force-dynamic';
/**
* Get the base audiobooks directory, accounting for test namespaces.
* When auth is disabled, returns AUDIOBOOKS_V1_DIR.
* When auth is enabled, returns the user-specific directory.
*/
function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string {
const namespace = getOpenReaderTestNamespace(request.headers);
if (!authEnabled || !userId) {
return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace);
}
const userDir = getUserAudiobookDir(userId);
return applyOpenReaderTestNamespacePath(userDir, namespace);
}
const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
function isSafeId(value: string): boolean {
@ -80,7 +64,14 @@ export async function GET(request: NextRequest) {
});
}
const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`);
const intermediateDir = join(
getAudiobooksRootDir({
userId: existingBook.userId,
authEnabled,
namespace: testNamespace,
}),
`${bookId}-audiobook`,
);
if (!existsSync(intermediateDir)) {
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);

View file

@ -1,16 +1,13 @@
import { open, readFile } from 'fs/promises';
import path from 'path';
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
import { requireAuthContext } from '@/lib/server/auth';
import { ensureDbIndexed } from '@/lib/server/db-indexing';
import { contentTypeForName } from '@/lib/server/library';
import { extractRawTextSnippet } from '@/lib/text-snippets';
import { isEnoent } from '@/lib/server/documents-utils';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { getDocumentBlob, getDocumentRange, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
@ -19,67 +16,55 @@ function clampInt(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, Math.trunc(value)));
}
async function readHeadBuffer(filePath: string, maxBytes: number): Promise<Buffer> {
const handle = await open(filePath, 'r');
try {
const buf = Buffer.allocUnsafe(maxBytes);
const result = await handle.read(buf, 0, maxBytes, 0);
return buf.subarray(0, result.bytesRead);
} finally {
await handle.close();
}
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
export async function GET(req: NextRequest) {
try {
await ensureDocumentsV1Ready();
if (!(await isDocumentsV1Ready())) {
return NextResponse.json(
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
{ status: 409 },
);
}
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
await ensureDbIndexed();
const url = new URL(req.url);
const id = url.searchParams.get('id');
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
const format = (url.searchParams.get('format') || '').toLowerCase().trim();
if (!id) {
return NextResponse.json({ error: 'Missing id' }, { status: 400 });
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
}
const docs = await db
const rows = (await db
.select({ id: documents.id, userId: documents.userId, name: documents.name, filePath: documents.filePath })
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
name: string;
filePath: string;
}>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const doc = docs.find((d: any) => d.userId === storageUserId) ?? docs[0];
if (!doc?.filePath) {
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
if (!doc) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
const filePath = path.join(documentsDir, doc.filePath);
const filename = doc.name || doc.filePath;
const filename = doc.name || `${id}.bin`;
const responseType = contentTypeForName(filename);
if (format === 'snippet') {
const maxChars = clampInt(Number.parseInt(url.searchParams.get('maxChars') || '1600', 10), 100, 8000);
const maxBytes = clampInt(Number.parseInt(url.searchParams.get('maxBytes') || '131072', 10), 4096, 1024 * 1024);
try {
const head = await readHeadBuffer(filePath, maxBytes);
const head = await getDocumentRange(id, 0, maxBytes - 1, testNamespace);
const decoded = new TextDecoder().decode(new Uint8Array(head));
const snippet = extractRawTextSnippet(decoded, maxChars);
return NextResponse.json(
@ -87,7 +72,7 @@ export async function GET(req: NextRequest) {
{ headers: { 'Cache-Control': 'no-store' } },
);
} catch (error) {
if (isEnoent(error)) {
if (isMissingBlobError(error)) {
await db
.delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
@ -97,14 +82,22 @@ export async function GET(req: NextRequest) {
}
}
let content: ArrayBuffer;
try {
const buf = await readFile(filePath);
content = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
const content = await getDocumentBlob(id, testNamespace);
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(content));
controller.close();
},
});
return new NextResponse(stream, {
headers: {
'Content-Type': responseType,
'Cache-Control': 'no-store',
},
});
} catch (error) {
if (isEnoent(error)) {
// The DB can become stale if a file is deleted manually from the docstore.
// Prune rows so the client stops showing ghost documents.
if (isMissingBlobError(error)) {
await db
.delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
@ -112,13 +105,6 @@ export async function GET(req: NextRequest) {
}
throw error;
}
return new NextResponse(content, {
headers: {
'Content-Type': contentTypeForName(filename),
'Cache-Control': 'no-store',
},
});
} catch (error) {
console.error('Error loading document content:', error);
return NextResponse.json({ error: 'Failed to load document content' }, { status: 500 });

View file

@ -0,0 +1,51 @@
import { NextRequest, NextResponse } from 'next/server';
import { requireAuthContext } from '@/lib/server/auth';
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore';
import { isS3Configured } from '@/lib/server/s3';
import { getOpenReaderTestNamespace } from '@/lib/server/test-namespace';
export const dynamic = 'force-dynamic';
function isPreconditionFailed(error: unknown): boolean {
if (!error || typeof error !== 'object') return false;
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
}
export async function PUT(req: NextRequest) {
try {
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const url = new URL(req.url);
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
}
const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream';
const body = Buffer.from(await req.arrayBuffer());
const namespace = getOpenReaderTestNamespace(req.headers);
try {
await putDocumentBlob(id, body, contentType, namespace);
} catch (error) {
if (!isPreconditionFailed(error)) {
throw error;
}
}
return NextResponse.json({ success: true, id });
} catch (error) {
console.error('Error proxy-uploading document blob:', error);
return NextResponse.json({ error: 'Failed to upload document blob' }, { status: 500 });
}
}

View file

@ -0,0 +1,72 @@
import { NextRequest, NextResponse } from 'next/server';
import { requireAuthContext } from '@/lib/server/auth';
import { isValidDocumentId, presignPut } from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
export const dynamic = 'force-dynamic';
type PresignUpload = {
id: string;
contentType: string;
size: number;
};
function parseUploads(body: unknown): PresignUpload[] {
if (!body || typeof body !== 'object') return [];
const rawUploads = (body as { uploads?: unknown }).uploads;
if (!Array.isArray(rawUploads)) return [];
const uploads: PresignUpload[] = [];
for (const raw of rawUploads) {
if (!raw || typeof raw !== 'object') continue;
const rec = raw as Record<string, unknown>;
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
if (!isValidDocumentId(id)) continue;
const contentType =
typeof rec.contentType === 'string' && rec.contentType.trim()
? rec.contentType.trim()
: 'application/octet-stream';
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
uploads.push({ id, contentType, size });
}
return uploads;
}
export async function POST(req: NextRequest) {
try {
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const body = await req.json().catch(() => null);
const uploads = parseUploads(body);
if (uploads.length === 0) {
return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 });
}
const namespace = getOpenReaderTestNamespace(req.headers);
const signed = await Promise.all(
uploads.map(async (upload) => {
const res = await presignPut(upload.id, upload.contentType, namespace);
return {
id: upload.id,
url: res.url,
headers: res.headers,
};
}),
);
return NextResponse.json({ uploads: signed });
} catch (error) {
console.error('Error creating document upload signatures:', error);
return NextResponse.json({ error: 'Failed to presign uploads' }, { status: 500 });
}
}

View file

@ -6,11 +6,12 @@ import { existsSync } from 'fs';
import { randomUUID, createHash } from 'crypto';
import { pathToFileURL } from 'url';
import { requireAuthContext } from '@/lib/server/auth';
import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { safeDocumentName, trySetFileMtime } from '@/lib/server/documents-utils';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { safeDocumentName } from '@/lib/server/documents-utils';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
import { putDocumentBlob } from '@/lib/server/documents-blobstore';
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
@ -66,18 +67,15 @@ async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100)
export async function POST(req: NextRequest) {
try {
await ensureDocumentsV1Ready();
if (!(await isDocumentsV1Ready())) {
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
{ status: 409 },
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
const testNamespace = getOpenReaderTestNamespace(req.headers);
const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
await mkdir(documentsDir, { recursive: true });
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
@ -96,7 +94,7 @@ export async function POST(req: NextRequest) {
}
const docxBytes = Buffer.from(await file.arrayBuffer());
// IMPORTANT: use sha of the source DOCX bytes for a stable ID across conversions.
// Keep stable IDs tied to source bytes.
const id = createHash('sha256').update(docxBytes).digest('hex');
const tempId = randomUUID();
@ -113,18 +111,19 @@ export async function POST(req: NextRequest) {
const pdfPath = await waitForPdfReady(jobDir);
const pdfContent = await readFile(pdfPath);
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
const targetFileName = `${id}__${encodeURIComponent(derivedName)}`;
const targetPath = path.join(documentsDir, targetFileName);
try {
await stat(targetPath);
} catch {
await writeFile(targetPath, pdfContent);
await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace);
} catch (error) {
// Idempotent behavior: if blob already exists for this sha, continue.
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined;
const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed';
if (!isPreconditionFailed) {
throw error;
}
}
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
await trySetFileMtime(targetPath, lastModified);
await db
.insert(documents)
@ -135,9 +134,18 @@ export async function POST(req: NextRequest) {
type: 'pdf',
size: pdfContent.length,
lastModified,
filePath: targetFileName,
filePath: id,
})
.onConflictDoNothing();
.onConflictDoUpdate({
target: [documents.id, documents.userId],
set: {
name: derivedName,
type: 'pdf',
size: pdfContent.length,
lastModified,
filePath: id,
},
});
return NextResponse.json({
stored: {

View file

@ -1,212 +1,207 @@
import { createHash } from 'crypto';
import { mkdir, readFile, stat, unlink, writeFile } from 'fs/promises';
import { existsSync } from 'fs';
import { NextRequest, NextResponse } from 'next/server';
import path from 'path';
import { DOCUMENTS_V1_DIR, ensureDocumentsV1Ready, isDocumentsV1Ready } from '@/lib/server/docstore';
import type { BaseDocument, DocumentType, SyncedDocument } from '@/types/documents';
import { and, count, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { eq, and, inArray, count } from 'drizzle-orm';
import { requireAuthContext } from '@/lib/server/auth';
import { ensureDbIndexed } from '@/lib/server/db-indexing';
import { isEnoent, toDocumentTypeFromName, trySetFileMtime } from '@/lib/server/documents-utils';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents-utils';
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
import { isS3Configured } from '@/lib/server/s3';
import type { BaseDocument, DocumentType } from '@/types/documents';
export const dynamic = 'force-dynamic';
type RegisterDocument = {
id: string;
name: string;
type: DocumentType;
size: number;
lastModified: number;
};
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType {
if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') {
return rawType;
}
return toDocumentTypeFromName(safeName);
}
function normalizeLastModified(value: unknown): number {
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
}
function parseDocumentPayload(body: unknown): RegisterDocument[] {
if (!body || typeof body !== 'object') return [];
const rawDocs = (body as { documents?: unknown }).documents;
if (!Array.isArray(rawDocs)) return [];
const docs: RegisterDocument[] = [];
for (const rawDoc of rawDocs) {
if (!rawDoc || typeof rawDoc !== 'object') continue;
const rec = rawDoc as Record<string, unknown>;
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
if (!isValidDocumentId(id)) continue;
const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`;
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
const type = normalizeDocumentType(rec.type, name);
const lastModified = normalizeLastModified(rec.lastModified);
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
docs.push({ id, name, type, size, lastModified });
}
return docs;
}
export async function POST(req: NextRequest) {
try {
await ensureDocumentsV1Ready();
if (!(await isDocumentsV1Ready())) {
return NextResponse.json(
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
{ status: 409 },
);
}
const testNamespace = getOpenReaderTestNamespace(req.headers);
const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
await mkdir(syncDir, { recursive: true });
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const data = await req.json();
const documentsData = data.documents as SyncedDocument[];
const stored: Array<{ oldId: string; id: string; name: string }> = [];
const body = await req.json().catch(() => null);
const documentsData = parseDocumentPayload(body);
if (documentsData.length === 0) {
return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 });
}
// Ensure directory exists (redundant with isDocumentsV1Ready but safe)
// const SYNC_DIR = DOCUMENTS_V1_DIR;
const stored: BaseDocument[] = [];
for (const doc of documentsData) {
const content = Buffer.from(new Uint8Array(doc.data));
const id = createHash('sha256').update(content).digest('hex');
const baseName = path.basename(doc.name || `${id}.${doc.type}`);
const safeName = baseName.replaceAll('\u0000', '').slice(0, 240) || `${id}.${doc.type}`;
const targetFileName = `${id}__${encodeURIComponent(safeName)}`;
const targetPath = path.join(syncDir, targetFileName);
// Write file if not exists
let headSize = doc.size;
try {
await stat(targetPath);
} catch {
await writeFile(targetPath, content);
const head = await headDocumentBlob(doc.id, testNamespace);
if (head.contentLength > 0) headSize = head.contentLength;
} catch (error) {
if (isMissingBlobError(error)) {
return NextResponse.json(
{
error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`,
},
{ status: 409 },
);
}
throw error;
}
await trySetFileMtime(targetPath, doc.lastModified);
// DB Upsert
// With composite PK (id, userId), we check if THIS user already has this document
await db
.insert(documents)
.values({
id,
id: doc.id,
userId: storageUserId,
name: safeName,
name: doc.name,
type: doc.type,
size: content.length,
size: headSize,
lastModified: doc.lastModified,
filePath: targetFileName,
filePath: doc.id,
})
.onConflictDoNothing();
.onConflictDoUpdate({
target: [documents.id, documents.userId],
set: {
name: doc.name,
type: doc.type,
size: headSize,
lastModified: doc.lastModified,
filePath: doc.id,
},
});
stored.push({ oldId: doc.id, id, name: safeName });
stored.push({
id: doc.id,
name: doc.name,
type: doc.type,
size: headSize,
lastModified: doc.lastModified,
scope: storageUserId === unclaimedUserId ? 'unclaimed' : 'user',
});
}
return NextResponse.json({ success: true, stored });
} catch (error) {
console.error('Error saving documents:', error);
return NextResponse.json({ error: 'Failed to save documents' }, { status: 500 });
console.error('Error registering documents:', error);
return NextResponse.json({ error: 'Failed to register documents' }, { status: 500 });
}
}
export async function GET(req: NextRequest) {
try {
await ensureDocumentsV1Ready();
if (!(await isDocumentsV1Ready())) {
return NextResponse.json(
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
{ status: 409 },
);
}
if (!isS3Configured()) return s3NotConfiguredResponse();
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
await ensureDbIndexed();
const url = new URL(req.url);
const list = url.searchParams.get('list') === 'true';
const format = url.searchParams.get('format');
const idsParam = url.searchParams.get('ids');
const targetIds = idsParam
? idsParam
.split(',')
.map((id) => id.trim().toLowerCase())
.filter((id) => isValidDocumentId(id))
: null;
// If list=true, force metadata only.
// If format=metadata, force metadata only.
// Otherwise include data.
const includeData = !list && format !== 'metadata';
const targetIds = idsParam ? idsParam.split(',').filter(Boolean) : null;
// Query database for documents the user is allowed to access
let allowedDocs: { id: string; userId: string; name: string; type: string; size: number; lastModified: number; filePath: string }[] = [];
if (idsParam && (!targetIds || targetIds.length === 0)) {
return NextResponse.json({ documents: [] });
}
const conditions = [
inArray(documents.userId, allowedUserIds),
...(targetIds ? [inArray(documents.id, targetIds)] : []),
...(targetIds && targetIds.length > 0 ? [inArray(documents.id, targetIds)] : []),
];
const rows = await db.select().from(documents).where(and(...conditions));
allowedDocs = rows as unknown as { id: string; userId: string; name: string; type: string; size: number; lastModified: number; filePath: string }[];
const rows = (await db.select().from(documents).where(and(...conditions))) as Array<{
id: string;
userId: string;
name: string;
type: string;
size: number;
lastModified: number;
filePath: string;
}>;
const results: (BaseDocument | SyncedDocument)[] = [];
for (const doc of allowedDocs) {
const type: DocumentType =
doc.type === 'pdf' || doc.type === 'epub' || doc.type === 'docx' || doc.type === 'html'
? (doc.type as DocumentType)
: toDocumentTypeFromName(doc.name);
// If the underlying file was deleted manually, keep the API self-healing:
// prune the DB row so clients stop listing ghost documents.
const absolutePath = doc.filePath ? path.join(syncDir, doc.filePath) : '';
if (!absolutePath || !existsSync(absolutePath)) {
await db
.delete(documents)
.where(and(eq(documents.id, doc.id), eq(documents.userId, doc.userId)));
continue;
}
const metadata: BaseDocument = {
id: doc.id!,
const results: BaseDocument[] = rows.map((doc) => {
const type = normalizeDocumentType(doc.type, doc.name);
return {
id: doc.id,
name: doc.name,
size: doc.size,
lastModified: doc.lastModified,
size: Number(doc.size),
lastModified: Number(doc.lastModified),
type,
scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user',
};
if (!includeData) {
results.push(metadata);
continue;
}
try {
const content = await readFile(absolutePath);
results.push({
...metadata,
data: Array.from(new Uint8Array(content)),
});
} catch (err) {
if (isEnoent(err)) {
await db
.delete(documents)
.where(and(eq(documents.id, doc.id), eq(documents.userId, doc.userId)));
continue;
}
console.warn(`Failed to read content for document ${doc.id} at ${doc.filePath}`, err);
}
}
});
return NextResponse.json({ documents: results });
} catch (error) {
console.error('Error loading documents:', error);
console.error('Error loading document metadata:', error);
return NextResponse.json({ error: 'Failed to load documents' }, { status: 500 });
}
}
export async function DELETE(req: NextRequest) {
try {
await ensureDocumentsV1Ready();
if (!(await isDocumentsV1Ready())) {
return NextResponse.json(
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
{ status: 409 },
);
}
if (!isS3Configured()) return s3NotConfiguredResponse();
// Auth check - require session
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
await ensureDbIndexed();
const url = new URL(req.url);
const idsParam = url.searchParams.get('ids');
const scopeParam = (url.searchParams.get('scope') || '').toLowerCase().trim();
@ -221,7 +216,6 @@ export async function DELETE(req: NextRequest) {
);
}
// Deleting the global unclaimed pool is a privileged operation when auth is enabled.
if (ctxOrRes.authEnabled && wantsUnclaimed && ctxOrRes.user?.isAnonymous) {
return NextResponse.json({ error: 'Forbidden' }, { status: 403 });
}
@ -239,58 +233,45 @@ export async function DELETE(req: NextRequest) {
return NextResponse.json({ success: true, deleted: 0 });
}
// Determine which IDs to try to delete
let targetIds: string[] = [];
if (idsParam) {
targetIds = idsParam.split(',').filter(Boolean);
targetIds = idsParam
.split(',')
.map((id) => id.trim().toLowerCase())
.filter((id) => isValidDocumentId(id));
} else {
// Existing behavior was "nuke everything"; keep it scoped to the selected user buckets.
const rows = await db
const rows = (await db
.select({ id: documents.id })
.from(documents)
.where(inArray(documents.userId, targetUserIds));
targetIds = rows.map((d: { id: string | null }) => d.id!).filter(Boolean) as string[];
.where(inArray(documents.userId, targetUserIds))) as Array<{ id: string }>;
targetIds = rows.map((row) => row.id);
}
if (targetIds.length === 0) {
return NextResponse.json({ success: true, deleted: 0 });
}
const deletedRows: { id: string; filePath: string }[] = [];
const rows = await db
const deletedRows = (await db
.delete(documents)
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))
.returning({ id: documents.id, filePath: documents.filePath });
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rows.forEach((r: any) => deletedRows.push({ id: r.id!, filePath: r.filePath }));
.returning({ id: documents.id })) as Array<{ id: string }>;
// If driver doesn't support returning (e.g. older SQLite without properly configured returning), we might fallback.
// But Drizzle usually handles this.
const uniqueIds = Array.from(new Set(deletedRows.map((row) => row.id)));
for (const id of uniqueIds) {
const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, id));
const refCount = Number(ref?.count ?? 0);
if (refCount > 0) continue;
let deletedCount = 0;
for (const row of deletedRows) {
deletedCount++;
// Chech reference count for this ID
// If 0 remaining, delete file
let refCount = 0;
const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, row.id!));
refCount = Number(ref?.count ?? 0);
if (refCount === 0) {
const filePath = path.join(syncDir, row.filePath);
try {
await unlink(filePath);
} catch {
// Ignore if missing
try {
await deleteDocumentBlob(id, testNamespace);
} catch (error) {
if (!isMissingBlobError(error)) {
throw error;
}
}
}
return NextResponse.json({ success: true, deleted: deletedCount });
return NextResponse.json({ success: true, deleted: deletedRows.length });
} catch (error) {
console.error('Error deleting documents:', error);
return NextResponse.json({ error: 'Failed to delete documents' }, { status: 500 });

View file

@ -1,95 +0,0 @@
import { createHash } from 'crypto';
import { mkdir, stat, writeFile } from 'fs/promises';
import path from 'path';
import { NextRequest, NextResponse } from 'next/server';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
import { requireAuthContext } from '@/lib/server/auth';
import { safeDocumentName, toDocumentTypeFromName, trySetFileMtime } from '@/lib/server/documents-utils';
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
export const dynamic = 'force-dynamic';
export async function POST(req: NextRequest) {
try {
await ensureDocumentsV1Ready();
if (!(await isDocumentsV1Ready())) {
return NextResponse.json(
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
{ status: 409 },
);
}
const testNamespace = getOpenReaderTestNamespace(req.headers);
const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
await mkdir(documentsDir, { recursive: true });
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
const form = await req.formData();
const files = form.getAll('files').filter((value): value is File => value instanceof File);
if (files.length === 0) {
return NextResponse.json({ error: 'Missing files' }, { status: 400 });
}
const stored: Array<{
id: string;
name: string;
type: 'pdf' | 'epub' | 'docx' | 'html';
size: number;
lastModified: number;
}> = [];
for (const file of files) {
const arrayBuffer = await file.arrayBuffer();
const content = Buffer.from(new Uint8Array(arrayBuffer));
const id = createHash('sha256').update(content).digest('hex');
const safeName = safeDocumentName(file.name, `${id}.${toDocumentTypeFromName(file.name)}`);
const targetFileName = `${id}__${encodeURIComponent(safeName)}`;
const targetPath = path.join(documentsDir, targetFileName);
try {
await stat(targetPath);
} catch {
await writeFile(targetPath, content);
}
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
await trySetFileMtime(targetPath, lastModified);
const type = toDocumentTypeFromName(safeName);
await db
.insert(documents)
.values({
id,
userId: storageUserId,
name: safeName,
type,
size: content.length,
lastModified,
filePath: targetFileName,
})
.onConflictDoNothing();
stored.push({
id,
name: safeName,
type,
size: content.length,
lastModified,
});
}
return NextResponse.json({ stored });
} catch (error) {
console.error('Error uploading documents:', error);
return NextResponse.json({ error: 'Failed to upload documents' }, { status: 500 });
}
}

View file

@ -0,0 +1,275 @@
import { createHash } from 'crypto';
import { existsSync } from 'fs';
import { readdir, readFile, stat, unlink } from 'fs/promises';
import path from 'path';
import { and, eq } from 'drizzle-orm';
import { NextRequest, NextResponse } from 'next/server';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { auth } from '@/lib/server/auth';
import { DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore';
import { toDocumentTypeFromName } from '@/lib/server/documents-utils';
import { contentTypeForName } from '@/lib/server/library';
import { isS3Configured } from '@/lib/server/s3';
import type { DocumentType } from '@/types/documents';
import {
applyOpenReaderTestNamespacePath,
getOpenReaderTestNamespace,
getUnclaimedUserIdForNamespace,
} from '@/lib/server/test-namespace';
export const dynamic = 'force-dynamic';
type V2Body = {
deleteLocal?: boolean;
dryRun?: boolean;
};
type LegacyDocumentCandidate = {
id: string;
name: string;
type: string;
size: number;
lastModified: number;
};
function isPreconditionFailed(error: unknown): boolean {
if (!error || typeof error !== 'object') return false;
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
}
function extractIdFromFileName(fileName: string): string | null {
const match = /^([a-f0-9]{64})__/i.exec(fileName);
if (!match) return null;
const id = match[1].toLowerCase();
return isValidDocumentId(id) ? id : null;
}
function decodeNameFromFileName(fileName: string, id: string): string {
const prefix = `${id}__`;
if (!fileName.startsWith(prefix)) return `${id}.bin`;
const encoded = fileName.slice(prefix.length);
try {
return decodeURIComponent(encoded);
} catch {
return `${id}.bin`;
}
}
function sniffBinaryDocumentType(bytes: Buffer): Exclude<DocumentType, 'html'> | null {
// PDF signature: "%PDF-"
if (bytes.length >= 5 && bytes.subarray(0, 5).toString('ascii') === '%PDF-') {
return 'pdf';
}
// ZIP signatures: PK..
const isZip =
bytes.length >= 4 &&
bytes[0] === 0x50 &&
bytes[1] === 0x4b &&
(bytes[2] === 0x03 || bytes[2] === 0x05 || bytes[2] === 0x07) &&
(bytes[3] === 0x04 || bytes[3] === 0x06 || bytes[3] === 0x08);
if (!isZip) return null;
// EPUB/DOCX markers usually appear in ZIP local headers near the start.
const probe = bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString('latin1');
if (probe.includes('application/epub+zip') || probe.includes('META-INF/container.xml')) {
return 'epub';
}
if (probe.includes('[Content_Types].xml') && probe.includes('word/')) {
return 'docx';
}
return null;
}
function normalizeNameForType(name: string, id: string, type: DocumentType): string {
if (type === 'html') return name;
const expectedExt = type === 'pdf' ? '.pdf' : type === 'epub' ? '.epub' : '.docx';
if (name.toLowerCase().endsWith(expectedExt)) return name;
const base = name.replace(/\.bin$/i, '');
return `${base || id}${expectedExt}`;
}
function contentTypeForDocument(type: DocumentType, name: string): string {
if (type === 'pdf') return 'application/pdf';
if (type === 'epub') return 'application/epub+zip';
if (type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
return contentTypeForName(name);
}
async function migrateDocumentRowsToBlobHandle(dryRun: boolean): Promise<number> {
const rows = (await db.select().from(documents)) as Array<{
id: string;
userId: string;
filePath: string;
}>;
let updated = 0;
for (const row of rows) {
if (!row.id || !row.userId || row.filePath === row.id) continue;
updated++;
if (dryRun) continue;
await db
.update(documents)
.set({ filePath: row.id })
.where(and(eq(documents.id, row.id), eq(documents.userId, row.userId)));
}
return updated;
}
async function seedMissingDocumentRows(
userId: string,
candidates: LegacyDocumentCandidate[],
dryRun: boolean,
): Promise<number> {
if (candidates.length === 0) return 0;
const existingRows = (await db.select().from(documents).where(eq(documents.userId, userId))) as Array<{
id: string;
}>;
const existingIds = new Set(existingRows.map((row) => row.id));
const seen = new Set<string>();
const toInsert: LegacyDocumentCandidate[] = [];
for (const candidate of candidates) {
if (seen.has(candidate.id)) continue;
seen.add(candidate.id);
if (existingIds.has(candidate.id)) continue;
toInsert.push(candidate);
}
if (toInsert.length === 0) return 0;
if (dryRun) return toInsert.length;
await db.insert(documents).values(
toInsert.map((candidate) => ({
id: candidate.id,
userId,
name: candidate.name,
type: candidate.type,
size: candidate.size,
lastModified: candidate.lastModified,
filePath: candidate.id,
})),
).onConflictDoNothing();
return toInsert.length;
}
export async function POST(request: NextRequest) {
try {
const session = await auth?.api.getSession({ headers: request.headers });
if (auth && !session?.user) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
}
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'S3 is not configured. Set S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.' },
{ status: 409 },
);
}
const raw = (await request.json().catch(() => ({}))) as V2Body;
const dryRun = raw.dryRun === true;
const deleteLocal = raw.deleteLocal === true;
const testNamespace = getOpenReaderTestNamespace(request.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const docsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
if (!existsSync(docsDir)) {
const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun);
return NextResponse.json({
success: true,
dryRun,
deleteLocal,
docsDir,
filesScanned: 0,
uploaded: 0,
alreadyPresent: 0,
skippedInvalid: 0,
deletedLocal: 0,
dbRowsUpdated: rowsUpdated,
dbRowsSeeded: 0,
});
}
const entries = await readdir(docsDir, { withFileTypes: true });
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
let uploaded = 0;
let alreadyPresent = 0;
let skippedInvalid = 0;
let deletedLocal = 0;
const candidates: LegacyDocumentCandidate[] = [];
for (const fileName of files) {
const fullPath = path.join(docsDir, fileName);
const bytes = await readFile(fullPath);
const fileStats = await stat(fullPath);
const extractedId = extractIdFromFileName(fileName);
const id = extractedId ?? createHash('sha256').update(bytes).digest('hex');
if (!isValidDocumentId(id)) {
skippedInvalid++;
continue;
}
const inferredName = decodeNameFromFileName(fileName, id);
const inferredType = toDocumentTypeFromName(inferredName);
const type = inferredType === 'html' ? (sniffBinaryDocumentType(bytes) ?? inferredType) : inferredType;
const normalizedName = normalizeNameForType(inferredName, id, type);
const contentType = contentTypeForDocument(type, normalizedName);
const lastModified = Number.isFinite(fileStats.mtimeMs) ? Math.floor(fileStats.mtimeMs) : Date.now();
candidates.push({
id,
name: normalizedName,
type,
size: bytes.length,
lastModified,
});
if (!dryRun) {
try {
await putDocumentBlob(id, bytes, contentType, testNamespace);
uploaded++;
} catch (error) {
if (isPreconditionFailed(error)) {
alreadyPresent++;
} else {
throw error;
}
}
}
if (deleteLocal && !dryRun) {
await unlink(fullPath).catch(() => {});
deletedLocal++;
}
}
const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun);
const rowsSeeded = await seedMissingDocumentRows(unclaimedUserId, candidates, dryRun);
return NextResponse.json({
success: true,
dryRun,
deleteLocal,
docsDir,
filesScanned: files.length,
uploaded,
alreadyPresent,
skippedInvalid,
deletedLocal,
dbRowsUpdated: rowsUpdated,
dbRowsSeeded: rowsSeeded,
});
} catch (error) {
console.error('Error running v2 migrations:', error);
return NextResponse.json({ error: 'Failed to run v2 migrations' }, { status: 500 });
}
}

View file

@ -2,6 +2,34 @@ import { NextRequest, NextResponse } from 'next/server';
import { claimAnonymousData } from '@/lib/server/claim-data';
import { auth } from '@/lib/server/auth';
import { ensureDbIndexed, getUnclaimedCounts } from '@/lib/server/db-indexing';
import { isDocumentsV1Ready } from '@/lib/server/docstore';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { count, ne } from 'drizzle-orm';
async function checkClaimMigrationReadiness(): Promise<NextResponse | null> {
const documentsV1Ready = await isDocumentsV1Ready();
if (!documentsV1Ready) {
return NextResponse.json(
{ error: 'Document migration is not ready. Run startup migrations first.' },
{ status: 409 },
);
}
const [legacyRows] = await db
.select({ count: count() })
.from(documents)
.where(ne(documents.filePath, documents.id));
if (Number(legacyRows?.count ?? 0) > 0) {
return NextResponse.json(
{ error: 'Document metadata migration is still pending. Wait for startup migrations to complete.' },
{ status: 409 },
);
}
return null;
}
export async function GET(req: NextRequest) {
try {
@ -10,6 +38,9 @@ export async function GET(req: NextRequest) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
}
const readiness = await checkClaimMigrationReadiness();
if (readiness) return readiness;
await ensureDbIndexed();
const counts = await getUnclaimedCounts();
return NextResponse.json({ success: true, ...counts });
@ -26,6 +57,9 @@ export async function POST(req: NextRequest) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
}
const readiness = await checkClaimMigrationReadiness();
if (readiness) return readiness;
const userId = session.user.id;
await ensureDbIndexed();

View file

@ -57,8 +57,9 @@ export default function HTMLPage() {
}, [isLoading, id, router, setCurrentDocument, stop]);
useEffect(() => {
if (!isLoading) return;
loadDocument();
}, [loadDocument]);
}, [loadDocument, isLoading]);
// Compute available height = viewport - (header height + tts bar height)
useEffect(() => {

View file

@ -5,7 +5,8 @@ import {
extractEpubCoverToDataUrl,
renderPdfFirstPageToDataUrl,
} from '@/lib/documentPreview';
import { downloadDocumentContent, getDocumentContentSnippet } from '@/lib/client-documents';
import { getDocumentContentSnippet } from '@/lib/client-documents';
import { ensureCachedDocument } from '@/lib/document-cache';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
@ -36,6 +37,16 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
const [isGenerating, setIsGenerating] = useState(false);
const previewKey = useMemo(() => `${doc.type}:${doc.id}`, [doc.id, doc.type]);
const cacheMeta = useMemo(
() => ({
id: doc.id,
name: doc.name,
type: doc.type,
size: doc.size,
lastModified: doc.lastModified,
}),
[doc.id, doc.lastModified, doc.name, doc.size, doc.type],
);
useEffect(() => {
const el = containerRef.current;
@ -81,7 +92,9 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
const targetWidth = 240;
if (doc.type === 'pdf') {
const data = await downloadDocumentContent(doc.id, { signal: controller.signal });
const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal });
if (cached.type !== 'pdf') return;
const data = cached.data;
if (cancelled) return;
const dataUrl = await renderPdfFirstPageToDataUrl(data, targetWidth);
if (cancelled) return;
@ -92,7 +105,9 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
}
if (doc.type === 'epub') {
const data = await downloadDocumentContent(doc.id, { signal: controller.signal });
const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal });
if (cached.type !== 'epub') return;
const data = cached.data;
if (cancelled) return;
const cover = await extractEpubCoverToDataUrl(data, targetWidth);
if (cancelled) return;
@ -130,7 +145,7 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
cancelled = true;
controller.abort();
};
}, [doc.id, doc.type, isVisible, previewKey]);
}, [cacheMeta, doc.id, doc.type, isVisible, previewKey]);
const gradientClass = isPDF
? 'from-red-500/80 via-red-400/60 to-red-600/80'

View file

@ -111,13 +111,40 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
if (response?.ok) {
const data = await response.json();
const didMigrate =
const v2ApplyResponse = await fetch('/api/migrations/v2', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ dryRun: false, deleteLocal: false }),
}).catch(() => null);
const v2ApplyData = v2ApplyResponse?.ok ? await v2ApplyResponse.json().catch(() => null) : null;
const didMigrateV1 =
data.documentsMigrated ||
data.audiobooksMigrated ||
(data.rekey?.renamed ?? 0) > 0 ||
(data.rekey?.merged ?? 0) > 0;
if (didMigrate) {
if (v2ApplyData) {
const uploaded = Number(v2ApplyData.uploaded ?? 0);
const alreadyPresent = Number(v2ApplyData.alreadyPresent ?? 0);
const dbRowsUpdated = Number(v2ApplyData.dbRowsUpdated ?? 0);
const dbRowsSeeded = Number(v2ApplyData.dbRowsSeeded ?? 0);
const deletedLocal = Number(v2ApplyData.deletedLocal ?? 0);
const dbRowsMigrated = dbRowsUpdated + dbRowsSeeded;
const didMigrateV2 = uploaded > 0 || dbRowsMigrated > 0 || deletedLocal > 0;
if (didMigrateV2) {
toast.success(
`Legacy document migration complete: ${uploaded} uploaded, ${alreadyPresent} already in S3, ${dbRowsMigrated} DB row(s) migrated.`,
{ duration: 6000, icon: '📦' },
);
window.dispatchEvent(new CustomEvent('openreader:documentsChanged', {
detail: { reason: 'migration-v2-complete' },
}));
}
}
if (didMigrateV1) {
toast.success('Library migration complete', {
duration: 5000,
icon: '📦',

View file

@ -50,6 +50,19 @@ export function DocumentProvider({ children }: { children: ReactNode }) {
});
}, [refreshDocuments]);
useEffect(() => {
const handler = () => {
refreshDocuments().catch((err) => {
console.error('Failed to refresh documents after change event:', err);
});
};
window.addEventListener('openreader:documentsChanged', handler as EventListener);
return () => {
window.removeEventListener('openreader:documentsChanged', handler as EventListener);
};
}, [refreshDocuments]);
const docsByType = useMemo(() => {
const pdfDocs = (docs ?? []).filter((d) => d.type === 'pdf') as Array<BaseDocument & { type: 'pdf' }>;
const epubDocs = (docs ?? []).filter((d) => d.type === 'epub') as Array<BaseDocument & { type: 'epub' }>;

View file

@ -7,6 +7,8 @@ import {
ReactNode,
useCallback,
useMemo,
useEffect,
useRef,
} from 'react';
import { getDocumentMetadata } from '@/lib/client-documents';
import { ensureCachedDocument } from '@/lib/document-cache';
@ -30,12 +32,16 @@ const HTMLContext = createContext<HTMLContextType | undefined>(undefined);
*/
export function HTMLProvider({ children }: { children: ReactNode }) {
const { setText: setTTSText, stop } = useTTS();
const setTTSTextRef = useRef(setTTSText);
// Current document state
const [currDocData, setCurrDocData] = useState<string>();
const [currDocName, setCurrDocName] = useState<string>();
const [currDocText, setCurrDocText] = useState<string>();
useEffect(() => {
setTTSTextRef.current = setTTSText;
}, [setTTSText]);
/**
* Clears all current document state and stops any active TTS
@ -69,12 +75,12 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
setCurrDocName(doc.name);
setCurrDocData(doc.data);
setCurrDocText(doc.data); // Use the same text for TTS
setTTSText(doc.data);
setTTSTextRef.current(doc.data);
} catch (error) {
console.error('Failed to get HTML document:', error);
clearCurrDoc();
}
}, [clearCurrDoc, setTTSText]);
}, [clearCurrDoc]);

View file

@ -12,22 +12,32 @@ function createAuthClientWithUrl(baseUrl: string) {
// Cache for auth client instances by baseUrl
const clientCache = new Map<string, ReturnType<typeof createAuthClientWithUrl>>();
function resolveAuthClientBaseUrl(baseUrl: string | null): string {
if (typeof window !== 'undefined' && window.location?.origin) {
// Always use same-origin in the browser so local hostname variants
// (localhost vs LAN IP) do not break cookie/session bootstrap.
return window.location.origin;
}
if (baseUrl) return baseUrl;
throw new Error(
'Cannot create auth client without baseUrl in a non-browser context. ' +
'Use useAuthConfig() in components to get the properly configured baseUrl.'
);
}
/**
* Factory function to get auth client with specific baseUrl.
* In components, prefer reading `baseUrl` from `useAuthConfig()` and then calling `getAuthClient(baseUrl)`.
* @param baseUrl - The auth server base URL. If null, will throw an error.
* @param baseUrl - Server-provided auth URL; in the browser we use same-origin automatically.
*/
export function getAuthClient(baseUrl: string | null) {
if (!baseUrl) {
throw new Error(
'Cannot create auth client without baseUrl. ' +
'Use useAuthConfig() in components to get the properly configured baseUrl.'
);
const resolvedBaseUrl = resolveAuthClientBaseUrl(baseUrl);
if (!clientCache.has(resolvedBaseUrl)) {
clientCache.set(resolvedBaseUrl, createAuthClientWithUrl(resolvedBaseUrl));
}
if (!clientCache.has(baseUrl)) {
clientCache.set(baseUrl, createAuthClientWithUrl(baseUrl));
}
return clientCache.get(baseUrl)!;
return clientCache.get(resolvedBaseUrl)!;
}

View file

@ -1,8 +1,52 @@
import type { BaseDocument } from '@/types/documents';
import { sha256HexFromArrayBuffer } from '@/lib/sha256';
import type { BaseDocument, DocumentType } from '@/types/documents';
export type UploadSource = {
id: string;
name: string;
type: DocumentType;
size: number;
lastModified: number;
contentType: string;
body: Blob | ArrayBuffer | Uint8Array;
};
type UploadOptions = {
signal?: AbortSignal;
};
function toUploadBody(body: UploadSource['body']): BodyInit {
if (body instanceof Blob) return body;
if (body instanceof ArrayBuffer) return body;
return body as unknown as BodyInit;
}
async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise<void> {
const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, {
method: 'PUT',
headers: { 'Content-Type': source.contentType || 'application/octet-stream' },
body: toUploadBody(source.body),
signal: options?.signal,
});
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || `Proxy upload failed (status ${res.status})`);
}
}
function documentTypeForName(name: string): DocumentType {
const lower = name.toLowerCase();
if (lower.endsWith('.pdf')) return 'pdf';
if (lower.endsWith('.epub')) return 'epub';
if (lower.endsWith('.docx')) return 'docx';
return 'html';
}
export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string {
if (doc.type === 'pdf') return 'application/pdf';
if (doc.type === 'epub') return 'application/epub+zip';
if (doc.type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
const lower = doc.name.toLowerCase();
if (lower.endsWith('.md') || lower.endsWith('.markdown') || lower.endsWith('.mdown') || lower.endsWith('.mkd')) {
@ -13,7 +57,6 @@ export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string
export async function listDocuments(options?: { ids?: string[]; signal?: AbortSignal }): Promise<BaseDocument[]> {
const params = new URLSearchParams();
params.set('list', 'true');
if (options?.ids?.length) {
params.set('ids', options.ids.join(','));
}
@ -33,27 +76,114 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
return docs[0] ?? null;
}
export async function uploadDocuments(files: File[], options?: { signal?: AbortSignal }): Promise<BaseDocument[]> {
const form = new FormData();
for (const file of files) {
form.append('files', file);
}
export async function uploadDocumentSources(sources: UploadSource[], options?: UploadOptions): Promise<BaseDocument[]> {
if (sources.length === 0) return [];
const res = await fetch('/api/documents/upload', {
const presignRes = await fetch('/api/documents/blob/upload/presign', {
method: 'POST',
body: form,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
uploads: sources.map((source) => ({
id: source.id,
contentType: source.contentType,
size: source.size,
})),
}),
signal: options?.signal,
});
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to upload documents');
if (!presignRes.ok) {
const data = (await presignRes.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to prepare uploads');
}
const data = (await res.json()) as { stored: BaseDocument[] };
const presigned = (await presignRes.json()) as {
uploads?: Array<{ id: string; url: string; headers?: Record<string, string> }>;
};
const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload]));
for (const source of sources) {
const upload = byId.get(source.id);
if (!upload?.url) {
throw new Error(`Missing presigned upload for document ${source.id}`);
}
let putError: unknown = null;
try {
const putRes = await fetch(upload.url, {
method: 'PUT',
headers: new Headers(upload.headers || {}),
body: toUploadBody(source.body),
signal: options?.signal,
});
// 412 means the content-hash object already exists (idempotent upload).
if (putRes.ok || putRes.status === 412) {
continue;
}
putError = new Error(`Direct upload failed with status ${putRes.status}`);
} catch (error) {
if (options?.signal?.aborted) throw error;
putError = error;
}
try {
await uploadDocumentSourceViaProxy(source, options);
} catch (proxyError) {
const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error';
const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error';
throw new Error(`Failed to upload document ${source.name}: ${directMessage}; fallback failed: ${proxyMessage}`);
}
}
const registerRes = await fetch('/api/documents', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
documents: sources.map((source) => ({
id: source.id,
name: source.name,
type: source.type,
size: source.size,
lastModified: source.lastModified,
})),
}),
signal: options?.signal,
});
if (!registerRes.ok) {
const data = (await registerRes.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to register uploaded documents');
}
const data = (await registerRes.json()) as { stored: BaseDocument[] };
return data.stored || [];
}
export async function uploadDocuments(files: File[], options?: UploadOptions): Promise<BaseDocument[]> {
if (files.length === 0) return [];
const sources: UploadSource[] = [];
for (const file of files) {
const bytes = await file.arrayBuffer();
const id = await sha256HexFromArrayBuffer(bytes);
const type = documentTypeForName(file.name);
const name = file.name || `${id}.${type}`;
const contentType = file.type || mimeTypeForDoc({ name, type });
sources.push({
id,
name,
type,
size: file.size,
lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(),
contentType,
body: file,
});
}
return uploadDocumentSources(sources, options);
}
export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user' | 'unclaimed'; signal?: AbortSignal }): Promise<void> {
const params = new URLSearchParams();
if (options?.ids?.length) {
@ -72,7 +202,7 @@ export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user'
}
export async function downloadDocumentContent(id: string, options?: { signal?: AbortSignal }): Promise<ArrayBuffer> {
const res = await fetch(`/api/documents/content?id=${encodeURIComponent(id)}`, { signal: options?.signal });
const res = await fetch(`/api/documents/blob?id=${encodeURIComponent(id)}`, { signal: options?.signal });
if (!res.ok) {
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
@ -95,7 +225,7 @@ export async function getDocumentContentSnippet(
if (typeof options?.maxChars === 'number') params.set('maxChars', String(options.maxChars));
if (typeof options?.maxBytes === 'number') params.set('maxBytes', String(options.maxBytes));
const res = await fetch(`/api/documents/content?${params.toString()}`, { signal: options?.signal });
const res = await fetch(`/api/documents/blob?${params.toString()}`, { signal: options?.signal });
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || `Failed to load content snippet (status ${res.status})`);

View file

@ -5,11 +5,12 @@ import {
EPUBDocument,
HTMLDocument,
DocumentListState,
SyncedDocument,
BaseDocument,
DocumentListDocument,
} from '@/types/documents';
import { sha256HexFromBytes, sha256HexFromString } from '@/lib/sha256';
import { downloadDocumentContent, listDocuments, uploadDocumentSources, type UploadSource } from '@/lib/client-documents';
import { cacheStoredDocumentFromBytes } from '@/lib/document-cache';
const DB_NAME = 'openreader-db';
// Managed via Dexie (version bumped from the original manual IndexedDB)
@ -657,15 +658,26 @@ export async function syncDocumentsToServer(
const epubDocs = await getAllEpubDocuments();
const htmlDocs = await getAllHtmlDocuments();
const documents: SyncedDocument[] = [];
const uploads: Array<{ oldId: string; source: UploadSource }> = [];
const totalDocs = pdfDocs.length + epubDocs.length + htmlDocs.length;
let processedDocs = 0;
const textEncoder = new TextEncoder();
for (const doc of pdfDocs) {
documents.push({
...doc,
type: 'pdf',
data: Array.from(new Uint8Array(doc.data)),
const bytes = new Uint8Array(doc.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: doc.id,
source: {
id,
name: doc.name,
type: 'pdf',
size: bytes.byteLength,
lastModified: doc.lastModified,
contentType: 'application/pdf',
body: bytes,
},
});
processedDocs++;
if (onProgress) {
@ -674,10 +686,19 @@ export async function syncDocumentsToServer(
}
for (const doc of epubDocs) {
documents.push({
...doc,
type: 'epub',
data: Array.from(new Uint8Array(doc.data)),
const bytes = new Uint8Array(doc.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: doc.id,
source: {
id,
name: doc.name,
type: 'epub',
size: bytes.byteLength,
lastModified: doc.lastModified,
contentType: 'application/epub+zip',
body: bytes,
},
});
processedDocs++;
if (onProgress) {
@ -685,13 +706,20 @@ export async function syncDocumentsToServer(
}
}
const encoder = new TextEncoder();
for (const doc of htmlDocs) {
const encoded = encoder.encode(doc.data);
documents.push({
...doc,
type: 'html',
data: Array.from(encoded),
const encoded = textEncoder.encode(doc.data);
const id = await sha256HexFromBytes(encoded);
uploads.push({
oldId: doc.id,
source: {
id,
name: doc.name,
type: 'html',
size: encoded.byteLength,
lastModified: doc.lastModified,
contentType: 'text/plain; charset=utf-8',
body: encoded,
},
});
processedDocs++;
if (onProgress) {
@ -703,25 +731,11 @@ export async function syncDocumentsToServer(
onProgress(50, 'Uploading to server...');
}
const response = await fetch('/api/documents', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ documents }),
signal,
});
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
if (!response.ok) {
throw new Error('Failed to sync documents to server');
}
const payload = (await response.json().catch(() => null)) as
| { stored?: Array<{ oldId: string; id: string }> }
| null;
const stored = payload?.stored ?? [];
for (const mapping of stored) {
if (!mapping || typeof mapping.oldId !== 'string' || typeof mapping.id !== 'string') continue;
if (mapping.oldId === mapping.id) continue;
await applyDocumentIdMapping(mapping.oldId, mapping.id);
for (const entry of uploads) {
if (entry.oldId === entry.source.id) continue;
await applyDocumentIdMapping(entry.oldId, entry.source.id);
}
if (onProgress) {
@ -736,52 +750,77 @@ export async function syncSelectedDocumentsToServer(
onProgress?: (progress: number, status?: string) => void,
signal?: AbortSignal,
): Promise<{ lastSync: number }> {
// Re-use logic from syncDocumentsToServer but only for specific documents
// Actually, syncDocumentsToServer fetches all docs from DB.
// We need to fetch the *full content* of the selected docs from DB.
const fullDocs: SyncedDocument[] = [];
let processed = 0;
for (const doc of documents) {
if (doc.type === 'pdf') {
const data = await getPdfDocument(doc.id);
if (data) fullDocs.push({ ...data, type: 'pdf', data: Array.from(new Uint8Array(data.data)) });
} else if (doc.type === 'epub') {
const data = await getEpubDocument(doc.id);
if (data) fullDocs.push({ ...data, type: 'epub', data: Array.from(new Uint8Array(data.data)) });
} else {
const data = await getHtmlDocument(doc.id);
if (data) {
const encoder = new TextEncoder();
fullDocs.push({ ...data, type: 'html', data: Array.from(encoder.encode(data.data)) });
}
}
processed++;
if (onProgress) onProgress((processed / documents.length) * 50, `Preparing ${processed}/${documents.length}...`);
const uploads: Array<{ oldId: string; source: UploadSource }> = [];
const textEncoder = new TextEncoder();
let processed = 0;
for (const doc of documents) {
if (doc.type === 'pdf') {
const data = await getPdfDocument(doc.id);
if (data) {
const bytes = new Uint8Array(data.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: data.id,
source: {
id,
name: data.name,
type: 'pdf',
size: bytes.byteLength,
lastModified: data.lastModified,
contentType: 'application/pdf',
body: bytes,
},
});
}
} else if (doc.type === 'epub') {
const data = await getEpubDocument(doc.id);
if (data) {
const bytes = new Uint8Array(data.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: data.id,
source: {
id,
name: data.name,
type: 'epub',
size: bytes.byteLength,
lastModified: data.lastModified,
contentType: 'application/epub+zip',
body: bytes,
},
});
}
} else {
const data = await getHtmlDocument(doc.id);
if (data) {
const bytes = textEncoder.encode(data.data);
const id = await sha256HexFromBytes(bytes);
uploads.push({
oldId: data.id,
source: {
id,
name: data.name,
type: 'html',
size: bytes.byteLength,
lastModified: data.lastModified,
contentType: 'text/plain; charset=utf-8',
body: bytes,
},
});
}
}
if (onProgress) onProgress(50, 'Uploading to server...');
const response = await fetch('/api/documents', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ documents: fullDocs }),
signal,
});
if (!response.ok) {
throw new Error('Failed to sync documents to server');
processed++;
if (onProgress) onProgress((processed / documents.length) * 50, `Preparing ${processed}/${documents.length}...`);
}
const payload = (await response.json().catch(() => null)) as
| { stored?: Array<{ oldId: string; id: string }> }
| null;
const stored = payload?.stored ?? [];
for (const mapping of stored) {
if (!mapping || typeof mapping.oldId !== 'string' || typeof mapping.id !== 'string') continue;
if (mapping.oldId === mapping.id) continue;
await applyDocumentIdMapping(mapping.oldId, mapping.id);
if (onProgress) onProgress(50, 'Uploading to server...');
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
for (const entry of uploads) {
if (entry.oldId === entry.source.id) continue;
await applyDocumentIdMapping(entry.oldId, entry.source.id);
}
if (onProgress) {
@ -800,22 +839,8 @@ export async function loadDocumentsFromServer(
onProgress(10, 'Starting download...');
}
const response = await fetch('/api/documents', { signal });
if (!response.ok) {
throw new Error('Failed to fetch documents from server');
}
if (onProgress) {
onProgress(30, 'Download complete');
}
const { documents } = (await response.json()) as { documents: SyncedDocument[] };
if (onProgress) {
onProgress(40, 'Parsing documents...');
}
await saveSyncedDocumentsLocally(documents, onProgress);
const documents = await listDocuments({ signal });
await downloadAndCacheServerDocuments(documents, onProgress, signal);
if (onProgress) {
onProgress(100, 'Load complete!');
@ -832,26 +857,9 @@ export async function loadSelectedDocumentsFromServer(
if (onProgress) {
onProgress(10, 'Starting download...');
}
// Use new filtered API
const idsParam = selectedIds.join(',');
const response = await fetch(`/api/documents?ids=${encodeURIComponent(idsParam)}`, { signal });
if (!response.ok) {
throw new Error('Failed to fetch documents from server');
}
if (onProgress) {
onProgress(30, 'Download complete');
}
const { documents } = (await response.json()) as { documents: SyncedDocument[] };
if (onProgress) {
onProgress(40, 'Parsing documents...');
}
await saveSyncedDocumentsLocally(documents, onProgress);
const documents = await listDocuments({ ids: selectedIds, signal });
await downloadAndCacheServerDocuments(documents, onProgress, signal);
if (onProgress) {
onProgress(100, 'Load complete!');
@ -860,52 +868,23 @@ export async function loadSelectedDocumentsFromServer(
return { lastSync: Date.now() };
}
async function saveSyncedDocumentsLocally(documents: SyncedDocument[], onProgress?: (progress: number, status?: string) => void) {
const textDecoder = new TextDecoder();
async function downloadAndCacheServerDocuments(
documents: BaseDocument[],
onProgress?: (progress: number, status?: string) => void,
signal?: AbortSignal,
) {
if (onProgress) onProgress(30, 'List complete');
if (documents.length === 0) {
if (onProgress) onProgress(95, 'No documents to import');
return;
}
for (let i = 0; i < documents.length; i++) {
const doc = documents[i];
if (doc.type === 'pdf') {
const uint8Array = new Uint8Array(doc.data);
const documentData: PDFDocument = {
id: doc.id,
type: 'pdf',
name: doc.name,
size: doc.size,
lastModified: doc.lastModified,
data: uint8Array.buffer,
};
await addPdfDocument(documentData);
} else if (doc.type === 'epub') {
const uint8Array = new Uint8Array(doc.data);
const documentData: EPUBDocument = {
id: doc.id,
type: 'epub',
name: doc.name,
size: doc.size,
lastModified: doc.lastModified,
data: uint8Array.buffer,
};
await addEpubDocument(documentData);
} else if (doc.type === 'html') {
const uint8Array = new Uint8Array(doc.data);
const decoded = textDecoder.decode(uint8Array);
const documentData: HTMLDocument = {
id: doc.id,
type: 'html',
name: doc.name,
size: doc.size,
lastModified: doc.lastModified,
data: decoded,
};
await addHtmlDocument(documentData);
} else {
console.warn(`Unknown document type: ${doc.type}`);
}
const bytes = await downloadDocumentContent(doc.id, { signal });
await cacheStoredDocumentFromBytes(doc, bytes);
if (onProgress) {
onProgress(40 + ((i + 1) / documents.length) * 50, `Processing document ${i + 1}/${documents.length}...`);
onProgress(30 + ((i + 1) / documents.length) * 65, `Downloading ${i + 1}/${documents.length}: ${doc.name}`);
}
}
}
@ -1007,5 +986,3 @@ export async function importDocumentsFromLibrary(
onProgress(100, 'Library import complete!');
}
}

View file

@ -1,9 +1,9 @@
/**
* Centralized auth configuration check.
* Auth is only enabled when BOTH BETTER_AUTH_SECRET and BETTER_AUTH_URL are set.
* Auth is only enabled when BOTH AUTH_SECRET and BASE_URL are set.
*/
export function isAuthEnabled(): boolean {
return !!(process.env.BETTER_AUTH_SECRET && process.env.BETTER_AUTH_URL);
return !!(process.env.AUTH_SECRET && process.env.BASE_URL);
}
/**
@ -13,5 +13,5 @@ export function getAuthBaseUrl(): string | null {
if (!isAuthEnabled()) {
return null;
}
return process.env.BETTER_AUTH_URL || null;
return process.env.BASE_URL || null;
}

View file

@ -13,6 +13,34 @@ import * as schema from "@/db/schema"; // Import the dynamic schema
// ...
function tryGetOrigin(url: string | undefined): string | null {
if (!url) return null;
try {
return new URL(url).origin;
} catch {
return null;
}
}
function getTrustedOrigins(): string[] {
const origins = new Set<string>();
const baseOrigin = tryGetOrigin(process.env.BASE_URL);
if (baseOrigin) origins.add(baseOrigin);
// Comma-separated list for local multi-host setups (e.g., localhost + LAN IP).
const extra = (process.env.AUTH_TRUSTED_ORIGINS || '')
.split(',')
.map((value) => value.trim())
.filter(Boolean);
for (const candidate of extra) {
const origin = tryGetOrigin(candidate);
if (origin) origins.add(origin);
}
return Array.from(origins);
}
const createAuth = () => betterAuth({
// eslint-disable-next-line @typescript-eslint/no-explicit-any
database: drizzleAdapter(db as any, {
@ -25,8 +53,9 @@ const createAuth = () => betterAuth({
verification: schema.verification,
}
}),
secret: process.env.BETTER_AUTH_SECRET!,
baseURL: process.env.BETTER_AUTH_URL || "http://localhost:3003",
secret: process.env.AUTH_SECRET!,
baseURL: process.env.BASE_URL!,
trustedOrigins: getTrustedOrigins(),
emailAndPassword: {
enabled: true,
requireEmailVerification: false, // Set to true in production

View file

@ -6,7 +6,7 @@ import { db } from '@/db';
import { audiobookChapters, audiobooks, documents } from '@/db/schema';
import { and, count, eq } from 'drizzle-orm';
import { listStoredChapters } from '@/lib/server/audiobook';
import { AUDIOBOOKS_V1_DIR, DOCUMENTS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore';
import { AUDIOBOOKS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore';
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
const MIGRATIONS_DIR = path.join(DOCSTORE_DIR, '.migrations');
@ -38,33 +38,14 @@ async function writeState(): Promise<void> {
await fs.writeFile(STATE_PATH, JSON.stringify(state, null, 2));
}
async function hasFilesystemContent(mode: DbIndexState['mode']): Promise<{ documents: boolean; audiobooks: boolean }> {
let documentsPresent = false;
let audiobooksPresent = false;
// Documents are always stored under documents_v1.
try {
const entries = await fs.readdir(DOCUMENTS_V1_DIR);
documentsPresent = entries.some((name) => /^[a-f0-9]{64}__.+$/i.test(name));
} catch {
// ignore
}
// Audiobooks differ by auth-mode layout.
async function hasAudiobookFilesystemContent(mode: DbIndexState['mode']): Promise<boolean> {
const audiobookDir = mode === 'auth' ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR;
try {
const entries = await fs.readdir(audiobookDir, { withFileTypes: true });
audiobooksPresent = entries.some((e) => e.isDirectory() && e.name.endsWith('-audiobook'));
return entries.some((entry) => entry.isDirectory() && entry.name.endsWith('-audiobook'));
} catch {
// ignore
return false;
}
return { documents: documentsPresent, audiobooks: audiobooksPresent };
}
async function isDocumentIndexed(id: string): Promise<boolean> {
const result = await db.select({ id: documents.id }).from(documents).where(eq(documents.id, id));
return result.length > 0;
}
async function isAudiobookIndexedForUser(id: string, userId: string): Promise<boolean> {
@ -90,7 +71,6 @@ async function migrateLegacyAudiobooksToUnclaimed(): Promise<number> {
const sourceDir = path.join(AUDIOBOOKS_V1_DIR, entry.name);
const targetDir = path.join(unclaimedDir, entry.name);
if (existsSync(targetDir)) continue;
try {
@ -115,133 +95,83 @@ export async function getUnclaimedCounts(): Promise<{ documents: number; audiobo
};
}
async function scanAndPopulateDb(): Promise<{ documents: number; audiobooks: number }> {
async function scanAndPopulateAudiobookDb(): Promise<void> {
const authEnabled = isAuthEnabled();
console.log('Scanning file system for un-indexed content...');
console.log('Scanning file system for un-indexed audiobooks...');
if (authEnabled) {
await migrateLegacyAudiobooksToUnclaimed();
}
if (existsSync(DOCUMENTS_V1_DIR)) {
const files = await fs.readdir(DOCUMENTS_V1_DIR);
for (const file of files) {
const match = /^([a-f0-9]{64})__(.+)$/i.exec(file);
if (!match) continue;
const id = match[1];
const encodedName = match[2];
if (await isDocumentIndexed(id)) continue;
let name: string;
try {
name = decodeURIComponent(encodedName);
} catch {
continue;
}
const filePath = path.join(DOCUMENTS_V1_DIR, file);
const stats = await fs.stat(filePath);
const ext = path.extname(name).toLowerCase().replace('.', '');
await db.insert(documents).values({
id,
userId: UNCLAIMED_USER_ID,
name,
type: ext,
size: stats.size,
lastModified: Math.floor(stats.mtimeMs),
filePath: file,
});
console.log(`Indexed document: ${name} (${id})`);
}
}
const audiobookScanDir = authEnabled ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR;
if (existsSync(audiobookScanDir)) {
const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
if (!entry.name.endsWith('-audiobook')) continue;
if (!existsSync(audiobookScanDir)) return;
const bookId = entry.name.replace('-audiobook', '');
if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue;
const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
if (!entry.name.endsWith('-audiobook')) continue;
const dirPath = path.join(audiobookScanDir, entry.name);
const bookId = entry.name.replace('-audiobook', '');
if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue;
let title = 'Unknown Title';
try {
const metaPath = path.join(dirPath, 'audiobook.meta.json');
const metaContent = await fs.readFile(metaPath, 'utf8');
JSON.parse(metaContent);
} catch {
// ignore
}
const dirPath = path.join(audiobookScanDir, entry.name);
const chapters = await listStoredChapters(dirPath);
const totalDuration = chapters.reduce((acc, c) => acc + (c.durationSec || 0), 0);
if (chapters.length > 0) title = chapters[0].title || title;
let title = 'Unknown Title';
try {
const metaPath = path.join(dirPath, 'audiobook.meta.json');
const metaContent = await fs.readFile(metaPath, 'utf8');
JSON.parse(metaContent);
} catch {
// ignore
}
await db.insert(audiobooks).values({
id: bookId,
const chapters = await listStoredChapters(dirPath);
const totalDuration = chapters.reduce((acc, chapter) => acc + (chapter.durationSec || 0), 0);
if (chapters.length > 0) title = chapters[0].title || title;
await db.insert(audiobooks).values({
id: bookId,
userId: UNCLAIMED_USER_ID,
title,
duration: totalDuration,
});
console.log(`Indexed audiobook: ${bookId}`);
for (const chapter of chapters) {
await db.insert(audiobookChapters).values({
id: `${bookId}-${chapter.index}`,
bookId,
userId: UNCLAIMED_USER_ID,
title,
duration: totalDuration,
chapterIndex: chapter.index,
title: chapter.title,
duration: chapter.durationSec || 0,
filePath: chapter.filePath,
format: chapter.format,
});
console.log(`Indexed audiobook: ${bookId}`);
for (const chapter of chapters) {
await db.insert(audiobookChapters).values({
id: `${bookId}-${chapter.index}`,
bookId,
userId: UNCLAIMED_USER_ID,
chapterIndex: chapter.index,
title: chapter.title,
duration: chapter.durationSec || 0,
filePath: chapter.filePath,
format: chapter.format,
});
}
}
}
return getUnclaimedCounts();
}
/**
* Ensure DB has rows for existing filesystem content (documents/audiobooks).
*
* This is intentionally safe to call from routes:
* - Runs at most once per process (memoryIndexed)
* - Uses a persisted state file under docstore/.migrations to avoid rescanning every boot
*/
export async function ensureDbIndexed(): Promise<void> {
export async function ensureAudiobooksIndexed(): Promise<void> {
const mode: DbIndexState['mode'] = isAuthEnabled() ? 'auth' : 'noauth';
if (memoryIndexedMode === mode) return;
inflight ??= (async () => {
const hasState = existsSync(STATE_PATH) ? await readState() : null;
if (hasState && hasState.mode === mode) {
// If the DB was reset but the state file survived, don't get stuck "indexed" forever.
// Only skip if:
// - the DB already has rows for any on-disk content (per category), OR
// - there is no content on disk to index (per category).
//
// This avoids a bad state where (for example) audiobooks are indexed, but documents exist on disk
// and aren't counted/claimable because we early-exit based on "some DB rows exist".
const [counts, fsHas] = await Promise.all([getUnclaimedCounts(), hasFilesystemContent(mode)]);
const docsOk = counts.documents > 0 || !fsHas.documents;
const audiobooksOk = counts.audiobooks > 0 || !fsHas.audiobooks;
if (docsOk && audiobooksOk) {
const [counts, fsHasAudiobooks] = await Promise.all([
getUnclaimedCounts(),
hasAudiobookFilesystemContent(mode),
]);
const audiobooksOk = counts.audiobooks > 0 || !fsHasAudiobooks;
if (audiobooksOk) {
memoryIndexedMode = mode;
return;
}
}
await fs.mkdir(DOCSTORE_DIR, { recursive: true });
await scanAndPopulateDb();
await scanAndPopulateAudiobookDb();
await writeState();
memoryIndexedMode = mode;
})().finally(() => {
@ -250,3 +180,7 @@ export async function ensureDbIndexed(): Promise<void> {
await inflight;
}
export async function ensureDbIndexed(): Promise<void> {
await ensureAudiobooksIndexed();
}

View file

@ -11,6 +11,7 @@ export const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1');
export const AUDIOBOOKS_USERS_DIR = path.join(DOCSTORE_DIR, 'audiobooks_users_v1');
export const UNCLAIMED_USER_ID = 'unclaimed';
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
/**
* Get the audiobook directory for a specific user when auth is enabled.
@ -34,15 +35,29 @@ export function getUnclaimedAudiobookDir(): string {
}
/**
* Get the full path to a specific audiobook directory.
* - When auth is disabled: docstore/audiobooks_v1/{bookId}-audiobook
* - When auth is enabled: docstore/audiobooks_users_v1/{userId}/{bookId}-audiobook
* Resolve the base audiobooks directory for request context, including optional test namespace.
* - When auth is disabled or userId is absent: docstore/audiobooks_v1[/<namespace>]
* - When auth is enabled: docstore/audiobooks_users_v1/{userId}[/<namespace>]
*/
export function getAudiobookPath(bookId: string, userId: string | null, authEnabled: boolean): string {
if (!authEnabled || !userId) {
return path.join(AUDIOBOOKS_V1_DIR, `${bookId}-audiobook`);
}
return path.join(getUserAudiobookDir(userId), `${bookId}-audiobook`);
export function getAudiobooksRootDir({
userId,
authEnabled,
namespace,
}: {
userId: string | null;
authEnabled: boolean;
namespace: string | null;
}): string {
const baseDir = !authEnabled || !userId ? AUDIOBOOKS_V1_DIR : getUserAudiobookDir(userId);
if (!namespace) return baseDir;
const safe = namespace.trim();
if (!safe || safe === '.' || safe === '..' || safe.includes('..')) return baseDir;
if (!SAFE_NAMESPACE_REGEX.test(safe)) return baseDir;
const resolved = path.resolve(baseDir, safe);
if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir;
return resolved;
}
/**
@ -210,15 +225,6 @@ export function getMigratedDocumentFileName(id: string, name: string): string {
return targetFileName;
}
export type FSDocument = {
id: string;
name: string;
type: string;
size: number;
lastModified: number;
filePath: string;
};
export async function ensureDocumentsV1Ready(): Promise<boolean> {
await mkdir(DOCSTORE_DIR, { recursive: true });
await mkdir(DOCUMENTS_V1_DIR, { recursive: true });

View file

@ -0,0 +1,222 @@
import {
DeleteObjectCommand,
DeleteObjectsCommand,
GetObjectCommand,
HeadObjectCommand,
ListObjectsV2Command,
PutObjectCommand,
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { getS3Client, getS3Config } from '@/lib/server/s3';
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
function sanitizeNamespace(namespace: string | null): string | null {
if (!namespace) return null;
if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null;
return namespace;
}
function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream {
return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function';
}
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
if (Buffer.isBuffer(chunk)) {
chunks.push(chunk);
} else if (typeof chunk === 'string') {
chunks.push(Buffer.from(chunk));
} else {
chunks.push(Buffer.from(chunk as Uint8Array));
}
}
return Buffer.concat(chunks);
}
async function bodyToBuffer(body: unknown): Promise<Buffer> {
if (!body) return Buffer.alloc(0);
if (body instanceof Uint8Array) return Buffer.from(body);
if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
if (body instanceof ArrayBuffer) return Buffer.from(body);
if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) {
const maybe = body as { transformToByteArray?: () => Promise<Uint8Array> };
if (typeof maybe.transformToByteArray === 'function') {
return Buffer.from(await maybe.transformToByteArray());
}
}
if (isNodeReadableStream(body)) {
return streamToBuffer(body);
}
throw new Error('Unsupported S3 response body type');
}
export function isValidDocumentId(id: string): boolean {
return DOCUMENT_ID_REGEX.test(id);
}
export function documentKey(id: string, namespace: string | null): string {
if (!isValidDocumentId(id)) {
throw new Error(`Invalid document id: ${id}`);
}
const cfg = getS3Config();
const ns = sanitizeNamespace(namespace);
const nsSegment = ns ? `ns/${ns}/` : '';
return `${cfg.prefix}/documents_v1/${nsSegment}${id}`;
}
export async function presignPut(
id: string,
contentType: string,
namespace: string | null,
): Promise<{ url: string; headers: Record<string, string> }> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream';
const command = new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
ContentType: normalizedType,
IfNoneMatch: '*',
});
const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 });
return {
url,
headers: {
'Content-Type': normalizedType,
'If-None-Match': '*',
},
};
}
export async function headDocumentBlob(
id: string,
namespace: string | null,
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
return {
contentLength: Number(res.ContentLength ?? 0),
contentType: res.ContentType ?? null,
eTag: res.ETag ?? null,
};
}
export async function getDocumentRange(
id: string,
start: number,
endInclusive: number,
namespace: string | null,
): Promise<Buffer> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
const res = await client.send(
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
Range: `bytes=${Math.max(0, start)}-${Math.max(0, endInclusive)}`,
}),
);
return bodyToBuffer(res.Body);
}
export async function getDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
const res = await client.send(
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
}),
);
return bodyToBuffer(res.Body);
}
export async function putDocumentBlob(
id: string,
body: Buffer,
contentType: string,
namespace: string | null,
): Promise<void> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
await client.send(
new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
Body: body,
ContentType: contentType,
IfNoneMatch: '*',
}),
);
}
export async function deleteDocumentBlob(id: string, namespace: string | null): Promise<void> {
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
}
export function isMissingBlobError(error: unknown): boolean {
if (!error || typeof error !== 'object') return false;
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
if (maybe.$metadata?.httpStatusCode === 404) return true;
if (maybe.name === 'NotFound' || maybe.name === 'NoSuchKey') return true;
if (maybe.Code === 'NotFound' || maybe.Code === 'NoSuchKey') return true;
return false;
}
export async function deleteDocumentPrefix(prefix: string): Promise<number> {
const cfg = getS3Config();
const client = getS3Client();
const cleanedPrefix = prefix.replace(/^\/+/, '');
let deleted = 0;
let continuationToken: string | undefined;
do {
const listRes = await client.send(
new ListObjectsV2Command({
Bucket: cfg.bucket,
Prefix: cleanedPrefix,
ContinuationToken: continuationToken,
}),
);
const keys = (listRes.Contents ?? [])
.map((item) => item.Key)
.filter((value): value is string => typeof value === 'string' && value.length > 0);
if (keys.length > 0) {
const deleteRes = await client.send(
new DeleteObjectsCommand({
Bucket: cfg.bucket,
Delete: {
Objects: keys.map((Key) => ({ Key })),
Quiet: true,
},
}),
);
deleted += deleteRes.Deleted?.length ?? 0;
}
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
} while (continuationToken);
return deleted;
}

View file

@ -1,11 +1,6 @@
import path from 'path';
import { utimes } from 'fs/promises';
import type { DocumentType } from '@/types/documents';
export function isEnoent(error: unknown): boolean {
return typeof error === 'object' && error !== null && 'code' in error && (error as { code?: unknown }).code === 'ENOENT';
}
export function safeDocumentName(rawName: string, fallback: string): string {
const baseName = path.basename(rawName || fallback);
return baseName.replaceAll('\u0000', '').slice(0, 240) || fallback;
@ -18,16 +13,3 @@ export function toDocumentTypeFromName(name: string): DocumentType {
if (ext === '.docx') return 'docx';
return 'html';
}
export async function trySetFileMtime(filePath: string, lastModifiedMs: number): Promise<void> {
if (!Number.isFinite(lastModifiedMs)) return;
const mtime = new Date(lastModifiedMs);
if (Number.isNaN(mtime.getTime())) return;
try {
await utimes(filePath, mtime, mtime);
} catch (error) {
console.warn('Failed to set document mtime:', filePath, error);
}
}

View file

@ -111,6 +111,17 @@ export class RateLimiter {
return this.isPostgres() ? new Date() : Date.now();
}
// Use a transaction only when running with Postgres.
// better-sqlite3 transactions require sync callbacks and cannot be awaited.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private async runMutation<T>(fn: (conn: any) => Promise<T>): Promise<T> {
if (this.isPostgres()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return safeDb().transaction(async (tx: any) => fn(tx));
}
return fn(safeDb());
}
/**
* Check if a user can use TTS and increment their char count if allowed
*/
@ -147,16 +158,10 @@ export class RateLimiter {
try {
const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue;
// Use a DB transaction to avoid partial increments across buckets and to avoid
// non-transactional "rollback" logic that can corrupt counts under concurrency.
// Note: We intentionally allow a request to push a bucket over its limit; we only
// block when the bucket was already exhausted before this request starts updating.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return await safeDb().transaction(async (tx: any) => {
return await this.runMutation(async (conn) => {
// Ensure records exist for each bucket
for (const bucket of buckets) {
await tx.insert(userTtsChars)
await conn.insert(userTtsChars)
.values({
userId: bucket.key,
date: dateValue,
@ -172,7 +177,7 @@ export class RateLimiter {
// that start after the bucket is already exhausted, while still allowing a
// request to push the count over the limit.
for (const bucket of buckets) {
const updateResult = await tx.update(userTtsChars)
const updateResult = await conn.update(userTtsChars)
.set({
charCount: sql`${userTtsChars.charCount} + ${charCount}`,
updatedAt,
@ -191,7 +196,7 @@ export class RateLimiter {
// Fetch current counts
const bucketResults: Array<{ currentCount: number; limit: number }> = [];
for (const bucket of buckets) {
const result = await tx.select({ currentCount: userTtsChars.charCount })
const result = await conn.select({ currentCount: userTtsChars.charCount })
.from(userTtsChars)
.where(and(eq(userTtsChars.userId, bucket.key), eq(userTtsChars.date, dateValue)));

82
src/lib/server/s3.ts Normal file
View file

@ -0,0 +1,82 @@
import { S3Client } from '@aws-sdk/client-s3';
type S3Config = {
bucket: string;
region: string;
endpoint?: string;
accessKeyId: string;
secretAccessKey: string;
forcePathStyle: boolean;
prefix: string;
};
let cachedClient: S3Client | null = null;
let cachedConfig: S3Config | null = null;
function parseBool(value: string | undefined): boolean {
if (!value) return false;
const normalized = value.trim().toLowerCase();
return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
}
function normalizePrefix(prefix: string | undefined): string {
const base = (prefix || 'openreader').trim();
if (!base) return 'openreader';
return base.replace(/^\/+|\/+$/g, '');
}
function loadS3ConfigFromEnv(): S3Config | null {
const bucket = process.env.S3_BUCKET?.trim();
const region = process.env.S3_REGION?.trim();
const accessKeyId = process.env.S3_ACCESS_KEY_ID?.trim();
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY?.trim();
const endpoint = process.env.S3_ENDPOINT?.trim();
if (!bucket || !region || !accessKeyId || !secretAccessKey) {
return null;
}
return {
bucket,
region,
endpoint: endpoint || undefined,
accessKeyId,
secretAccessKey,
forcePathStyle: parseBool(process.env.S3_FORCE_PATH_STYLE),
prefix: normalizePrefix(process.env.S3_PREFIX),
};
}
export function isS3Configured(): boolean {
return loadS3ConfigFromEnv() !== null;
}
export function getS3Config(): S3Config {
if (cachedConfig) return cachedConfig;
const config = loadS3ConfigFromEnv();
if (!config) {
throw new Error(
'S3 is not configured. Required env vars: S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.',
);
}
cachedConfig = config;
return config;
}
export function getS3Client(): S3Client {
if (cachedClient) return cachedClient;
const config = getS3Config();
cachedClient = new S3Client({
region: config.region,
endpoint: config.endpoint,
forcePathStyle: config.forcePathStyle,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
});
return cachedClient;
}

View file

@ -251,7 +251,8 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async
await startGeneration(page);
// Progress card should appear with a Cancel button while chapters are being generated
const cancelButton = page.getByRole('button', { name: 'Cancel' });
const generationCard = page.locator('div', { hasText: 'Generating Audiobook' }).first();
const cancelButton = generationCard.getByRole('button', { name: 'Cancel' });
await expect(cancelButton).toBeVisible({ timeout: 60_000 });
await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 });
@ -266,8 +267,10 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async
// Now cancel the in-flight generation
await cancelButton.click();
// After cancellation, the inline progress card's Cancel button should be gone
await expect(page.getByRole('button', { name: 'Cancel' })).toHaveCount(0);
// Cancellation is asynchronous: wait for generation to settle before asserting
// that the inline progress card has disappeared.
await expect(page.getByRole('button', { name: 'Resume' })).toBeVisible({ timeout: 30_000 });
await expect(generationCard).toHaveCount(0, { timeout: 30_000 });
// After cancellation, wait for the chapter count to stabilize. In-flight TTS
// requests may still complete after we click cancel, so we poll until the

View file

@ -1,29 +1,11 @@
import fs from 'fs/promises';
import path from 'path';
async function listWorkerNamespaces(documentsRoot: string): Promise<string[]> {
let entries: Array<{ name: string; isDirectory: () => boolean }> = [];
try {
entries = await fs.readdir(documentsRoot, { withFileTypes: true });
} catch {
return [];
}
return entries
.filter((d) => d.isDirectory())
.map((d) => d.name)
.filter((name) => /^(chromium|firefox|webkit)-worker\d+$/.test(name));
}
import { deleteDocumentPrefix } from '../src/lib/server/documents-blobstore';
import { getS3Config, isS3Configured } from '../src/lib/server/s3';
export default async function globalTeardown(): Promise<void> {
const documentsRoot = path.join(process.cwd(), 'docstore', 'documents_v1');
const namespaces = await listWorkerNamespaces(documentsRoot);
if (!namespaces.length) return;
if (!isS3Configured()) return;
await Promise.all(
namespaces.map(async (ns) => {
const dir = path.join(documentsRoot, ns);
await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
}),
);
const config = getS3Config();
const nsRootPrefix = `${config.prefix}/documents_v1/ns/`;
await deleteDocumentPrefix(nsRootPrefix);
}

View file

@ -7,8 +7,8 @@ import { transferUserDocuments } from '../../src/lib/server/claim-data';
test.describe('transferUserDocuments', () => {
test('moves document rows to new user without PK conflicts', async () => {
process.env.BETTER_AUTH_URL = 'http://localhost:3003';
process.env.BETTER_AUTH_SECRET = 'test-secret';
process.env.BASE_URL = 'http://localhost:3003';
process.env.AUTH_SECRET = 'test-secret';
const sqlite = new Database(':memory:');
sqlite.exec(`
@ -72,4 +72,3 @@ test.describe('transferUserDocuments', () => {
expect(ids).toEqual(['doc-a', 'doc-b']);
});
});