feat(storage): implement S3/SeaweedFS blob storage for documents
Replaces local filesystem document storage with S3-compatible object storage. Adds embedded SeaweedFS 'weed mini' for local development and Docker deployments. Updates document upload flow to use presigned URLs with a server fallback proxy. Refactors auth configuration to use BASE_URL and AUTH_SECRET. BREAKING CHANGE: Renamed BETTER_AUTH_URL to BASE_URL and BETTER_AUTH_SECRET to AUTH_SECRET. Removed legacy document upload/content endpoints. Requires S3 environment variables (auto-configured for embedded SeaweedFS).
This commit is contained in:
parent
4a5f3060f2
commit
81d249ed52
38 changed files with 3314 additions and 887 deletions
30
.env.example
30
.env.example
|
|
@ -9,14 +9,30 @@ API_KEY=api_key_optional
|
|||
# Path to your local whisper.cpp CLI binary for STT timestamp generation
|
||||
WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli
|
||||
|
||||
# Auth (recommended for contributors and public instances)
|
||||
# (Optional) Auth is only enabled when **both** AUTH_SECRET and BASE_URL are set
|
||||
BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network)
|
||||
AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32`
|
||||
AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional trusted origins (BASE_URL is always trusted)
|
||||
# (Optional) Sign in w/ GitHub Configuration
|
||||
GITHUB_CLIENT_ID=
|
||||
GITHUB_CLIENT_SECRET=
|
||||
|
||||
# Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables.
|
||||
# Defaults to SQLite at docstore/sqlite3.db when not set.
|
||||
POSTGRES_URL=
|
||||
|
||||
# Auth (recommended for contributors and public instances)
|
||||
# (Optional) Auth is only enabled when **both** BETTER_AUTH_SECRET and BETTER_AUTH_URL are set
|
||||
BETTER_AUTH_URL=http://localhost:3003 # Your externally facing URL for this app
|
||||
BETTER_AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32`
|
||||
# (Optional) Sign in w/ GitHub Configuration
|
||||
GITHUB_CLIENT_ID=
|
||||
GITHUB_CLIENT_SECRET=
|
||||
# Embedded SeaweedFS weed mini config
|
||||
USE_EMBEDDED_WEED_MINI=
|
||||
WEED_MINI_DIR=
|
||||
WEED_MINI_WAIT_SEC=
|
||||
# S3 storage config (use with embedded weed mini or external S3-compatible storage)
|
||||
# For embedded weed mini, set explicit keys if you want stable credentials across restarts.
|
||||
S3_ACCESS_KEY_ID=
|
||||
S3_SECRET_ACCESS_KEY=
|
||||
S3_BUCKET=
|
||||
S3_REGION=
|
||||
# If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host.
|
||||
S3_ENDPOINT=
|
||||
S3_FORCE_PATH_STYLE=
|
||||
S3_PREFIX=
|
||||
|
|
|
|||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnpm-store
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
|
|
@ -52,4 +53,4 @@ node_modules/
|
|||
/playwright/.cache/
|
||||
|
||||
# vscode
|
||||
.vscode
|
||||
.vscode
|
||||
|
|
|
|||
12
Dockerfile
12
Dockerfile
|
|
@ -12,6 +12,10 @@ RUN git clone --depth 1 https://github.com/ggml-org/whisper.cpp.git && \
|
|||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF $( [ "$TARGETARCH" = "arm64" ] && echo "-DGGML_CPU_ARM_ARCH=armv8-a" || true ) && \
|
||||
cmake --build build -j
|
||||
|
||||
# Stage 1b: extract seaweedfs weed binary (for optional embedded weed mini)
|
||||
FROM chrislusf/seaweedfs:latest AS seaweedfs-builder
|
||||
RUN cp "$(command -v weed)" /tmp/weed
|
||||
|
||||
|
||||
# Stage 2: build the Next.js app
|
||||
FROM node:lts-alpine AS app-builder
|
||||
|
|
@ -42,7 +46,7 @@ FROM node:lts-alpine AS runner
|
|||
# Add runtime OS dependencies:
|
||||
# - ffmpeg: required for audiobook export and word-by-word alignment (/api/whisper)
|
||||
# - libreoffice-writer: required for DOCX → PDF conversion
|
||||
RUN apk add --no-cache ffmpeg libreoffice-writer
|
||||
RUN apk add --no-cache ca-certificates ffmpeg libreoffice-writer
|
||||
|
||||
# Install pnpm globally for running the app
|
||||
RUN npm install -g pnpm
|
||||
|
|
@ -56,6 +60,9 @@ COPY --from=app-builder /app ./
|
|||
# Copy the compiled whisper.cpp build output into the runtime image
|
||||
# (includes whisper-cli and its shared libraries, e.g. libwhisper.so, libggml.so)
|
||||
COPY --from=whisper-builder /opt/whisper.cpp/build /opt/whisper.cpp/build
|
||||
# Copy seaweedfs weed binary for optional embedded local S3.
|
||||
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
|
||||
RUN chmod +x /usr/local/bin/weed
|
||||
|
||||
# Point the app at the compiled whisper-cli binary and ensure its libs are discoverable
|
||||
ENV WHISPER_CPP_BIN=/opt/whisper.cpp/build/bin/whisper-cli
|
||||
|
|
@ -65,4 +72,5 @@ ENV LD_LIBRARY_PATH=/opt/whisper.cpp/build
|
|||
EXPOSE 3003
|
||||
|
||||
# Start the application
|
||||
CMD ["pnpm", "start"]
|
||||
ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"]
|
||||
CMD ["pnpm", "start:raw"]
|
||||
|
|
|
|||
189
README.md
189
README.md
|
|
@ -42,63 +42,110 @@ OpenReader WebUI is an open source text to speech document reader web app built
|
|||
|
||||
### 1. 🐳 Start the Docker container
|
||||
|
||||
Minimal (no persistence, auth disabled unless you set auth env vars):
|
||||
Minimal (auth disabled, embedded storage is ephemeral, no library import):
|
||||
|
||||
```bash
|
||||
docker run --name openreader-webui \
|
||||
--restart unless-stopped \
|
||||
-p 3003:3003 \
|
||||
-p 8333:8333 \
|
||||
ghcr.io/richardr1126/openreader-webui:latest
|
||||
```
|
||||
|
||||
Fully featured (persistent storage + server library import + KokoroFastAPI in Docker + optional auth):
|
||||
Fully featured (persistent storage, embedded SeaweedFS `weed mini` for documents, optional auth):
|
||||
|
||||
```bash
|
||||
docker run --name openreader-webui \
|
||||
--restart unless-stopped \
|
||||
-p 3003:3003 \
|
||||
-p 8333:8333 \
|
||||
-v openreader_docstore:/app/docstore \
|
||||
-v /path/to/your/library:/app/docstore/library:ro \
|
||||
-e API_BASE=http://host.docker.internal:8880/v1 \
|
||||
-e API_KEY=none \
|
||||
-e BETTER_AUTH_URL=http://localhost:3003 \
|
||||
-e BETTER_AUTH_SECRET=<paste_the_output_of_openssl_here> \
|
||||
-e BASE_URL=http://localhost:3003 \
|
||||
-e AUTH_SECRET=<paste_the_output_of_openssl_here> \
|
||||
ghcr.io/richardr1126/openreader-webui:latest
|
||||
```
|
||||
|
||||
You can remove the `/app/docstore/library` mount if you don't need server library import.
|
||||
You can remove both `BETTER_AUTH_*` env vars to keep auth disabled.
|
||||
You can remove either `BASE_URL` or `AUTH_SECRET` to keep auth disabled.
|
||||
|
||||
> **Notes:**
|
||||
>
|
||||
> - `API_BASE` should point to your TTS API server's base URL (if running Kokoro-FastAPI locally in Docker, use `http://host.docker.internal:8880/v1`).
|
||||
> - `BETTER_AUTH_URL` should be your externally-facing URL for this app (for example `https://reader.example.com` or `http://localhost:3003`).
|
||||
> - To enable auth, set **both** `BETTER_AUTH_URL` and `BETTER_AUTH_SECRET` generated with `openssl rand -base64 32`.
|
||||
> - OpenReader always uses a backend DB for server-side metadata. By default it uses SQLite at `/app/docstore/sqlite3.db` (persisted when `/app/docstore` is mounted).
|
||||
> - If you set `POSTGRES_URL`, the container uses Postgres instead of SQLite and will run migrations against it on startup. Ensure the database is accessible.
|
||||
Quick notes:
|
||||
- `API_BASE` should point to your TTS API server's base URL (for local Docker Kokoro, use `http://host.docker.internal:8880/v1`).
|
||||
- Expose `-p 8333:8333` for direct browser access to embedded SeaweedFS presigned URLs.
|
||||
- If port `8333` is not exposed, uploads still work via `/api/documents/blob/upload/fallback`.
|
||||
- To enable auth, set both `BASE_URL` and `AUTH_SECRET`.
|
||||
|
||||
<details>
|
||||
<summary><strong>Docker environment variables</strong> (Click to expand)</summary>
|
||||
<summary><strong>Docker networking and blob behavior</strong> (Click to expand)</summary>
|
||||
|
||||
**Ports**
|
||||
|
||||
- `3003` serves the OpenReader app and API routes.
|
||||
- `8333` serves embedded SeaweedFS S3 for browser direct blob access.
|
||||
|
||||
**Upload behavior**
|
||||
|
||||
- Primary upload path: browser uploads to presigned URL from `/api/documents/blob/upload/presign`.
|
||||
- Fallback upload path: `/api/documents/blob/upload/fallback` if direct upload fails.
|
||||
- Content serving path: `/api/documents/blob` (server-served bytes/snippets).
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Which Docker setup should I use?</strong> (Click to expand)</summary>
|
||||
|
||||
| Goal | Recommended setup |
|
||||
| --- | --- |
|
||||
| Fast local setup, no persistence | Minimal `docker run` example |
|
||||
| Persistent docs/audiobooks and optional auth | Fully featured `docker run` example with `/app/docstore` volume |
|
||||
| Browser direct blob uploads/downloads | Publish both `3003` and `8333` |
|
||||
| Private blob endpoint (no `8333` published) | Keep using app APIs; uploads use fallback proxy when direct presigned upload is unreachable |
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Common Docker environment variables</strong> (Click to expand)</summary>
|
||||
|
||||
| Variable | Purpose | Example / Notes |
|
||||
| --- | --- | --- |
|
||||
| `API_BASE` | Default TTS API base URL (server-side) | `http://host.docker.internal:8880/v1` |
|
||||
| `API_KEY` | Default TTS API key | `none` or your provider key |
|
||||
| `BETTER_AUTH_URL` | Enables auth when set with `BETTER_AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` |
|
||||
| `BETTER_AUTH_SECRET` | Enables auth when set with `BETTER_AUTH_URL` | Generate with `openssl rand -base64 32` |
|
||||
| `BASE_URL` | Enables auth when set with `AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` |
|
||||
| `AUTH_SECRET` | Enables auth when set with `BASE_URL` | Generate with `openssl rand -base64 32` |
|
||||
| `AUTH_TRUSTED_ORIGINS` | Extra allowed auth request origins | Comma-separated list; leave empty to trust only `BASE_URL` |
|
||||
| `POSTGRES_URL` | Use Postgres for server DB (metadata + auth tables) instead of SQLite | If set, startup migrations target Postgres |
|
||||
| `GITHUB_CLIENT_ID` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_SECRET` |
|
||||
| `GITHUB_CLIENT_SECRET` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_ID` |
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Blob and embedded storage environment variables</strong> (Click to expand)</summary>
|
||||
|
||||
| Variable | Purpose | Example / Notes |
|
||||
| --- | --- | --- |
|
||||
| `USE_EMBEDDED_WEED_MINI` | Start SeaweedFS before app startup | default: `true` when unset in shared entrypoint |
|
||||
| `WEED_MINI_DIR` | Data directory for embedded `weed mini` | default: `docstore/seaweedfs` |
|
||||
| `WEED_MINI_WAIT_SEC` | Startup wait timeout for embedded `weed mini` | default: `20` |
|
||||
| `S3_BUCKET` | S3 bucket used for document blobs | default: `openreader-documents` in embedded mode |
|
||||
| `S3_REGION` | S3 region used by AWS SDK | default: `us-east-1` in embedded mode |
|
||||
| `S3_ENDPOINT` | Custom endpoint for S3-compatible providers | default: `http://<BASE_URL host>:8333` when `BASE_URL` is set, otherwise detected LAN host |
|
||||
| `S3_ACCESS_KEY_ID` | S3 access key | auto-generated in embedded mode if unset |
|
||||
| `S3_SECRET_ACCESS_KEY` | S3 secret key | auto-generated in embedded mode if unset |
|
||||
| `S3_FORCE_PATH_STYLE` | Force path-style addressing | default: `true` in embedded mode |
|
||||
| `S3_PREFIX` | Prefix for stored object keys | default: `openreader` |
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Docker volume mounts</strong> (Click to expand)</summary>
|
||||
|
||||
| Mount | Type | Recommended | Purpose | Example |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `/app/docstore` | Docker named volume | Yes | Persists server-side storage (documents, audiobook exports, settings, SQLite DB if used) | `-v openreader_docstore:/app/docstore` |
|
||||
| `/app/docstore/library` | Bind mount (host folder) | Optional + `:ro` | Exposes an existing folder of documents for **Server Library Import** | `-v /path/to/your/library:/app/docstore/library:ro` |
|
||||
| `/app/docstore` | Docker named volume | Yes (if you want persistence) | Persists embedded SeaweedFS blob data (`docstore/seaweedfs`), SQLite metadata DB (`docstore/sqlite3.db` when `POSTGRES_URL` is unset), audiobook export artifacts, and migration/runtime files | `-v openreader_docstore:/app/docstore` |
|
||||
| `/app/docstore/library` | Bind mount (host folder) | Optional + `:ro` | Read-only source directory for **Server Library Import**; files are imported/copied into browser storage, not modified in place | `-v /path/to/your/library:/app/docstore/library:ro` |
|
||||
|
||||
To import from the mounted library: **Settings → Documents → Server Library Import**
|
||||
|
||||
|
|
@ -117,8 +164,9 @@ OpenReader WebUI is an open source text to speech document reader web app built
|
|||
### 3. ⬆️ Updating Docker Image
|
||||
|
||||
```bash
|
||||
docker stop openreader-webui && \
|
||||
docker rm openreader-webui && \
|
||||
docker stop openreader-webui || true && \
|
||||
docker rm openreader-webui || true && \
|
||||
docker image rm ghcr.io/richardr1126/openreader-webui:latest || true && \
|
||||
docker pull ghcr.io/richardr1126/openreader-webui:latest
|
||||
```
|
||||
|
||||
|
|
@ -194,7 +242,15 @@ docker run -d \
|
|||
```
|
||||
|
||||
- A TTS API server (Kokoro-FastAPI, Orpheus-FastAPI, Deepinfra, OpenAI, etc.) running and accessible
|
||||
Optionally required for different features:
|
||||
- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) `weed` binary (required)
|
||||
|
||||
```bash
|
||||
brew install seaweedfs
|
||||
```
|
||||
|
||||
> **Note:** Verify install with `weed version`.
|
||||
|
||||
#### Optionally required for different features:
|
||||
- [FFmpeg](https://ffmpeg.org) (required for audiobook m4b creation only)
|
||||
|
||||
```bash
|
||||
|
|
@ -248,16 +304,52 @@ Optionally required for different features:
|
|||
|
||||
Auth is recommended for contributors and is enabled when **both** values are set:
|
||||
|
||||
- Set `BETTER_AUTH_URL` to your local URL (default: `http://localhost:3003`)
|
||||
- Generate a `BETTER_AUTH_SECRET` and paste it into `.env`:
|
||||
- Set `BASE_URL` to your local URL (default: `http://localhost:3003`)
|
||||
- Generate a `AUTH_SECRET` and paste it into `.env`:
|
||||
|
||||
```bash
|
||||
openssl rand -base64 32
|
||||
```
|
||||
- (Optional) If you use both localhost and LAN URL, set `AUTH_TRUSTED_ORIGINS` (leave empty to trust only `BASE_URL`):
|
||||
|
||||
> Note: To disable auth, remove either `BETTER_AUTH_URL` or `BETTER_AUTH_SECRET`.
|
||||
```bash
|
||||
AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://192.168.0.116:3003
|
||||
```
|
||||
|
||||
The embedded weed mini object store is enabled by default for local development, and started through the shared entrypoint. The ACCESS_KEY_ID and SECRET_ACCESS_KEY are auto-generated if unset, but for stable credentials across restarts, you can generate and set them in `.env`:
|
||||
- (Optional) Generate `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` and paste them into `.env` for stable credentials:
|
||||
|
||||
```bash
|
||||
S3_ACCESS_KEY_ID=<`openssl rand -hex 16`>
|
||||
S3_SECRET_ACCESS_KEY=<`openssl rand -hex 32`>
|
||||
```
|
||||
- (Optional) Connect to external S3-compatible storage instead of embedded `weed mini`:
|
||||
|
||||
```bash
|
||||
USE_EMBEDDED_WEED_MINI=false
|
||||
|
||||
# Required
|
||||
S3_BUCKET=your-bucket
|
||||
S3_REGION=us-east-1
|
||||
S3_ACCESS_KEY_ID=your-access-key
|
||||
S3_SECRET_ACCESS_KEY=your-secret-key
|
||||
|
||||
# Optional / provider-specific
|
||||
S3_ENDPOINT=
|
||||
S3_FORCE_PATH_STYLE=
|
||||
S3_PREFIX=openreader
|
||||
```
|
||||
|
||||
Notes:
|
||||
- For AWS S3: usually leave `S3_ENDPOINT` empty and `S3_FORCE_PATH_STYLE` empty/false.
|
||||
- For MinIO/SeaweedFS/R2/B2 S3: set `S3_ENDPOINT` to your endpoint URL and set `S3_FORCE_PATH_STYLE=true` when required by that provider.
|
||||
|
||||
> Notes:
|
||||
> - The base URL for the TTS API should be accessible and relative to the Next.js server
|
||||
>
|
||||
> Note: The base URL for the TTS API should be accessible and relative to the Next.js server
|
||||
> - To disable auth, remove either `BASE_URL` or `AUTH_SECRET`.
|
||||
>
|
||||
> - If S3 credentials are unset, they are auto-generated per startup.
|
||||
|
||||
4. Run DB migrations:
|
||||
|
||||
|
|
@ -316,6 +408,10 @@ This project would not be possible without standing on the shoulders of these gi
|
|||
|
||||
- [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M) model
|
||||
- [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI)
|
||||
- [Better Auth](https://www.better-auth.com/)
|
||||
- [SQLite](https://www.sqlite.org/)
|
||||
- [PostgreSQL](https://www.postgresql.org/)
|
||||
- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`)
|
||||
- [whisper.cpp](https://github.com/ggerganov/whisper.cpp)
|
||||
- [ffmpeg](https://ffmpeg.org)
|
||||
- [react-pdf](https://github.com/wojtekmaj/react-pdf) npm package
|
||||
|
|
@ -328,31 +424,26 @@ This project would not be possible without standing on the shoulders of these gi
|
|||
|
||||
## Stack
|
||||
|
||||
- **Framework:** Next.js (React)
|
||||
- **Containerization:** Docker
|
||||
- **Storage:**
|
||||
- [Dexie.js](https://dexie.org/) IndexedDB wrapper for client-side storage
|
||||
- **PDF:**
|
||||
- [react-pdf](https://github.com/wojtekmaj/react-pdf)
|
||||
- [pdf.js](https://mozilla.github.io/pdf.js/)
|
||||
- **EPUB:**
|
||||
- [react-reader](https://github.com/gerhardsletten/react-reader)
|
||||
- [epubjs](https://github.com/futurepress/epub.js/)
|
||||
- **Markdown/Text:**
|
||||
- [react-markdown](https://github.com/remarkjs/react-markdown)
|
||||
- [remark-gfm](https://github.com/remarkjs/remark-gfm)
|
||||
- **UI:**
|
||||
- [Tailwind CSS](https://tailwindcss.com)
|
||||
- [Headless UI](https://headlessui.com)
|
||||
- [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin)
|
||||
- **TTS:** (tested on)
|
||||
- [Deepinfra API](https://deepinfra.com) (Kokoro-82M, Orpheus-3B, Sesame-1B)
|
||||
- [Kokoro FastAPI TTS](https://github.com/remsky/Kokoro-FastAPI/tree/v0.0.5post1-stable)
|
||||
- [Orpheus FastAPI TTS](https://github.com/Lex-au/Orpheus-FastAPI)
|
||||
- **NLP:**
|
||||
- [compromise](https://github.com/spencermountain/compromise) NLP library for sentence splitting
|
||||
- [cmpstr](https://github.com/remsky/cmpstr) String comparison library
|
||||
- [whisper.cpp](https://github.com/ggerganov/whisper.cpp) for TTS timestamps (word-by-word highlighting)
|
||||
- **Framework:** [Next.js](https://nextjs.org/) 15 (App Router), [React](https://react.dev/) 19, [TypeScript](https://www.typescriptlang.org/)
|
||||
- **Containerization / Runtime:** [Docker](https://www.docker.com/) (linux/amd64 + linux/arm64), with a shared entrypoint that can bootstrap embedded SeaweedFS before app startup
|
||||
- **Next.js Client:**
|
||||
- **UI:** [Tailwind CSS](https://tailwindcss.com), [Headless UI](https://headlessui.com), [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin)
|
||||
- **Interactions:** `react-dnd`, `react-dropzone`
|
||||
- **Authentication:** [Better Auth](https://www.better-auth.com/) client SDK (`better-auth/react`, anonymous client plugin)
|
||||
- **Local storage/cache:** [Dexie.js](https://dexie.org/) (IndexedDB) for documents, cache, and app settings
|
||||
- **Document rendering:**
|
||||
- **PDF:** [react-pdf](https://github.com/wojtekmaj/react-pdf), [pdf.js](https://mozilla.github.io/pdf.js/)
|
||||
- **EPUB:** [react-reader](https://github.com/gerhardsletten/react-reader), [epubjs](https://github.com/futurepress/epub.js/)
|
||||
- **Markdown/Text:** [react-markdown](https://github.com/remarkjs/react-markdown), [remark-gfm](https://github.com/remarkjs/remark-gfm)
|
||||
- **Text preprocessing / matching:** [compromise](https://github.com/spencermountain/compromise), [cmpstr](https://github.com/remsky/cmpstr)
|
||||
- **Next.js Server:**
|
||||
- **APIs:** Next.js Route Handlers for document sync, blob/content access, migration flows, audiobook export, and TTS/Whisper proxying
|
||||
- **Authentication:** [Better Auth](https://www.better-auth.com/) server handlers/adapters for session and auth routing
|
||||
- **Text preprocessing / NLP utilities:** [compromise](https://github.com/spencermountain/compromise) via shared `lib/nlp` helpers used in server processing paths
|
||||
- **Metadata database:** [Drizzle ORM](https://orm.drizzle.team/), [SQLite](https://www.sqlite.org/) (`better-sqlite3`) by default, optional [PostgreSQL](https://www.postgresql.org/) (`pg`)
|
||||
- **Blob/object storage:** embedded [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`) by default, or external S3-compatible storage via AWS SDK v3 (`@aws-sdk/client-s3`, presigned URLs, upload fallback proxy)
|
||||
- **Audio/processing pipeline:** OpenAI-compatible TTS providers (OpenAI, DeepInfra, Kokoro, Orpheus, custom), [ffmpeg](https://ffmpeg.org/) for audiobook assembly, optional [whisper.cpp](https://github.com/ggerganov/whisper.cpp) for word-level timestamps
|
||||
- **Tooling / Testing:** ESLint, TypeScript, [Playwright](https://playwright.dev/) end-to-end tests, and Drizzle migrations/generation scripts
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
services:
|
||||
kokoro-tts:
|
||||
container_name: kokoro-tts
|
||||
image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4
|
||||
ports:
|
||||
- "8880:8880"
|
||||
environment:
|
||||
# ONNX Optimization Settings for vectorized operations
|
||||
- ONNX_NUM_THREADS=8 # Maximize core usage for vectorized ops
|
||||
- ONNX_INTER_OP_THREADS=4 # Higher inter-op for parallel matrix operations
|
||||
- ONNX_EXECUTION_MODE=parallel
|
||||
- ONNX_OPTIMIZATION_LEVEL=all
|
||||
- ONNX_MEMORY_PATTERN=true
|
||||
- ONNX_ARENA_EXTEND_STRATEGY=kNextPowerOfTwo
|
||||
- API_LOG_LEVEL=DEBUG
|
||||
restart: unless-stopped
|
||||
|
||||
openreader-webui:
|
||||
container_name: openreader-webui
|
||||
image: ghcr.io/richardr1126/openreader-webui:latest
|
||||
environment:
|
||||
- API_BASE=http://host.docker.internal:8880/v1
|
||||
ports:
|
||||
- "3003:3003"
|
||||
volumes:
|
||||
- docstore:/app/docstore
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
docstore:
|
||||
|
|
@ -3,15 +3,19 @@
|
|||
"version": "v1.3.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev --turbopack -p 3003",
|
||||
"dev": "node scripts/openreader-entrypoint.mjs -- pnpm dev:raw",
|
||||
"dev:raw": "next dev --turbopack -p 3003",
|
||||
"build": "next build",
|
||||
"start": "node drizzle/scripts/migrate.mjs && next start -p 3003",
|
||||
"start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw",
|
||||
"start:raw": "next start -p 3003",
|
||||
"lint": "next lint",
|
||||
"test": "playwright test",
|
||||
"migrate": "node drizzle/scripts/migrate.mjs",
|
||||
"generate": "node drizzle/scripts/generate.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.985.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.985.0",
|
||||
"@headlessui/react": "^2.2.9",
|
||||
"@types/howler": "^2.2.12",
|
||||
"@types/uuid": "^10.0.0",
|
||||
|
|
|
|||
1252
pnpm-lock.yaml
1252
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
367
scripts/openreader-entrypoint.mjs
Normal file
367
scripts/openreader-entrypoint.mjs
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
#!/usr/bin/env node
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { once } from 'node:events';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import * as dotenv from 'dotenv';
|
||||
|
||||
function loadEnvFiles() {
|
||||
const cwd = process.cwd();
|
||||
const envPath = path.join(cwd, '.env');
|
||||
const envLocalPath = path.join(cwd, '.env.local');
|
||||
|
||||
if (fs.existsSync(envPath)) {
|
||||
dotenv.config({ path: envPath });
|
||||
}
|
||||
if (fs.existsSync(envLocalPath)) {
|
||||
dotenv.config({ path: envLocalPath, override: true });
|
||||
}
|
||||
}
|
||||
|
||||
function isTrue(value, defaultValue) {
|
||||
if (value == null || value.trim() === '') return defaultValue;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
|
||||
}
|
||||
|
||||
function withDefault(value, fallback) {
|
||||
return value && value.trim() ? value.trim() : fallback;
|
||||
}
|
||||
|
||||
function isPrivateIPv4(address) {
|
||||
if (!address) return false;
|
||||
if (address.startsWith('10.')) return true;
|
||||
if (address.startsWith('192.168.')) return true;
|
||||
const m = /^172\.(\d+)\./.exec(address);
|
||||
if (m) {
|
||||
const second = Number.parseInt(m[1], 10);
|
||||
if (second >= 16 && second <= 31) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function detectHostForDefaultEndpoint() {
|
||||
const interfaces = os.networkInterfaces();
|
||||
const ipv4 = [];
|
||||
|
||||
for (const entries of Object.values(interfaces)) {
|
||||
for (const entry of entries || []) {
|
||||
if (!entry) continue;
|
||||
const family = typeof entry.family === 'string' ? entry.family : String(entry.family);
|
||||
if (family !== 'IPv4') continue;
|
||||
if (entry.internal) continue;
|
||||
ipv4.push(entry.address);
|
||||
}
|
||||
}
|
||||
|
||||
const privateAddr = ipv4.find(isPrivateIPv4);
|
||||
if (privateAddr) return privateAddr;
|
||||
if (ipv4[0]) return ipv4[0];
|
||||
return '127.0.0.1';
|
||||
}
|
||||
|
||||
function parseS3Endpoint(endpoint) {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(endpoint);
|
||||
} catch {
|
||||
throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`);
|
||||
}
|
||||
|
||||
if (!url.hostname) {
|
||||
throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`);
|
||||
}
|
||||
|
||||
const port = Number.parseInt(url.port || '8333', 10);
|
||||
if (!Number.isFinite(port) || port < 1 || port > 65535) {
|
||||
throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`);
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: url.hostname,
|
||||
port,
|
||||
normalized: `${url.protocol}//${url.hostname}:${port}`,
|
||||
};
|
||||
}
|
||||
|
||||
function parseUrlHost(urlValue, fieldName) {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(urlValue);
|
||||
} catch {
|
||||
throw new Error(`Invalid ${fieldName}: ${urlValue}`);
|
||||
}
|
||||
|
||||
if (!url.hostname) {
|
||||
throw new Error(`Invalid ${fieldName} host: ${urlValue}`);
|
||||
}
|
||||
|
||||
return url.hostname;
|
||||
}
|
||||
|
||||
function parseCommandFromArgs(argv) {
|
||||
const marker = argv.indexOf('--');
|
||||
if (marker >= 0) return argv.slice(marker + 1);
|
||||
return argv;
|
||||
}
|
||||
|
||||
function forwardChildStream(stream, target) {
|
||||
if (!stream) return () => {};
|
||||
const onData = (chunk) => {
|
||||
target.write(chunk);
|
||||
};
|
||||
stream.on('data', onData);
|
||||
return () => {
|
||||
stream.off('data', onData);
|
||||
};
|
||||
}
|
||||
|
||||
function hasWeedBinary() {
|
||||
const probe = spawnSync('weed', ['version'], { stdio: 'ignore' });
|
||||
if (probe.error) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
async function waitForEndpoint(url, timeoutSeconds) {
|
||||
const waitMs = Math.max(1, timeoutSeconds) * 1000;
|
||||
const deadline = Date.now() + waitMs;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const res = await fetch(url, { method: 'GET' });
|
||||
if (res) return;
|
||||
} catch {
|
||||
// retry
|
||||
}
|
||||
await delay(1000);
|
||||
}
|
||||
|
||||
throw new Error(`Embedded weed mini did not become ready at ${url} within ${timeoutSeconds}s.`);
|
||||
}
|
||||
|
||||
function spawnMainCommand(command, env) {
|
||||
const [cmd, ...args] = command;
|
||||
const child = spawn(cmd, args, {
|
||||
env,
|
||||
stdio: 'inherit',
|
||||
shell: process.platform === 'win32',
|
||||
});
|
||||
|
||||
const exitPromise = new Promise((resolve) => {
|
||||
child.on('error', (error) => {
|
||||
console.error('Failed to launch command:', error);
|
||||
resolve(1);
|
||||
});
|
||||
|
||||
child.on('exit', (code, signal) => {
|
||||
if (typeof code === 'number') {
|
||||
resolve(code);
|
||||
return;
|
||||
}
|
||||
if (signal) {
|
||||
resolve(1);
|
||||
return;
|
||||
}
|
||||
resolve(0);
|
||||
});
|
||||
});
|
||||
|
||||
return { child, exitPromise };
|
||||
}
|
||||
|
||||
function runDbMigrations(env) {
|
||||
const migrateScript = path.join(process.cwd(), 'drizzle', 'scripts', 'migrate.mjs');
|
||||
if (!fs.existsSync(migrateScript)) {
|
||||
throw new Error(`Could not find migration script at ${migrateScript}`);
|
||||
}
|
||||
|
||||
console.log('Running database migrations...');
|
||||
const migration = spawnSync(process.execPath, [migrateScript], {
|
||||
env,
|
||||
stdio: 'inherit',
|
||||
});
|
||||
|
||||
if (migration.error) {
|
||||
throw migration.error;
|
||||
}
|
||||
if (typeof migration.status === 'number' && migration.status !== 0) {
|
||||
throw new Error(`Database migrations failed with exit code ${migration.status}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function sendSignal(child, signal, useProcessGroup) {
|
||||
if (!child) return false;
|
||||
if (useProcessGroup && process.platform !== 'win32' && typeof child.pid === 'number' && child.pid > 0) {
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
child.kill(signal);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function terminateChild(child, signal = 'SIGTERM', graceMs = 3000, useProcessGroup = false) {
|
||||
if (!child) return;
|
||||
if (child.exitCode != null) return;
|
||||
|
||||
if (!sendSignal(child, signal, useProcessGroup)) return;
|
||||
|
||||
const exited = await Promise.race([
|
||||
once(child, 'exit').then(() => true).catch(() => true),
|
||||
delay(graceMs).then(() => false),
|
||||
]);
|
||||
|
||||
if (exited) return;
|
||||
|
||||
if (!sendSignal(child, 'SIGKILL', useProcessGroup)) return;
|
||||
|
||||
await Promise.race([
|
||||
once(child, 'exit').then(() => true).catch(() => true),
|
||||
delay(1000).then(() => false),
|
||||
]);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
loadEnvFiles();
|
||||
|
||||
const command = parseCommandFromArgs(process.argv.slice(2));
|
||||
if (command.length === 0) {
|
||||
console.error('Usage: node scripts/openreader-entrypoint.mjs -- <command> [args]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const embeddedEnvRaw = process.env.USE_EMBEDDED_WEED_MINI;
|
||||
let useEmbeddedWeed = isTrue(embeddedEnvRaw, true);
|
||||
|
||||
if (useEmbeddedWeed && !hasWeedBinary()) {
|
||||
if (embeddedEnvRaw && isTrue(embeddedEnvRaw, true)) {
|
||||
console.error('USE_EMBEDDED_WEED_MINI=true but `weed` binary is not available in PATH.');
|
||||
process.exit(1);
|
||||
}
|
||||
useEmbeddedWeed = false;
|
||||
console.warn('`weed` binary not found; skipping embedded SeaweedFS startup.');
|
||||
}
|
||||
|
||||
const runtimeEnv = { ...process.env };
|
||||
let weedProc = null;
|
||||
let weedExitPromise = Promise.resolve();
|
||||
let appProc = null;
|
||||
let shutdownPromise = null;
|
||||
let stopWeedStdoutForward = () => {};
|
||||
let stopWeedStderrForward = () => {};
|
||||
let didExit = false;
|
||||
|
||||
const exitOnce = (code) => {
|
||||
if (didExit) return;
|
||||
didExit = true;
|
||||
process.exit(code);
|
||||
};
|
||||
|
||||
const shutdown = async (signal = 'SIGTERM') => {
|
||||
if (shutdownPromise) return shutdownPromise;
|
||||
shutdownPromise = (async () => {
|
||||
await Promise.all([
|
||||
terminateChild(appProc, signal, 4000),
|
||||
terminateChild(weedProc, 'SIGTERM', 4000),
|
||||
]);
|
||||
await weedExitPromise;
|
||||
stopWeedStdoutForward();
|
||||
stopWeedStderrForward();
|
||||
})();
|
||||
return shutdownPromise;
|
||||
};
|
||||
|
||||
process.once('SIGINT', () => {
|
||||
void shutdown('SIGINT').finally(() => exitOnce(130));
|
||||
});
|
||||
process.once('SIGTERM', () => {
|
||||
void shutdown('SIGTERM').finally(() => exitOnce(143));
|
||||
});
|
||||
|
||||
try {
|
||||
const shouldRunDbMigrations = isTrue(runtimeEnv.RUN_DB_MIGRATIONS, true);
|
||||
if (shouldRunDbMigrations) {
|
||||
runDbMigrations(runtimeEnv);
|
||||
}
|
||||
|
||||
if (useEmbeddedWeed) {
|
||||
runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, 'docstore/seaweedfs');
|
||||
runtimeEnv.WEED_MINI_WAIT_SEC = withDefault(runtimeEnv.WEED_MINI_WAIT_SEC, '20');
|
||||
runtimeEnv.S3_BUCKET = withDefault(runtimeEnv.S3_BUCKET, 'openreader-documents');
|
||||
runtimeEnv.S3_REGION = withDefault(runtimeEnv.S3_REGION, 'us-east-1');
|
||||
const configuredBaseUrl = runtimeEnv.BASE_URL?.trim() || '';
|
||||
const baseUrlHost = configuredBaseUrl ? parseUrlHost(configuredBaseUrl, 'BASE_URL') : '';
|
||||
const configuredS3Endpoint = runtimeEnv.S3_ENDPOINT?.trim() || '';
|
||||
const defaultS3Host = baseUrlHost || detectHostForDefaultEndpoint();
|
||||
runtimeEnv.S3_ENDPOINT = configuredS3Endpoint || `http://${defaultS3Host}:8333`;
|
||||
runtimeEnv.S3_FORCE_PATH_STYLE = withDefault(runtimeEnv.S3_FORCE_PATH_STYLE, 'true');
|
||||
runtimeEnv.S3_PREFIX = withDefault(runtimeEnv.S3_PREFIX, 'openreader');
|
||||
runtimeEnv.S3_ACCESS_KEY_ID = withDefault(runtimeEnv.S3_ACCESS_KEY_ID, randomBytes(16).toString('hex'));
|
||||
runtimeEnv.S3_SECRET_ACCESS_KEY = withDefault(runtimeEnv.S3_SECRET_ACCESS_KEY, randomBytes(32).toString('hex'));
|
||||
runtimeEnv.AWS_ACCESS_KEY_ID = runtimeEnv.S3_ACCESS_KEY_ID;
|
||||
runtimeEnv.AWS_SECRET_ACCESS_KEY = runtimeEnv.S3_SECRET_ACCESS_KEY;
|
||||
|
||||
if (!runtimeEnv.S3_ACCESS_KEY_ID || !runtimeEnv.S3_SECRET_ACCESS_KEY) {
|
||||
throw new Error('Failed to initialize embedded S3 credentials.');
|
||||
}
|
||||
|
||||
fs.mkdirSync(runtimeEnv.WEED_MINI_DIR, { recursive: true });
|
||||
|
||||
console.log('Starting embedded SeaweedFS weed mini...');
|
||||
const weedArgs = ['mini', `-dir=${runtimeEnv.WEED_MINI_DIR}`];
|
||||
if (configuredS3Endpoint || baseUrlHost) {
|
||||
const endpoint = parseS3Endpoint(runtimeEnv.S3_ENDPOINT);
|
||||
weedArgs.push(`-ip=${endpoint.hostname}`);
|
||||
weedArgs.push(`-s3.port=${endpoint.port}`);
|
||||
}
|
||||
|
||||
weedProc = spawn('weed', weedArgs, {
|
||||
env: runtimeEnv,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
stopWeedStdoutForward = forwardChildStream(weedProc.stdout, process.stdout);
|
||||
stopWeedStderrForward = forwardChildStream(weedProc.stderr, process.stderr);
|
||||
weedExitPromise = once(weedProc, 'exit').then(() => undefined).catch(() => undefined);
|
||||
|
||||
weedProc.on('exit', (code, signal) => {
|
||||
if (typeof code === 'number' && code !== 0) {
|
||||
console.error(`Embedded weed mini exited with code ${code}.`);
|
||||
return;
|
||||
}
|
||||
if (signal) {
|
||||
console.error(`Embedded weed mini exited due to signal ${signal}.`);
|
||||
}
|
||||
});
|
||||
|
||||
const endpoint = runtimeEnv.S3_ENDPOINT;
|
||||
const waitSec = Number.parseInt(runtimeEnv.WEED_MINI_WAIT_SEC || '20', 10);
|
||||
await waitForEndpoint(endpoint, Number.isFinite(waitSec) ? waitSec : 20);
|
||||
console.log(`Embedded SeaweedFS is ready at ${endpoint}`);
|
||||
}
|
||||
|
||||
const { child, exitPromise } = spawnMainCommand(command, runtimeEnv);
|
||||
appProc = child;
|
||||
const exitCode = await exitPromise;
|
||||
|
||||
await shutdown('SIGTERM');
|
||||
exitOnce(exitCode);
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
await shutdown('SIGTERM');
|
||||
exitOnce(1);
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
|
|
@ -2,34 +2,18 @@ import { NextRequest, NextResponse } from 'next/server';
|
|||
import { createReadStream, existsSync } from 'fs';
|
||||
import { readdir, unlink } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
|
||||
import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
|
||||
import { findStoredChapterByIndex } from '@/lib/server/audiobook';
|
||||
import { db } from '@/db';
|
||||
import { audiobooks, audiobookChapters } from '@/db/schema';
|
||||
import { and, eq, inArray } from 'drizzle-orm';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { pruneAudiobookChapterIfMissingFile, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
/**
|
||||
* Get the base audiobooks directory, accounting for test namespaces.
|
||||
* When auth is disabled, returns AUDIOBOOKS_V1_DIR.
|
||||
* When auth is enabled, returns the user-specific directory.
|
||||
*/
|
||||
function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string {
|
||||
const namespace = getOpenReaderTestNamespace(request.headers);
|
||||
|
||||
if (!authEnabled || !userId) {
|
||||
return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace);
|
||||
}
|
||||
|
||||
const userDir = getUserAudiobookDir(userId);
|
||||
return applyOpenReaderTestNamespacePath(userDir, namespace);
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const bookId = request.nextUrl.searchParams.get('bookId');
|
||||
|
|
@ -77,7 +61,14 @@ export async function GET(request: NextRequest) {
|
|||
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`);
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: existingBook.userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
const dirExists = existsSync(intermediateDir);
|
||||
if (!dirExists) {
|
||||
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);
|
||||
|
|
@ -185,7 +176,14 @@ export async function DELETE(request: NextRequest) {
|
|||
),
|
||||
);
|
||||
|
||||
const intermediateDir = join(getAudiobooksRootDir(request, storageUserId, authEnabled), `${bookId}-audiobook`);
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: storageUserId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
const chapterPrefix = `${String(chapterIndex + 1).padStart(4, '0')}__`;
|
||||
const files = await readdir(intermediateDir).catch(() => []);
|
||||
for (const file of files) {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { readFile, writeFile, mkdir, unlink, rm, rename, readdir } from 'fs/prom
|
|||
import { existsSync, createReadStream } from 'fs';
|
||||
import { basename, join } from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { AUDIOBOOKS_V1_DIR, ensureAudiobooksV1Ready, isAudiobooksV1Ready, getUserAudiobookDir } from '@/lib/server/docstore';
|
||||
import { ensureAudiobooksV1Ready, isAudiobooksV1Ready, getAudiobooksRootDir } from '@/lib/server/docstore';
|
||||
import { encodeChapterFileName, encodeChapterTitleTag, listStoredChapters, ffprobeAudio, escapeFFMetadata } from '@/lib/server/audiobook';
|
||||
import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
|
|
@ -13,35 +13,11 @@ import { audiobooks, audiobookChapters } from '@/db/schema';
|
|||
import { eq, and, inArray } from 'drizzle-orm';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
/**
|
||||
* Apply test namespace to a directory path if present in request headers.
|
||||
*/
|
||||
function applyTestNamespace(baseDir: string, request: NextRequest): string {
|
||||
const namespace = getOpenReaderTestNamespace(request.headers);
|
||||
return applyOpenReaderTestNamespacePath(baseDir, namespace);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the base audiobooks directory, accounting for test namespaces.
|
||||
* When auth is disabled, returns AUDIOBOOKS_V1_DIR (possibly with test namespace).
|
||||
* When auth is enabled, returns the user-specific directory under AUDIOBOOKS_USERS_DIR.
|
||||
*/
|
||||
function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string {
|
||||
// When auth is disabled, use the flat audiobooks_v1 directory
|
||||
if (!authEnabled || !userId) {
|
||||
return applyTestNamespace(AUDIOBOOKS_V1_DIR, request);
|
||||
}
|
||||
|
||||
// When auth is enabled, use user-specific directory
|
||||
const userDir = getUserAudiobookDir(userId);
|
||||
return applyTestNamespace(userDir, request);
|
||||
}
|
||||
|
||||
interface ConversionRequest {
|
||||
chapterTitle: string;
|
||||
buffer: TTSAudioBytes;
|
||||
|
|
@ -207,7 +183,14 @@ export async function POST(request: NextRequest) {
|
|||
})
|
||||
.onConflictDoNothing();
|
||||
|
||||
const intermediateDir = join(getAudiobooksRootDir(request, userId, ctxOrRes.authEnabled), `${bookId}-audiobook`);
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId,
|
||||
authEnabled: ctxOrRes.authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
// Create intermediate directory
|
||||
await mkdir(intermediateDir, { recursive: true });
|
||||
|
|
@ -434,7 +417,11 @@ export async function GET(request: NextRequest) {
|
|||
}
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir(request, existingBook.userId, authEnabled),
|
||||
getAudiobooksRootDir({
|
||||
userId: existingBook.userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
|
|
@ -657,7 +644,14 @@ export async function DELETE(request: NextRequest) {
|
|||
|
||||
await db.delete(audiobooks).where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId)));
|
||||
|
||||
const intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`);
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
// If directory doesn't exist, consider it already reset
|
||||
if (!existsSync(intermediateDir)) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
|
||||
import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
|
||||
import { listStoredChapters } from '@/lib/server/audiobook';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudiobookFormat, TTSAudiobookChapter } from '@/types/tts';
|
||||
|
|
@ -11,27 +11,11 @@ import { db } from '@/db';
|
|||
import { audiobooks } from '@/db/schema';
|
||||
import { eq, and, inArray } from 'drizzle-orm';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { pruneAudiobookChaptersNotOnDisk, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
/**
|
||||
* Get the base audiobooks directory, accounting for test namespaces.
|
||||
* When auth is disabled, returns AUDIOBOOKS_V1_DIR.
|
||||
* When auth is enabled, returns the user-specific directory.
|
||||
*/
|
||||
function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string {
|
||||
const namespace = getOpenReaderTestNamespace(request.headers);
|
||||
|
||||
if (!authEnabled || !userId) {
|
||||
return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace);
|
||||
}
|
||||
|
||||
const userDir = getUserAudiobookDir(userId);
|
||||
return applyOpenReaderTestNamespacePath(userDir, namespace);
|
||||
}
|
||||
|
||||
const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
function isSafeId(value: string): boolean {
|
||||
|
|
@ -80,7 +64,14 @@ export async function GET(request: NextRequest) {
|
|||
});
|
||||
}
|
||||
|
||||
const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`);
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: existingBook.userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
if (!existsSync(intermediateDir)) {
|
||||
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);
|
||||
|
|
|
|||
|
|
@ -1,16 +1,13 @@
|
|||
import { open, readFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { and, eq, inArray } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import { contentTypeForName } from '@/lib/server/library';
|
||||
import { extractRawTextSnippet } from '@/lib/text-snippets';
|
||||
import { isEnoent } from '@/lib/server/documents-utils';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { getDocumentBlob, getDocumentRange, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -19,67 +16,55 @@ function clampInt(value: number, min: number, max: number): number {
|
|||
return Math.max(min, Math.min(max, Math.trunc(value)));
|
||||
}
|
||||
|
||||
async function readHeadBuffer(filePath: string, maxBytes: number): Promise<Buffer> {
|
||||
const handle = await open(filePath, 'r');
|
||||
try {
|
||||
const buf = Buffer.allocUnsafe(maxBytes);
|
||||
const result = await handle.read(buf, 0, maxBytes, 0);
|
||||
return buf.subarray(0, result.bytesRead);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
await ensureDocumentsV1Ready();
|
||||
if (!(await isDocumentsV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
|
||||
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
|
||||
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
const url = new URL(req.url);
|
||||
const id = url.searchParams.get('id');
|
||||
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
|
||||
const format = (url.searchParams.get('format') || '').toLowerCase().trim();
|
||||
if (!id) {
|
||||
return NextResponse.json({ error: 'Missing id' }, { status: 400 });
|
||||
if (!isValidDocumentId(id)) {
|
||||
return NextResponse.json({ error: 'Invalid id' }, { status: 400 });
|
||||
}
|
||||
|
||||
const docs = await db
|
||||
const rows = (await db
|
||||
.select({ id: documents.id, userId: documents.userId, name: documents.name, filePath: documents.filePath })
|
||||
.from(documents)
|
||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
|
||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
|
||||
id: string;
|
||||
userId: string;
|
||||
name: string;
|
||||
filePath: string;
|
||||
}>;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const doc = docs.find((d: any) => d.userId === storageUserId) ?? docs[0];
|
||||
|
||||
if (!doc?.filePath) {
|
||||
const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0];
|
||||
if (!doc) {
|
||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const filePath = path.join(documentsDir, doc.filePath);
|
||||
const filename = doc.name || doc.filePath;
|
||||
const filename = doc.name || `${id}.bin`;
|
||||
const responseType = contentTypeForName(filename);
|
||||
|
||||
if (format === 'snippet') {
|
||||
const maxChars = clampInt(Number.parseInt(url.searchParams.get('maxChars') || '1600', 10), 100, 8000);
|
||||
const maxBytes = clampInt(Number.parseInt(url.searchParams.get('maxBytes') || '131072', 10), 4096, 1024 * 1024);
|
||||
|
||||
try {
|
||||
const head = await readHeadBuffer(filePath, maxBytes);
|
||||
const head = await getDocumentRange(id, 0, maxBytes - 1, testNamespace);
|
||||
const decoded = new TextDecoder().decode(new Uint8Array(head));
|
||||
const snippet = extractRawTextSnippet(decoded, maxChars);
|
||||
return NextResponse.json(
|
||||
|
|
@ -87,7 +72,7 @@ export async function GET(req: NextRequest) {
|
|||
{ headers: { 'Cache-Control': 'no-store' } },
|
||||
);
|
||||
} catch (error) {
|
||||
if (isEnoent(error)) {
|
||||
if (isMissingBlobError(error)) {
|
||||
await db
|
||||
.delete(documents)
|
||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
|
||||
|
|
@ -97,14 +82,22 @@ export async function GET(req: NextRequest) {
|
|||
}
|
||||
}
|
||||
|
||||
let content: ArrayBuffer;
|
||||
try {
|
||||
const buf = await readFile(filePath);
|
||||
content = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
||||
const content = await getDocumentBlob(id, testNamespace);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(content));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
return new NextResponse(stream, {
|
||||
headers: {
|
||||
'Content-Type': responseType,
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (isEnoent(error)) {
|
||||
// The DB can become stale if a file is deleted manually from the docstore.
|
||||
// Prune rows so the client stops showing ghost documents.
|
||||
if (isMissingBlobError(error)) {
|
||||
await db
|
||||
.delete(documents)
|
||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
|
||||
|
|
@ -112,13 +105,6 @@ export async function GET(req: NextRequest) {
|
|||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return new NextResponse(content, {
|
||||
headers: {
|
||||
'Content-Type': contentTypeForName(filename),
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error loading document content:', error);
|
||||
return NextResponse.json({ error: 'Failed to load document content' }, { status: 500 });
|
||||
51
src/app/api/documents/blob/upload/fallback/route.ts
Normal file
51
src/app/api/documents/blob/upload/fallback/route.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/test-namespace';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
function isPreconditionFailed(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
|
||||
}
|
||||
|
||||
export async function PUT(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const url = new URL(req.url);
|
||||
const id = (url.searchParams.get('id') || '').trim().toLowerCase();
|
||||
if (!isValidDocumentId(id)) {
|
||||
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
|
||||
}
|
||||
|
||||
const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream';
|
||||
const body = Buffer.from(await req.arrayBuffer());
|
||||
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||
|
||||
try {
|
||||
await putDocumentBlob(id, body, contentType, namespace);
|
||||
} catch (error) {
|
||||
if (!isPreconditionFailed(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({ success: true, id });
|
||||
} catch (error) {
|
||||
console.error('Error proxy-uploading document blob:', error);
|
||||
return NextResponse.json({ error: 'Failed to upload document blob' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
72
src/app/api/documents/blob/upload/presign/route.ts
Normal file
72
src/app/api/documents/blob/upload/presign/route.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { isValidDocumentId, presignPut } from '@/lib/server/documents-blobstore';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type PresignUpload = {
|
||||
id: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
};
|
||||
|
||||
function parseUploads(body: unknown): PresignUpload[] {
|
||||
if (!body || typeof body !== 'object') return [];
|
||||
const rawUploads = (body as { uploads?: unknown }).uploads;
|
||||
if (!Array.isArray(rawUploads)) return [];
|
||||
|
||||
const uploads: PresignUpload[] = [];
|
||||
for (const raw of rawUploads) {
|
||||
if (!raw || typeof raw !== 'object') continue;
|
||||
const rec = raw as Record<string, unknown>;
|
||||
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
|
||||
if (!isValidDocumentId(id)) continue;
|
||||
const contentType =
|
||||
typeof rec.contentType === 'string' && rec.contentType.trim()
|
||||
? rec.contentType.trim()
|
||||
: 'application/octet-stream';
|
||||
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
||||
uploads.push({ id, contentType, size });
|
||||
}
|
||||
return uploads;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const body = await req.json().catch(() => null);
|
||||
const uploads = parseUploads(body);
|
||||
if (uploads.length === 0) {
|
||||
return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 });
|
||||
}
|
||||
|
||||
const namespace = getOpenReaderTestNamespace(req.headers);
|
||||
const signed = await Promise.all(
|
||||
uploads.map(async (upload) => {
|
||||
const res = await presignPut(upload.id, upload.contentType, namespace);
|
||||
return {
|
||||
id: upload.id,
|
||||
url: res.url,
|
||||
headers: res.headers,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
return NextResponse.json({ uploads: signed });
|
||||
} catch (error) {
|
||||
console.error('Error creating document upload signatures:', error);
|
||||
return NextResponse.json({ error: 'Failed to presign uploads' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -6,11 +6,12 @@ import { existsSync } from 'fs';
|
|||
import { randomUUID, createHash } from 'crypto';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { safeDocumentName, trySetFileMtime } from '@/lib/server/documents-utils';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { safeDocumentName } from '@/lib/server/documents-utils';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { putDocumentBlob } from '@/lib/server/documents-blobstore';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||
|
|
@ -66,18 +67,15 @@ async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100)
|
|||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
await ensureDocumentsV1Ready();
|
||||
if (!(await isDocumentsV1Ready())) {
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
await mkdir(documentsDir, { recursive: true });
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
|
@ -96,7 +94,7 @@ export async function POST(req: NextRequest) {
|
|||
}
|
||||
|
||||
const docxBytes = Buffer.from(await file.arrayBuffer());
|
||||
// IMPORTANT: use sha of the source DOCX bytes for a stable ID across conversions.
|
||||
// Keep stable IDs tied to source bytes.
|
||||
const id = createHash('sha256').update(docxBytes).digest('hex');
|
||||
|
||||
const tempId = randomUUID();
|
||||
|
|
@ -113,18 +111,19 @@ export async function POST(req: NextRequest) {
|
|||
const pdfPath = await waitForPdfReady(jobDir);
|
||||
const pdfContent = await readFile(pdfPath);
|
||||
|
||||
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
|
||||
const targetFileName = `${id}__${encodeURIComponent(derivedName)}`;
|
||||
const targetPath = path.join(documentsDir, targetFileName);
|
||||
|
||||
try {
|
||||
await stat(targetPath);
|
||||
} catch {
|
||||
await writeFile(targetPath, pdfContent);
|
||||
await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace);
|
||||
} catch (error) {
|
||||
// Idempotent behavior: if blob already exists for this sha, continue.
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined;
|
||||
const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed';
|
||||
if (!isPreconditionFailed) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
|
||||
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
|
||||
await trySetFileMtime(targetPath, lastModified);
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
|
|
@ -135,9 +134,18 @@ export async function POST(req: NextRequest) {
|
|||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: targetFileName,
|
||||
filePath: id,
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
set: {
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
stored: {
|
||||
|
|
|
|||
|
|
@ -1,212 +1,207 @@
|
|||
import { createHash } from 'crypto';
|
||||
import { mkdir, readFile, stat, unlink, writeFile } from 'fs/promises';
|
||||
import { existsSync } from 'fs';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import path from 'path';
|
||||
import { DOCUMENTS_V1_DIR, ensureDocumentsV1Ready, isDocumentsV1Ready } from '@/lib/server/docstore';
|
||||
import type { BaseDocument, DocumentType, SyncedDocument } from '@/types/documents';
|
||||
import { and, count, eq, inArray } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { eq, and, inArray, count } from 'drizzle-orm';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import { isEnoent, toDocumentTypeFromName, trySetFileMtime } from '@/lib/server/documents-utils';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents-utils';
|
||||
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type RegisterDocument = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: DocumentType;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
};
|
||||
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType {
|
||||
if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') {
|
||||
return rawType;
|
||||
}
|
||||
return toDocumentTypeFromName(safeName);
|
||||
}
|
||||
|
||||
function normalizeLastModified(value: unknown): number {
|
||||
return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now();
|
||||
}
|
||||
|
||||
function parseDocumentPayload(body: unknown): RegisterDocument[] {
|
||||
if (!body || typeof body !== 'object') return [];
|
||||
const rawDocs = (body as { documents?: unknown }).documents;
|
||||
if (!Array.isArray(rawDocs)) return [];
|
||||
|
||||
const docs: RegisterDocument[] = [];
|
||||
for (const rawDoc of rawDocs) {
|
||||
if (!rawDoc || typeof rawDoc !== 'object') continue;
|
||||
const rec = rawDoc as Record<string, unknown>;
|
||||
const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : '';
|
||||
if (!isValidDocumentId(id)) continue;
|
||||
const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`;
|
||||
const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName);
|
||||
const type = normalizeDocumentType(rec.type, name);
|
||||
const lastModified = normalizeLastModified(rec.lastModified);
|
||||
const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0;
|
||||
docs.push({ id, name, type, size, lastModified });
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
await ensureDocumentsV1Ready();
|
||||
if (!(await isDocumentsV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
await mkdir(syncDir, { recursive: true });
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
|
||||
|
||||
const data = await req.json();
|
||||
const documentsData = data.documents as SyncedDocument[];
|
||||
const stored: Array<{ oldId: string; id: string; name: string }> = [];
|
||||
const body = await req.json().catch(() => null);
|
||||
const documentsData = parseDocumentPayload(body);
|
||||
if (documentsData.length === 0) {
|
||||
return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 });
|
||||
}
|
||||
|
||||
// Ensure directory exists (redundant with isDocumentsV1Ready but safe)
|
||||
// const SYNC_DIR = DOCUMENTS_V1_DIR;
|
||||
const stored: BaseDocument[] = [];
|
||||
|
||||
for (const doc of documentsData) {
|
||||
const content = Buffer.from(new Uint8Array(doc.data));
|
||||
const id = createHash('sha256').update(content).digest('hex');
|
||||
|
||||
const baseName = path.basename(doc.name || `${id}.${doc.type}`);
|
||||
const safeName = baseName.replaceAll('\u0000', '').slice(0, 240) || `${id}.${doc.type}`;
|
||||
|
||||
const targetFileName = `${id}__${encodeURIComponent(safeName)}`;
|
||||
const targetPath = path.join(syncDir, targetFileName);
|
||||
|
||||
// Write file if not exists
|
||||
let headSize = doc.size;
|
||||
try {
|
||||
await stat(targetPath);
|
||||
} catch {
|
||||
await writeFile(targetPath, content);
|
||||
const head = await headDocumentBlob(doc.id, testNamespace);
|
||||
if (head.contentLength > 0) headSize = head.contentLength;
|
||||
} catch (error) {
|
||||
if (isMissingBlobError(error)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`,
|
||||
},
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await trySetFileMtime(targetPath, doc.lastModified);
|
||||
|
||||
// DB Upsert
|
||||
// With composite PK (id, userId), we check if THIS user already has this document
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
id,
|
||||
id: doc.id,
|
||||
userId: storageUserId,
|
||||
name: safeName,
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: content.length,
|
||||
size: headSize,
|
||||
lastModified: doc.lastModified,
|
||||
filePath: targetFileName,
|
||||
filePath: doc.id,
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
set: {
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: headSize,
|
||||
lastModified: doc.lastModified,
|
||||
filePath: doc.id,
|
||||
},
|
||||
});
|
||||
|
||||
stored.push({ oldId: doc.id, id, name: safeName });
|
||||
stored.push({
|
||||
id: doc.id,
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: headSize,
|
||||
lastModified: doc.lastModified,
|
||||
scope: storageUserId === unclaimedUserId ? 'unclaimed' : 'user',
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json({ success: true, stored });
|
||||
} catch (error) {
|
||||
console.error('Error saving documents:', error);
|
||||
return NextResponse.json({ error: 'Failed to save documents' }, { status: 500 });
|
||||
console.error('Error registering documents:', error);
|
||||
return NextResponse.json({ error: 'Failed to register documents' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
await ensureDocumentsV1Ready();
|
||||
if (!(await isDocumentsV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
|
||||
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
|
||||
const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
const url = new URL(req.url);
|
||||
const list = url.searchParams.get('list') === 'true';
|
||||
const format = url.searchParams.get('format');
|
||||
const idsParam = url.searchParams.get('ids');
|
||||
const targetIds = idsParam
|
||||
? idsParam
|
||||
.split(',')
|
||||
.map((id) => id.trim().toLowerCase())
|
||||
.filter((id) => isValidDocumentId(id))
|
||||
: null;
|
||||
|
||||
// If list=true, force metadata only.
|
||||
// If format=metadata, force metadata only.
|
||||
// Otherwise include data.
|
||||
const includeData = !list && format !== 'metadata';
|
||||
|
||||
const targetIds = idsParam ? idsParam.split(',').filter(Boolean) : null;
|
||||
|
||||
// Query database for documents the user is allowed to access
|
||||
let allowedDocs: { id: string; userId: string; name: string; type: string; size: number; lastModified: number; filePath: string }[] = [];
|
||||
if (idsParam && (!targetIds || targetIds.length === 0)) {
|
||||
return NextResponse.json({ documents: [] });
|
||||
}
|
||||
|
||||
const conditions = [
|
||||
inArray(documents.userId, allowedUserIds),
|
||||
...(targetIds ? [inArray(documents.id, targetIds)] : []),
|
||||
...(targetIds && targetIds.length > 0 ? [inArray(documents.id, targetIds)] : []),
|
||||
];
|
||||
const rows = await db.select().from(documents).where(and(...conditions));
|
||||
allowedDocs = rows as unknown as { id: string; userId: string; name: string; type: string; size: number; lastModified: number; filePath: string }[];
|
||||
const rows = (await db.select().from(documents).where(and(...conditions))) as Array<{
|
||||
id: string;
|
||||
userId: string;
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
filePath: string;
|
||||
}>;
|
||||
|
||||
const results: (BaseDocument | SyncedDocument)[] = [];
|
||||
|
||||
for (const doc of allowedDocs) {
|
||||
const type: DocumentType =
|
||||
doc.type === 'pdf' || doc.type === 'epub' || doc.type === 'docx' || doc.type === 'html'
|
||||
? (doc.type as DocumentType)
|
||||
: toDocumentTypeFromName(doc.name);
|
||||
|
||||
// If the underlying file was deleted manually, keep the API self-healing:
|
||||
// prune the DB row so clients stop listing ghost documents.
|
||||
const absolutePath = doc.filePath ? path.join(syncDir, doc.filePath) : '';
|
||||
if (!absolutePath || !existsSync(absolutePath)) {
|
||||
await db
|
||||
.delete(documents)
|
||||
.where(and(eq(documents.id, doc.id), eq(documents.userId, doc.userId)));
|
||||
continue;
|
||||
}
|
||||
|
||||
const metadata: BaseDocument = {
|
||||
id: doc.id!,
|
||||
const results: BaseDocument[] = rows.map((doc) => {
|
||||
const type = normalizeDocumentType(doc.type, doc.name);
|
||||
return {
|
||||
id: doc.id,
|
||||
name: doc.name,
|
||||
size: doc.size,
|
||||
lastModified: doc.lastModified,
|
||||
size: Number(doc.size),
|
||||
lastModified: Number(doc.lastModified),
|
||||
type,
|
||||
scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user',
|
||||
};
|
||||
|
||||
if (!includeData) {
|
||||
results.push(metadata);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await readFile(absolutePath);
|
||||
results.push({
|
||||
...metadata,
|
||||
data: Array.from(new Uint8Array(content)),
|
||||
});
|
||||
} catch (err) {
|
||||
if (isEnoent(err)) {
|
||||
await db
|
||||
.delete(documents)
|
||||
.where(and(eq(documents.id, doc.id), eq(documents.userId, doc.userId)));
|
||||
continue;
|
||||
}
|
||||
console.warn(`Failed to read content for document ${doc.id} at ${doc.filePath}`, err);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return NextResponse.json({ documents: results });
|
||||
} catch (error) {
|
||||
console.error('Error loading documents:', error);
|
||||
console.error('Error loading document metadata:', error);
|
||||
return NextResponse.json({ error: 'Failed to load documents' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export async function DELETE(req: NextRequest) {
|
||||
try {
|
||||
await ensureDocumentsV1Ready();
|
||||
if (!(await isDocumentsV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
// Auth check - require session
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
|
||||
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
const url = new URL(req.url);
|
||||
const idsParam = url.searchParams.get('ids');
|
||||
const scopeParam = (url.searchParams.get('scope') || '').toLowerCase().trim();
|
||||
|
|
@ -221,7 +216,6 @@ export async function DELETE(req: NextRequest) {
|
|||
);
|
||||
}
|
||||
|
||||
// Deleting the global unclaimed pool is a privileged operation when auth is enabled.
|
||||
if (ctxOrRes.authEnabled && wantsUnclaimed && ctxOrRes.user?.isAnonymous) {
|
||||
return NextResponse.json({ error: 'Forbidden' }, { status: 403 });
|
||||
}
|
||||
|
|
@ -239,58 +233,45 @@ export async function DELETE(req: NextRequest) {
|
|||
return NextResponse.json({ success: true, deleted: 0 });
|
||||
}
|
||||
|
||||
// Determine which IDs to try to delete
|
||||
let targetIds: string[] = [];
|
||||
|
||||
if (idsParam) {
|
||||
targetIds = idsParam.split(',').filter(Boolean);
|
||||
targetIds = idsParam
|
||||
.split(',')
|
||||
.map((id) => id.trim().toLowerCase())
|
||||
.filter((id) => isValidDocumentId(id));
|
||||
} else {
|
||||
// Existing behavior was "nuke everything"; keep it scoped to the selected user buckets.
|
||||
const rows = await db
|
||||
const rows = (await db
|
||||
.select({ id: documents.id })
|
||||
.from(documents)
|
||||
.where(inArray(documents.userId, targetUserIds));
|
||||
targetIds = rows.map((d: { id: string | null }) => d.id!).filter(Boolean) as string[];
|
||||
.where(inArray(documents.userId, targetUserIds))) as Array<{ id: string }>;
|
||||
targetIds = rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
if (targetIds.length === 0) {
|
||||
return NextResponse.json({ success: true, deleted: 0 });
|
||||
}
|
||||
|
||||
const deletedRows: { id: string; filePath: string }[] = [];
|
||||
|
||||
const rows = await db
|
||||
const deletedRows = (await db
|
||||
.delete(documents)
|
||||
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))
|
||||
.returning({ id: documents.id, filePath: documents.filePath });
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
rows.forEach((r: any) => deletedRows.push({ id: r.id!, filePath: r.filePath }));
|
||||
.returning({ id: documents.id })) as Array<{ id: string }>;
|
||||
|
||||
// If driver doesn't support returning (e.g. older SQLite without properly configured returning), we might fallback.
|
||||
// But Drizzle usually handles this.
|
||||
const uniqueIds = Array.from(new Set(deletedRows.map((row) => row.id)));
|
||||
for (const id of uniqueIds) {
|
||||
const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, id));
|
||||
const refCount = Number(ref?.count ?? 0);
|
||||
if (refCount > 0) continue;
|
||||
|
||||
let deletedCount = 0;
|
||||
|
||||
for (const row of deletedRows) {
|
||||
deletedCount++;
|
||||
// Chech reference count for this ID
|
||||
// If 0 remaining, delete file
|
||||
let refCount = 0;
|
||||
const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, row.id!));
|
||||
refCount = Number(ref?.count ?? 0);
|
||||
|
||||
if (refCount === 0) {
|
||||
const filePath = path.join(syncDir, row.filePath);
|
||||
try {
|
||||
await unlink(filePath);
|
||||
} catch {
|
||||
// Ignore if missing
|
||||
try {
|
||||
await deleteDocumentBlob(id, testNamespace);
|
||||
} catch (error) {
|
||||
if (!isMissingBlobError(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({ success: true, deleted: deletedCount });
|
||||
|
||||
return NextResponse.json({ success: true, deleted: deletedRows.length });
|
||||
} catch (error) {
|
||||
console.error('Error deleting documents:', error);
|
||||
return NextResponse.json({ error: 'Failed to delete documents' }, { status: 500 });
|
||||
|
|
|
|||
|
|
@ -1,95 +0,0 @@
|
|||
import { createHash } from 'crypto';
|
||||
import { mkdir, stat, writeFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { safeDocumentName, toDocumentTypeFromName, trySetFileMtime } from '@/lib/server/documents-utils';
|
||||
import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
await ensureDocumentsV1Ready();
|
||||
if (!(await isDocumentsV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
await mkdir(documentsDir, { recursive: true });
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
const storageUserId = ctxOrRes.userId ?? unclaimedUserId;
|
||||
|
||||
const form = await req.formData();
|
||||
const files = form.getAll('files').filter((value): value is File => value instanceof File);
|
||||
|
||||
if (files.length === 0) {
|
||||
return NextResponse.json({ error: 'Missing files' }, { status: 400 });
|
||||
}
|
||||
|
||||
const stored: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'pdf' | 'epub' | 'docx' | 'html';
|
||||
size: number;
|
||||
lastModified: number;
|
||||
}> = [];
|
||||
|
||||
for (const file of files) {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const content = Buffer.from(new Uint8Array(arrayBuffer));
|
||||
const id = createHash('sha256').update(content).digest('hex');
|
||||
|
||||
const safeName = safeDocumentName(file.name, `${id}.${toDocumentTypeFromName(file.name)}`);
|
||||
const targetFileName = `${id}__${encodeURIComponent(safeName)}`;
|
||||
const targetPath = path.join(documentsDir, targetFileName);
|
||||
|
||||
try {
|
||||
await stat(targetPath);
|
||||
} catch {
|
||||
await writeFile(targetPath, content);
|
||||
}
|
||||
|
||||
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
|
||||
await trySetFileMtime(targetPath, lastModified);
|
||||
|
||||
const type = toDocumentTypeFromName(safeName);
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
id,
|
||||
userId: storageUserId,
|
||||
name: safeName,
|
||||
type,
|
||||
size: content.length,
|
||||
lastModified,
|
||||
filePath: targetFileName,
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
|
||||
stored.push({
|
||||
id,
|
||||
name: safeName,
|
||||
type,
|
||||
size: content.length,
|
||||
lastModified,
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json({ stored });
|
||||
} catch (error) {
|
||||
console.error('Error uploading documents:', error);
|
||||
return NextResponse.json({ error: 'Failed to upload documents' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
275
src/app/api/migrations/v2/route.ts
Normal file
275
src/app/api/migrations/v2/route.ts
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
import { createHash } from 'crypto';
|
||||
import { existsSync } from 'fs';
|
||||
import { readdir, readFile, stat, unlink } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { auth } from '@/lib/server/auth';
|
||||
import { DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
|
||||
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore';
|
||||
import { toDocumentTypeFromName } from '@/lib/server/documents-utils';
|
||||
import { contentTypeForName } from '@/lib/server/library';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import type { DocumentType } from '@/types/documents';
|
||||
import {
|
||||
applyOpenReaderTestNamespacePath,
|
||||
getOpenReaderTestNamespace,
|
||||
getUnclaimedUserIdForNamespace,
|
||||
} from '@/lib/server/test-namespace';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type V2Body = {
|
||||
deleteLocal?: boolean;
|
||||
dryRun?: boolean;
|
||||
};
|
||||
|
||||
type LegacyDocumentCandidate = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
};
|
||||
|
||||
function isPreconditionFailed(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
|
||||
}
|
||||
|
||||
function extractIdFromFileName(fileName: string): string | null {
|
||||
const match = /^([a-f0-9]{64})__/i.exec(fileName);
|
||||
if (!match) return null;
|
||||
const id = match[1].toLowerCase();
|
||||
return isValidDocumentId(id) ? id : null;
|
||||
}
|
||||
|
||||
function decodeNameFromFileName(fileName: string, id: string): string {
|
||||
const prefix = `${id}__`;
|
||||
if (!fileName.startsWith(prefix)) return `${id}.bin`;
|
||||
const encoded = fileName.slice(prefix.length);
|
||||
try {
|
||||
return decodeURIComponent(encoded);
|
||||
} catch {
|
||||
return `${id}.bin`;
|
||||
}
|
||||
}
|
||||
|
||||
function sniffBinaryDocumentType(bytes: Buffer): Exclude<DocumentType, 'html'> | null {
|
||||
// PDF signature: "%PDF-"
|
||||
if (bytes.length >= 5 && bytes.subarray(0, 5).toString('ascii') === '%PDF-') {
|
||||
return 'pdf';
|
||||
}
|
||||
|
||||
// ZIP signatures: PK..
|
||||
const isZip =
|
||||
bytes.length >= 4 &&
|
||||
bytes[0] === 0x50 &&
|
||||
bytes[1] === 0x4b &&
|
||||
(bytes[2] === 0x03 || bytes[2] === 0x05 || bytes[2] === 0x07) &&
|
||||
(bytes[3] === 0x04 || bytes[3] === 0x06 || bytes[3] === 0x08);
|
||||
if (!isZip) return null;
|
||||
|
||||
// EPUB/DOCX markers usually appear in ZIP local headers near the start.
|
||||
const probe = bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString('latin1');
|
||||
if (probe.includes('application/epub+zip') || probe.includes('META-INF/container.xml')) {
|
||||
return 'epub';
|
||||
}
|
||||
if (probe.includes('[Content_Types].xml') && probe.includes('word/')) {
|
||||
return 'docx';
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeNameForType(name: string, id: string, type: DocumentType): string {
|
||||
if (type === 'html') return name;
|
||||
const expectedExt = type === 'pdf' ? '.pdf' : type === 'epub' ? '.epub' : '.docx';
|
||||
if (name.toLowerCase().endsWith(expectedExt)) return name;
|
||||
const base = name.replace(/\.bin$/i, '');
|
||||
return `${base || id}${expectedExt}`;
|
||||
}
|
||||
|
||||
function contentTypeForDocument(type: DocumentType, name: string): string {
|
||||
if (type === 'pdf') return 'application/pdf';
|
||||
if (type === 'epub') return 'application/epub+zip';
|
||||
if (type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
||||
return contentTypeForName(name);
|
||||
}
|
||||
|
||||
async function migrateDocumentRowsToBlobHandle(dryRun: boolean): Promise<number> {
|
||||
const rows = (await db.select().from(documents)) as Array<{
|
||||
id: string;
|
||||
userId: string;
|
||||
filePath: string;
|
||||
}>;
|
||||
|
||||
let updated = 0;
|
||||
for (const row of rows) {
|
||||
if (!row.id || !row.userId || row.filePath === row.id) continue;
|
||||
updated++;
|
||||
if (dryRun) continue;
|
||||
await db
|
||||
.update(documents)
|
||||
.set({ filePath: row.id })
|
||||
.where(and(eq(documents.id, row.id), eq(documents.userId, row.userId)));
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
async function seedMissingDocumentRows(
|
||||
userId: string,
|
||||
candidates: LegacyDocumentCandidate[],
|
||||
dryRun: boolean,
|
||||
): Promise<number> {
|
||||
if (candidates.length === 0) return 0;
|
||||
|
||||
const existingRows = (await db.select().from(documents).where(eq(documents.userId, userId))) as Array<{
|
||||
id: string;
|
||||
}>;
|
||||
const existingIds = new Set(existingRows.map((row) => row.id));
|
||||
|
||||
const seen = new Set<string>();
|
||||
const toInsert: LegacyDocumentCandidate[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (seen.has(candidate.id)) continue;
|
||||
seen.add(candidate.id);
|
||||
if (existingIds.has(candidate.id)) continue;
|
||||
toInsert.push(candidate);
|
||||
}
|
||||
|
||||
if (toInsert.length === 0) return 0;
|
||||
if (dryRun) return toInsert.length;
|
||||
|
||||
await db.insert(documents).values(
|
||||
toInsert.map((candidate) => ({
|
||||
id: candidate.id,
|
||||
userId,
|
||||
name: candidate.name,
|
||||
type: candidate.type,
|
||||
size: candidate.size,
|
||||
lastModified: candidate.lastModified,
|
||||
filePath: candidate.id,
|
||||
})),
|
||||
).onConflictDoNothing();
|
||||
|
||||
return toInsert.length;
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth?.api.getSession({ headers: request.headers });
|
||||
if (auth && !session?.user) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'S3 is not configured. Set S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const raw = (await request.json().catch(() => ({}))) as V2Body;
|
||||
const dryRun = raw.dryRun === true;
|
||||
const deleteLocal = raw.deleteLocal === true;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const docsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
|
||||
if (!existsSync(docsDir)) {
|
||||
const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
dryRun,
|
||||
deleteLocal,
|
||||
docsDir,
|
||||
filesScanned: 0,
|
||||
uploaded: 0,
|
||||
alreadyPresent: 0,
|
||||
skippedInvalid: 0,
|
||||
deletedLocal: 0,
|
||||
dbRowsUpdated: rowsUpdated,
|
||||
dbRowsSeeded: 0,
|
||||
});
|
||||
}
|
||||
|
||||
const entries = await readdir(docsDir, { withFileTypes: true });
|
||||
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
||||
|
||||
let uploaded = 0;
|
||||
let alreadyPresent = 0;
|
||||
let skippedInvalid = 0;
|
||||
let deletedLocal = 0;
|
||||
const candidates: LegacyDocumentCandidate[] = [];
|
||||
|
||||
for (const fileName of files) {
|
||||
const fullPath = path.join(docsDir, fileName);
|
||||
const bytes = await readFile(fullPath);
|
||||
const fileStats = await stat(fullPath);
|
||||
|
||||
const extractedId = extractIdFromFileName(fileName);
|
||||
const id = extractedId ?? createHash('sha256').update(bytes).digest('hex');
|
||||
if (!isValidDocumentId(id)) {
|
||||
skippedInvalid++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const inferredName = decodeNameFromFileName(fileName, id);
|
||||
const inferredType = toDocumentTypeFromName(inferredName);
|
||||
const type = inferredType === 'html' ? (sniffBinaryDocumentType(bytes) ?? inferredType) : inferredType;
|
||||
const normalizedName = normalizeNameForType(inferredName, id, type);
|
||||
const contentType = contentTypeForDocument(type, normalizedName);
|
||||
const lastModified = Number.isFinite(fileStats.mtimeMs) ? Math.floor(fileStats.mtimeMs) : Date.now();
|
||||
|
||||
candidates.push({
|
||||
id,
|
||||
name: normalizedName,
|
||||
type,
|
||||
size: bytes.length,
|
||||
lastModified,
|
||||
});
|
||||
|
||||
if (!dryRun) {
|
||||
try {
|
||||
await putDocumentBlob(id, bytes, contentType, testNamespace);
|
||||
uploaded++;
|
||||
} catch (error) {
|
||||
if (isPreconditionFailed(error)) {
|
||||
alreadyPresent++;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
await unlink(fullPath).catch(() => {});
|
||||
deletedLocal++;
|
||||
}
|
||||
}
|
||||
|
||||
const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun);
|
||||
const rowsSeeded = await seedMissingDocumentRows(unclaimedUserId, candidates, dryRun);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
dryRun,
|
||||
deleteLocal,
|
||||
docsDir,
|
||||
filesScanned: files.length,
|
||||
uploaded,
|
||||
alreadyPresent,
|
||||
skippedInvalid,
|
||||
deletedLocal,
|
||||
dbRowsUpdated: rowsUpdated,
|
||||
dbRowsSeeded: rowsSeeded,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error running v2 migrations:', error);
|
||||
return NextResponse.json({ error: 'Failed to run v2 migrations' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,34 @@ import { NextRequest, NextResponse } from 'next/server';
|
|||
import { claimAnonymousData } from '@/lib/server/claim-data';
|
||||
import { auth } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed, getUnclaimedCounts } from '@/lib/server/db-indexing';
|
||||
import { isDocumentsV1Ready } from '@/lib/server/docstore';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { count, ne } from 'drizzle-orm';
|
||||
|
||||
async function checkClaimMigrationReadiness(): Promise<NextResponse | null> {
|
||||
const documentsV1Ready = await isDocumentsV1Ready();
|
||||
if (!documentsV1Ready) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Document migration is not ready. Run startup migrations first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const [legacyRows] = await db
|
||||
.select({ count: count() })
|
||||
.from(documents)
|
||||
.where(ne(documents.filePath, documents.id));
|
||||
|
||||
if (Number(legacyRows?.count ?? 0) > 0) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Document metadata migration is still pending. Wait for startup migrations to complete.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
|
|
@ -10,6 +38,9 @@ export async function GET(req: NextRequest) {
|
|||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
const readiness = await checkClaimMigrationReadiness();
|
||||
if (readiness) return readiness;
|
||||
|
||||
await ensureDbIndexed();
|
||||
const counts = await getUnclaimedCounts();
|
||||
return NextResponse.json({ success: true, ...counts });
|
||||
|
|
@ -26,6 +57,9 @@ export async function POST(req: NextRequest) {
|
|||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
const readiness = await checkClaimMigrationReadiness();
|
||||
if (readiness) return readiness;
|
||||
|
||||
const userId = session.user.id;
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
|
|
|||
|
|
@ -57,8 +57,9 @@ export default function HTMLPage() {
|
|||
}, [isLoading, id, router, setCurrentDocument, stop]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isLoading) return;
|
||||
loadDocument();
|
||||
}, [loadDocument]);
|
||||
}, [loadDocument, isLoading]);
|
||||
|
||||
// Compute available height = viewport - (header height + tts bar height)
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ import {
|
|||
extractEpubCoverToDataUrl,
|
||||
renderPdfFirstPageToDataUrl,
|
||||
} from '@/lib/documentPreview';
|
||||
import { downloadDocumentContent, getDocumentContentSnippet } from '@/lib/client-documents';
|
||||
import { getDocumentContentSnippet } from '@/lib/client-documents';
|
||||
import { ensureCachedDocument } from '@/lib/document-cache';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
|
||||
|
|
@ -36,6 +37,16 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
|||
const [isGenerating, setIsGenerating] = useState(false);
|
||||
|
||||
const previewKey = useMemo(() => `${doc.type}:${doc.id}`, [doc.id, doc.type]);
|
||||
const cacheMeta = useMemo(
|
||||
() => ({
|
||||
id: doc.id,
|
||||
name: doc.name,
|
||||
type: doc.type,
|
||||
size: doc.size,
|
||||
lastModified: doc.lastModified,
|
||||
}),
|
||||
[doc.id, doc.lastModified, doc.name, doc.size, doc.type],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const el = containerRef.current;
|
||||
|
|
@ -81,7 +92,9 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
|||
const targetWidth = 240;
|
||||
|
||||
if (doc.type === 'pdf') {
|
||||
const data = await downloadDocumentContent(doc.id, { signal: controller.signal });
|
||||
const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal });
|
||||
if (cached.type !== 'pdf') return;
|
||||
const data = cached.data;
|
||||
if (cancelled) return;
|
||||
const dataUrl = await renderPdfFirstPageToDataUrl(data, targetWidth);
|
||||
if (cancelled) return;
|
||||
|
|
@ -92,7 +105,9 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
|||
}
|
||||
|
||||
if (doc.type === 'epub') {
|
||||
const data = await downloadDocumentContent(doc.id, { signal: controller.signal });
|
||||
const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal });
|
||||
if (cached.type !== 'epub') return;
|
||||
const data = cached.data;
|
||||
if (cancelled) return;
|
||||
const cover = await extractEpubCoverToDataUrl(data, targetWidth);
|
||||
if (cancelled) return;
|
||||
|
|
@ -130,7 +145,7 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
|||
cancelled = true;
|
||||
controller.abort();
|
||||
};
|
||||
}, [doc.id, doc.type, isVisible, previewKey]);
|
||||
}, [cacheMeta, doc.id, doc.type, isVisible, previewKey]);
|
||||
|
||||
const gradientClass = isPDF
|
||||
? 'from-red-500/80 via-red-400/60 to-red-600/80'
|
||||
|
|
|
|||
|
|
@ -111,13 +111,40 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
|||
|
||||
if (response?.ok) {
|
||||
const data = await response.json();
|
||||
const didMigrate =
|
||||
const v2ApplyResponse = await fetch('/api/migrations/v2', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ dryRun: false, deleteLocal: false }),
|
||||
}).catch(() => null);
|
||||
const v2ApplyData = v2ApplyResponse?.ok ? await v2ApplyResponse.json().catch(() => null) : null;
|
||||
|
||||
const didMigrateV1 =
|
||||
data.documentsMigrated ||
|
||||
data.audiobooksMigrated ||
|
||||
(data.rekey?.renamed ?? 0) > 0 ||
|
||||
(data.rekey?.merged ?? 0) > 0;
|
||||
|
||||
if (didMigrate) {
|
||||
if (v2ApplyData) {
|
||||
const uploaded = Number(v2ApplyData.uploaded ?? 0);
|
||||
const alreadyPresent = Number(v2ApplyData.alreadyPresent ?? 0);
|
||||
const dbRowsUpdated = Number(v2ApplyData.dbRowsUpdated ?? 0);
|
||||
const dbRowsSeeded = Number(v2ApplyData.dbRowsSeeded ?? 0);
|
||||
const deletedLocal = Number(v2ApplyData.deletedLocal ?? 0);
|
||||
const dbRowsMigrated = dbRowsUpdated + dbRowsSeeded;
|
||||
const didMigrateV2 = uploaded > 0 || dbRowsMigrated > 0 || deletedLocal > 0;
|
||||
|
||||
if (didMigrateV2) {
|
||||
toast.success(
|
||||
`Legacy document migration complete: ${uploaded} uploaded, ${alreadyPresent} already in S3, ${dbRowsMigrated} DB row(s) migrated.`,
|
||||
{ duration: 6000, icon: '📦' },
|
||||
);
|
||||
window.dispatchEvent(new CustomEvent('openreader:documentsChanged', {
|
||||
detail: { reason: 'migration-v2-complete' },
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if (didMigrateV1) {
|
||||
toast.success('Library migration complete', {
|
||||
duration: 5000,
|
||||
icon: '📦',
|
||||
|
|
|
|||
|
|
@ -50,6 +50,19 @@ export function DocumentProvider({ children }: { children: ReactNode }) {
|
|||
});
|
||||
}, [refreshDocuments]);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = () => {
|
||||
refreshDocuments().catch((err) => {
|
||||
console.error('Failed to refresh documents after change event:', err);
|
||||
});
|
||||
};
|
||||
|
||||
window.addEventListener('openreader:documentsChanged', handler as EventListener);
|
||||
return () => {
|
||||
window.removeEventListener('openreader:documentsChanged', handler as EventListener);
|
||||
};
|
||||
}, [refreshDocuments]);
|
||||
|
||||
const docsByType = useMemo(() => {
|
||||
const pdfDocs = (docs ?? []).filter((d) => d.type === 'pdf') as Array<BaseDocument & { type: 'pdf' }>;
|
||||
const epubDocs = (docs ?? []).filter((d) => d.type === 'epub') as Array<BaseDocument & { type: 'epub' }>;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import {
|
|||
ReactNode,
|
||||
useCallback,
|
||||
useMemo,
|
||||
useEffect,
|
||||
useRef,
|
||||
} from 'react';
|
||||
import { getDocumentMetadata } from '@/lib/client-documents';
|
||||
import { ensureCachedDocument } from '@/lib/document-cache';
|
||||
|
|
@ -30,12 +32,16 @@ const HTMLContext = createContext<HTMLContextType | undefined>(undefined);
|
|||
*/
|
||||
export function HTMLProvider({ children }: { children: ReactNode }) {
|
||||
const { setText: setTTSText, stop } = useTTS();
|
||||
const setTTSTextRef = useRef(setTTSText);
|
||||
|
||||
// Current document state
|
||||
const [currDocData, setCurrDocData] = useState<string>();
|
||||
const [currDocName, setCurrDocName] = useState<string>();
|
||||
const [currDocText, setCurrDocText] = useState<string>();
|
||||
|
||||
useEffect(() => {
|
||||
setTTSTextRef.current = setTTSText;
|
||||
}, [setTTSText]);
|
||||
|
||||
/**
|
||||
* Clears all current document state and stops any active TTS
|
||||
|
|
@ -69,12 +75,12 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
|
|||
setCurrDocName(doc.name);
|
||||
setCurrDocData(doc.data);
|
||||
setCurrDocText(doc.data); // Use the same text for TTS
|
||||
setTTSText(doc.data);
|
||||
setTTSTextRef.current(doc.data);
|
||||
} catch (error) {
|
||||
console.error('Failed to get HTML document:', error);
|
||||
clearCurrDoc();
|
||||
}
|
||||
}, [clearCurrDoc, setTTSText]);
|
||||
}, [clearCurrDoc]);
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,22 +12,32 @@ function createAuthClientWithUrl(baseUrl: string) {
|
|||
// Cache for auth client instances by baseUrl
|
||||
const clientCache = new Map<string, ReturnType<typeof createAuthClientWithUrl>>();
|
||||
|
||||
function resolveAuthClientBaseUrl(baseUrl: string | null): string {
|
||||
if (typeof window !== 'undefined' && window.location?.origin) {
|
||||
// Always use same-origin in the browser so local hostname variants
|
||||
// (localhost vs LAN IP) do not break cookie/session bootstrap.
|
||||
return window.location.origin;
|
||||
}
|
||||
|
||||
if (baseUrl) return baseUrl;
|
||||
|
||||
throw new Error(
|
||||
'Cannot create auth client without baseUrl in a non-browser context. ' +
|
||||
'Use useAuthConfig() in components to get the properly configured baseUrl.'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory function to get auth client with specific baseUrl.
|
||||
* In components, prefer reading `baseUrl` from `useAuthConfig()` and then calling `getAuthClient(baseUrl)`.
|
||||
* @param baseUrl - The auth server base URL. If null, will throw an error.
|
||||
* @param baseUrl - Server-provided auth URL; in the browser we use same-origin automatically.
|
||||
*/
|
||||
export function getAuthClient(baseUrl: string | null) {
|
||||
if (!baseUrl) {
|
||||
throw new Error(
|
||||
'Cannot create auth client without baseUrl. ' +
|
||||
'Use useAuthConfig() in components to get the properly configured baseUrl.'
|
||||
);
|
||||
const resolvedBaseUrl = resolveAuthClientBaseUrl(baseUrl);
|
||||
|
||||
if (!clientCache.has(resolvedBaseUrl)) {
|
||||
clientCache.set(resolvedBaseUrl, createAuthClientWithUrl(resolvedBaseUrl));
|
||||
}
|
||||
|
||||
if (!clientCache.has(baseUrl)) {
|
||||
clientCache.set(baseUrl, createAuthClientWithUrl(baseUrl));
|
||||
}
|
||||
|
||||
return clientCache.get(baseUrl)!;
|
||||
return clientCache.get(resolvedBaseUrl)!;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,52 @@
|
|||
import type { BaseDocument } from '@/types/documents';
|
||||
import { sha256HexFromArrayBuffer } from '@/lib/sha256';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
export type UploadSource = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: DocumentType;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
contentType: string;
|
||||
body: Blob | ArrayBuffer | Uint8Array;
|
||||
};
|
||||
|
||||
type UploadOptions = {
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
function toUploadBody(body: UploadSource['body']): BodyInit {
|
||||
if (body instanceof Blob) return body;
|
||||
if (body instanceof ArrayBuffer) return body;
|
||||
return body as unknown as BodyInit;
|
||||
}
|
||||
|
||||
async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise<void> {
|
||||
const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': source.contentType || 'application/octet-stream' },
|
||||
body: toUploadBody(source.body),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || `Proxy upload failed (status ${res.status})`);
|
||||
}
|
||||
}
|
||||
|
||||
function documentTypeForName(name: string): DocumentType {
|
||||
const lower = name.toLowerCase();
|
||||
if (lower.endsWith('.pdf')) return 'pdf';
|
||||
if (lower.endsWith('.epub')) return 'epub';
|
||||
if (lower.endsWith('.docx')) return 'docx';
|
||||
return 'html';
|
||||
}
|
||||
|
||||
export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string {
|
||||
if (doc.type === 'pdf') return 'application/pdf';
|
||||
if (doc.type === 'epub') return 'application/epub+zip';
|
||||
if (doc.type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
||||
|
||||
const lower = doc.name.toLowerCase();
|
||||
if (lower.endsWith('.md') || lower.endsWith('.markdown') || lower.endsWith('.mdown') || lower.endsWith('.mkd')) {
|
||||
|
|
@ -13,7 +57,6 @@ export function mimeTypeForDoc(doc: Pick<BaseDocument, 'type' | 'name'>): string
|
|||
|
||||
export async function listDocuments(options?: { ids?: string[]; signal?: AbortSignal }): Promise<BaseDocument[]> {
|
||||
const params = new URLSearchParams();
|
||||
params.set('list', 'true');
|
||||
if (options?.ids?.length) {
|
||||
params.set('ids', options.ids.join(','));
|
||||
}
|
||||
|
|
@ -33,27 +76,114 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
|
|||
return docs[0] ?? null;
|
||||
}
|
||||
|
||||
export async function uploadDocuments(files: File[], options?: { signal?: AbortSignal }): Promise<BaseDocument[]> {
|
||||
const form = new FormData();
|
||||
for (const file of files) {
|
||||
form.append('files', file);
|
||||
}
|
||||
export async function uploadDocumentSources(sources: UploadSource[], options?: UploadOptions): Promise<BaseDocument[]> {
|
||||
if (sources.length === 0) return [];
|
||||
|
||||
const res = await fetch('/api/documents/upload', {
|
||||
const presignRes = await fetch('/api/documents/blob/upload/presign', {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
uploads: sources.map((source) => ({
|
||||
id: source.id,
|
||||
contentType: source.contentType,
|
||||
size: source.size,
|
||||
})),
|
||||
}),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to upload documents');
|
||||
if (!presignRes.ok) {
|
||||
const data = (await presignRes.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to prepare uploads');
|
||||
}
|
||||
|
||||
const data = (await res.json()) as { stored: BaseDocument[] };
|
||||
const presigned = (await presignRes.json()) as {
|
||||
uploads?: Array<{ id: string; url: string; headers?: Record<string, string> }>;
|
||||
};
|
||||
const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload]));
|
||||
|
||||
for (const source of sources) {
|
||||
const upload = byId.get(source.id);
|
||||
if (!upload?.url) {
|
||||
throw new Error(`Missing presigned upload for document ${source.id}`);
|
||||
}
|
||||
|
||||
let putError: unknown = null;
|
||||
try {
|
||||
const putRes = await fetch(upload.url, {
|
||||
method: 'PUT',
|
||||
headers: new Headers(upload.headers || {}),
|
||||
body: toUploadBody(source.body),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
// 412 means the content-hash object already exists (idempotent upload).
|
||||
if (putRes.ok || putRes.status === 412) {
|
||||
continue;
|
||||
}
|
||||
putError = new Error(`Direct upload failed with status ${putRes.status}`);
|
||||
} catch (error) {
|
||||
if (options?.signal?.aborted) throw error;
|
||||
putError = error;
|
||||
}
|
||||
|
||||
try {
|
||||
await uploadDocumentSourceViaProxy(source, options);
|
||||
} catch (proxyError) {
|
||||
const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error';
|
||||
const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error';
|
||||
throw new Error(`Failed to upload document ${source.name}: ${directMessage}; fallback failed: ${proxyMessage}`);
|
||||
}
|
||||
}
|
||||
|
||||
const registerRes = await fetch('/api/documents', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
documents: sources.map((source) => ({
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
type: source.type,
|
||||
size: source.size,
|
||||
lastModified: source.lastModified,
|
||||
})),
|
||||
}),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!registerRes.ok) {
|
||||
const data = (await registerRes.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to register uploaded documents');
|
||||
}
|
||||
|
||||
const data = (await registerRes.json()) as { stored: BaseDocument[] };
|
||||
return data.stored || [];
|
||||
}
|
||||
|
||||
export async function uploadDocuments(files: File[], options?: UploadOptions): Promise<BaseDocument[]> {
|
||||
if (files.length === 0) return [];
|
||||
|
||||
const sources: UploadSource[] = [];
|
||||
for (const file of files) {
|
||||
const bytes = await file.arrayBuffer();
|
||||
const id = await sha256HexFromArrayBuffer(bytes);
|
||||
const type = documentTypeForName(file.name);
|
||||
const name = file.name || `${id}.${type}`;
|
||||
const contentType = file.type || mimeTypeForDoc({ name, type });
|
||||
sources.push({
|
||||
id,
|
||||
name,
|
||||
type,
|
||||
size: file.size,
|
||||
lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(),
|
||||
contentType,
|
||||
body: file,
|
||||
});
|
||||
}
|
||||
|
||||
return uploadDocumentSources(sources, options);
|
||||
}
|
||||
|
||||
export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user' | 'unclaimed'; signal?: AbortSignal }): Promise<void> {
|
||||
const params = new URLSearchParams();
|
||||
if (options?.ids?.length) {
|
||||
|
|
@ -72,7 +202,7 @@ export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user'
|
|||
}
|
||||
|
||||
export async function downloadDocumentContent(id: string, options?: { signal?: AbortSignal }): Promise<ArrayBuffer> {
|
||||
const res = await fetch(`/api/documents/content?id=${encodeURIComponent(id)}`, { signal: options?.signal });
|
||||
const res = await fetch(`/api/documents/blob?id=${encodeURIComponent(id)}`, { signal: options?.signal });
|
||||
if (!res.ok) {
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
|
|
@ -95,7 +225,7 @@ export async function getDocumentContentSnippet(
|
|||
if (typeof options?.maxChars === 'number') params.set('maxChars', String(options.maxChars));
|
||||
if (typeof options?.maxBytes === 'number') params.set('maxBytes', String(options.maxBytes));
|
||||
|
||||
const res = await fetch(`/api/documents/content?${params.toString()}`, { signal: options?.signal });
|
||||
const res = await fetch(`/api/documents/blob?${params.toString()}`, { signal: options?.signal });
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || `Failed to load content snippet (status ${res.status})`);
|
||||
|
|
|
|||
287
src/lib/dexie.ts
287
src/lib/dexie.ts
|
|
@ -5,11 +5,12 @@ import {
|
|||
EPUBDocument,
|
||||
HTMLDocument,
|
||||
DocumentListState,
|
||||
SyncedDocument,
|
||||
BaseDocument,
|
||||
DocumentListDocument,
|
||||
} from '@/types/documents';
|
||||
import { sha256HexFromBytes, sha256HexFromString } from '@/lib/sha256';
|
||||
import { downloadDocumentContent, listDocuments, uploadDocumentSources, type UploadSource } from '@/lib/client-documents';
|
||||
import { cacheStoredDocumentFromBytes } from '@/lib/document-cache';
|
||||
|
||||
const DB_NAME = 'openreader-db';
|
||||
// Managed via Dexie (version bumped from the original manual IndexedDB)
|
||||
|
|
@ -657,15 +658,26 @@ export async function syncDocumentsToServer(
|
|||
const epubDocs = await getAllEpubDocuments();
|
||||
const htmlDocs = await getAllHtmlDocuments();
|
||||
|
||||
const documents: SyncedDocument[] = [];
|
||||
const uploads: Array<{ oldId: string; source: UploadSource }> = [];
|
||||
const totalDocs = pdfDocs.length + epubDocs.length + htmlDocs.length;
|
||||
let processedDocs = 0;
|
||||
|
||||
const textEncoder = new TextEncoder();
|
||||
|
||||
for (const doc of pdfDocs) {
|
||||
documents.push({
|
||||
...doc,
|
||||
type: 'pdf',
|
||||
data: Array.from(new Uint8Array(doc.data)),
|
||||
const bytes = new Uint8Array(doc.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: doc.id,
|
||||
source: {
|
||||
id,
|
||||
name: doc.name,
|
||||
type: 'pdf',
|
||||
size: bytes.byteLength,
|
||||
lastModified: doc.lastModified,
|
||||
contentType: 'application/pdf',
|
||||
body: bytes,
|
||||
},
|
||||
});
|
||||
processedDocs++;
|
||||
if (onProgress) {
|
||||
|
|
@ -674,10 +686,19 @@ export async function syncDocumentsToServer(
|
|||
}
|
||||
|
||||
for (const doc of epubDocs) {
|
||||
documents.push({
|
||||
...doc,
|
||||
type: 'epub',
|
||||
data: Array.from(new Uint8Array(doc.data)),
|
||||
const bytes = new Uint8Array(doc.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: doc.id,
|
||||
source: {
|
||||
id,
|
||||
name: doc.name,
|
||||
type: 'epub',
|
||||
size: bytes.byteLength,
|
||||
lastModified: doc.lastModified,
|
||||
contentType: 'application/epub+zip',
|
||||
body: bytes,
|
||||
},
|
||||
});
|
||||
processedDocs++;
|
||||
if (onProgress) {
|
||||
|
|
@ -685,13 +706,20 @@ export async function syncDocumentsToServer(
|
|||
}
|
||||
}
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
for (const doc of htmlDocs) {
|
||||
const encoded = encoder.encode(doc.data);
|
||||
documents.push({
|
||||
...doc,
|
||||
type: 'html',
|
||||
data: Array.from(encoded),
|
||||
const encoded = textEncoder.encode(doc.data);
|
||||
const id = await sha256HexFromBytes(encoded);
|
||||
uploads.push({
|
||||
oldId: doc.id,
|
||||
source: {
|
||||
id,
|
||||
name: doc.name,
|
||||
type: 'html',
|
||||
size: encoded.byteLength,
|
||||
lastModified: doc.lastModified,
|
||||
contentType: 'text/plain; charset=utf-8',
|
||||
body: encoded,
|
||||
},
|
||||
});
|
||||
processedDocs++;
|
||||
if (onProgress) {
|
||||
|
|
@ -703,25 +731,11 @@ export async function syncDocumentsToServer(
|
|||
onProgress(50, 'Uploading to server...');
|
||||
}
|
||||
|
||||
const response = await fetch('/api/documents', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ documents }),
|
||||
signal,
|
||||
});
|
||||
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to sync documents to server');
|
||||
}
|
||||
|
||||
const payload = (await response.json().catch(() => null)) as
|
||||
| { stored?: Array<{ oldId: string; id: string }> }
|
||||
| null;
|
||||
const stored = payload?.stored ?? [];
|
||||
for (const mapping of stored) {
|
||||
if (!mapping || typeof mapping.oldId !== 'string' || typeof mapping.id !== 'string') continue;
|
||||
if (mapping.oldId === mapping.id) continue;
|
||||
await applyDocumentIdMapping(mapping.oldId, mapping.id);
|
||||
for (const entry of uploads) {
|
||||
if (entry.oldId === entry.source.id) continue;
|
||||
await applyDocumentIdMapping(entry.oldId, entry.source.id);
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
|
|
@ -736,52 +750,77 @@ export async function syncSelectedDocumentsToServer(
|
|||
onProgress?: (progress: number, status?: string) => void,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ lastSync: number }> {
|
||||
// Re-use logic from syncDocumentsToServer but only for specific documents
|
||||
// Actually, syncDocumentsToServer fetches all docs from DB.
|
||||
// We need to fetch the *full content* of the selected docs from DB.
|
||||
|
||||
const fullDocs: SyncedDocument[] = [];
|
||||
let processed = 0;
|
||||
|
||||
for (const doc of documents) {
|
||||
if (doc.type === 'pdf') {
|
||||
const data = await getPdfDocument(doc.id);
|
||||
if (data) fullDocs.push({ ...data, type: 'pdf', data: Array.from(new Uint8Array(data.data)) });
|
||||
} else if (doc.type === 'epub') {
|
||||
const data = await getEpubDocument(doc.id);
|
||||
if (data) fullDocs.push({ ...data, type: 'epub', data: Array.from(new Uint8Array(data.data)) });
|
||||
} else {
|
||||
const data = await getHtmlDocument(doc.id);
|
||||
if (data) {
|
||||
const encoder = new TextEncoder();
|
||||
fullDocs.push({ ...data, type: 'html', data: Array.from(encoder.encode(data.data)) });
|
||||
}
|
||||
}
|
||||
processed++;
|
||||
if (onProgress) onProgress((processed / documents.length) * 50, `Preparing ${processed}/${documents.length}...`);
|
||||
const uploads: Array<{ oldId: string; source: UploadSource }> = [];
|
||||
const textEncoder = new TextEncoder();
|
||||
let processed = 0;
|
||||
|
||||
for (const doc of documents) {
|
||||
if (doc.type === 'pdf') {
|
||||
const data = await getPdfDocument(doc.id);
|
||||
if (data) {
|
||||
const bytes = new Uint8Array(data.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: data.id,
|
||||
source: {
|
||||
id,
|
||||
name: data.name,
|
||||
type: 'pdf',
|
||||
size: bytes.byteLength,
|
||||
lastModified: data.lastModified,
|
||||
contentType: 'application/pdf',
|
||||
body: bytes,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else if (doc.type === 'epub') {
|
||||
const data = await getEpubDocument(doc.id);
|
||||
if (data) {
|
||||
const bytes = new Uint8Array(data.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: data.id,
|
||||
source: {
|
||||
id,
|
||||
name: data.name,
|
||||
type: 'epub',
|
||||
size: bytes.byteLength,
|
||||
lastModified: data.lastModified,
|
||||
contentType: 'application/epub+zip',
|
||||
body: bytes,
|
||||
},
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const data = await getHtmlDocument(doc.id);
|
||||
if (data) {
|
||||
const bytes = textEncoder.encode(data.data);
|
||||
const id = await sha256HexFromBytes(bytes);
|
||||
uploads.push({
|
||||
oldId: data.id,
|
||||
source: {
|
||||
id,
|
||||
name: data.name,
|
||||
type: 'html',
|
||||
size: bytes.byteLength,
|
||||
lastModified: data.lastModified,
|
||||
contentType: 'text/plain; charset=utf-8',
|
||||
body: bytes,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (onProgress) onProgress(50, 'Uploading to server...');
|
||||
|
||||
const response = await fetch('/api/documents', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ documents: fullDocs }),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to sync documents to server');
|
||||
processed++;
|
||||
if (onProgress) onProgress((processed / documents.length) * 50, `Preparing ${processed}/${documents.length}...`);
|
||||
}
|
||||
|
||||
const payload = (await response.json().catch(() => null)) as
|
||||
| { stored?: Array<{ oldId: string; id: string }> }
|
||||
| null;
|
||||
const stored = payload?.stored ?? [];
|
||||
for (const mapping of stored) {
|
||||
if (!mapping || typeof mapping.oldId !== 'string' || typeof mapping.id !== 'string') continue;
|
||||
if (mapping.oldId === mapping.id) continue;
|
||||
await applyDocumentIdMapping(mapping.oldId, mapping.id);
|
||||
if (onProgress) onProgress(50, 'Uploading to server...');
|
||||
await uploadDocumentSources(uploads.map((entry) => entry.source), { signal });
|
||||
|
||||
for (const entry of uploads) {
|
||||
if (entry.oldId === entry.source.id) continue;
|
||||
await applyDocumentIdMapping(entry.oldId, entry.source.id);
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
|
|
@ -800,22 +839,8 @@ export async function loadDocumentsFromServer(
|
|||
onProgress(10, 'Starting download...');
|
||||
}
|
||||
|
||||
const response = await fetch('/api/documents', { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch documents from server');
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(30, 'Download complete');
|
||||
}
|
||||
|
||||
const { documents } = (await response.json()) as { documents: SyncedDocument[] };
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(40, 'Parsing documents...');
|
||||
}
|
||||
|
||||
await saveSyncedDocumentsLocally(documents, onProgress);
|
||||
const documents = await listDocuments({ signal });
|
||||
await downloadAndCacheServerDocuments(documents, onProgress, signal);
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(100, 'Load complete!');
|
||||
|
|
@ -832,26 +857,9 @@ export async function loadSelectedDocumentsFromServer(
|
|||
if (onProgress) {
|
||||
onProgress(10, 'Starting download...');
|
||||
}
|
||||
|
||||
// Use new filtered API
|
||||
const idsParam = selectedIds.join(',');
|
||||
const response = await fetch(`/api/documents?ids=${encodeURIComponent(idsParam)}`, { signal });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch documents from server');
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(30, 'Download complete');
|
||||
}
|
||||
|
||||
const { documents } = (await response.json()) as { documents: SyncedDocument[] };
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(40, 'Parsing documents...');
|
||||
}
|
||||
|
||||
await saveSyncedDocumentsLocally(documents, onProgress);
|
||||
const documents = await listDocuments({ ids: selectedIds, signal });
|
||||
await downloadAndCacheServerDocuments(documents, onProgress, signal);
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(100, 'Load complete!');
|
||||
|
|
@ -860,52 +868,23 @@ export async function loadSelectedDocumentsFromServer(
|
|||
return { lastSync: Date.now() };
|
||||
}
|
||||
|
||||
async function saveSyncedDocumentsLocally(documents: SyncedDocument[], onProgress?: (progress: number, status?: string) => void) {
|
||||
const textDecoder = new TextDecoder();
|
||||
async function downloadAndCacheServerDocuments(
|
||||
documents: BaseDocument[],
|
||||
onProgress?: (progress: number, status?: string) => void,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
if (onProgress) onProgress(30, 'List complete');
|
||||
if (documents.length === 0) {
|
||||
if (onProgress) onProgress(95, 'No documents to import');
|
||||
return;
|
||||
}
|
||||
|
||||
for (let i = 0; i < documents.length; i++) {
|
||||
const doc = documents[i];
|
||||
|
||||
if (doc.type === 'pdf') {
|
||||
const uint8Array = new Uint8Array(doc.data);
|
||||
const documentData: PDFDocument = {
|
||||
id: doc.id,
|
||||
type: 'pdf',
|
||||
name: doc.name,
|
||||
size: doc.size,
|
||||
lastModified: doc.lastModified,
|
||||
data: uint8Array.buffer,
|
||||
};
|
||||
await addPdfDocument(documentData);
|
||||
} else if (doc.type === 'epub') {
|
||||
const uint8Array = new Uint8Array(doc.data);
|
||||
const documentData: EPUBDocument = {
|
||||
id: doc.id,
|
||||
type: 'epub',
|
||||
name: doc.name,
|
||||
size: doc.size,
|
||||
lastModified: doc.lastModified,
|
||||
data: uint8Array.buffer,
|
||||
};
|
||||
await addEpubDocument(documentData);
|
||||
} else if (doc.type === 'html') {
|
||||
const uint8Array = new Uint8Array(doc.data);
|
||||
const decoded = textDecoder.decode(uint8Array);
|
||||
const documentData: HTMLDocument = {
|
||||
id: doc.id,
|
||||
type: 'html',
|
||||
name: doc.name,
|
||||
size: doc.size,
|
||||
lastModified: doc.lastModified,
|
||||
data: decoded,
|
||||
};
|
||||
await addHtmlDocument(documentData);
|
||||
} else {
|
||||
console.warn(`Unknown document type: ${doc.type}`);
|
||||
}
|
||||
|
||||
const bytes = await downloadDocumentContent(doc.id, { signal });
|
||||
await cacheStoredDocumentFromBytes(doc, bytes);
|
||||
if (onProgress) {
|
||||
onProgress(40 + ((i + 1) / documents.length) * 50, `Processing document ${i + 1}/${documents.length}...`);
|
||||
onProgress(30 + ((i + 1) / documents.length) * 65, `Downloading ${i + 1}/${documents.length}: ${doc.name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1007,5 +986,3 @@ export async function importDocumentsFromLibrary(
|
|||
onProgress(100, 'Library import complete!');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
/**
|
||||
* Centralized auth configuration check.
|
||||
* Auth is only enabled when BOTH BETTER_AUTH_SECRET and BETTER_AUTH_URL are set.
|
||||
* Auth is only enabled when BOTH AUTH_SECRET and BASE_URL are set.
|
||||
*/
|
||||
export function isAuthEnabled(): boolean {
|
||||
return !!(process.env.BETTER_AUTH_SECRET && process.env.BETTER_AUTH_URL);
|
||||
return !!(process.env.AUTH_SECRET && process.env.BASE_URL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -13,5 +13,5 @@ export function getAuthBaseUrl(): string | null {
|
|||
if (!isAuthEnabled()) {
|
||||
return null;
|
||||
}
|
||||
return process.env.BETTER_AUTH_URL || null;
|
||||
return process.env.BASE_URL || null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,34 @@ import * as schema from "@/db/schema"; // Import the dynamic schema
|
|||
|
||||
// ...
|
||||
|
||||
function tryGetOrigin(url: string | undefined): string | null {
|
||||
if (!url) return null;
|
||||
try {
|
||||
return new URL(url).origin;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function getTrustedOrigins(): string[] {
|
||||
const origins = new Set<string>();
|
||||
const baseOrigin = tryGetOrigin(process.env.BASE_URL);
|
||||
if (baseOrigin) origins.add(baseOrigin);
|
||||
|
||||
// Comma-separated list for local multi-host setups (e.g., localhost + LAN IP).
|
||||
const extra = (process.env.AUTH_TRUSTED_ORIGINS || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const candidate of extra) {
|
||||
const origin = tryGetOrigin(candidate);
|
||||
if (origin) origins.add(origin);
|
||||
}
|
||||
|
||||
return Array.from(origins);
|
||||
}
|
||||
|
||||
const createAuth = () => betterAuth({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
database: drizzleAdapter(db as any, {
|
||||
|
|
@ -25,8 +53,9 @@ const createAuth = () => betterAuth({
|
|||
verification: schema.verification,
|
||||
}
|
||||
}),
|
||||
secret: process.env.BETTER_AUTH_SECRET!,
|
||||
baseURL: process.env.BETTER_AUTH_URL || "http://localhost:3003",
|
||||
secret: process.env.AUTH_SECRET!,
|
||||
baseURL: process.env.BASE_URL!,
|
||||
trustedOrigins: getTrustedOrigins(),
|
||||
emailAndPassword: {
|
||||
enabled: true,
|
||||
requireEmailVerification: false, // Set to true in production
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { db } from '@/db';
|
|||
import { audiobookChapters, audiobooks, documents } from '@/db/schema';
|
||||
import { and, count, eq } from 'drizzle-orm';
|
||||
import { listStoredChapters } from '@/lib/server/audiobook';
|
||||
import { AUDIOBOOKS_V1_DIR, DOCUMENTS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore';
|
||||
import { AUDIOBOOKS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const MIGRATIONS_DIR = path.join(DOCSTORE_DIR, '.migrations');
|
||||
|
|
@ -38,33 +38,14 @@ async function writeState(): Promise<void> {
|
|||
await fs.writeFile(STATE_PATH, JSON.stringify(state, null, 2));
|
||||
}
|
||||
|
||||
async function hasFilesystemContent(mode: DbIndexState['mode']): Promise<{ documents: boolean; audiobooks: boolean }> {
|
||||
let documentsPresent = false;
|
||||
let audiobooksPresent = false;
|
||||
|
||||
// Documents are always stored under documents_v1.
|
||||
try {
|
||||
const entries = await fs.readdir(DOCUMENTS_V1_DIR);
|
||||
documentsPresent = entries.some((name) => /^[a-f0-9]{64}__.+$/i.test(name));
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
// Audiobooks differ by auth-mode layout.
|
||||
async function hasAudiobookFilesystemContent(mode: DbIndexState['mode']): Promise<boolean> {
|
||||
const audiobookDir = mode === 'auth' ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR;
|
||||
try {
|
||||
const entries = await fs.readdir(audiobookDir, { withFileTypes: true });
|
||||
audiobooksPresent = entries.some((e) => e.isDirectory() && e.name.endsWith('-audiobook'));
|
||||
return entries.some((entry) => entry.isDirectory() && entry.name.endsWith('-audiobook'));
|
||||
} catch {
|
||||
// ignore
|
||||
return false;
|
||||
}
|
||||
|
||||
return { documents: documentsPresent, audiobooks: audiobooksPresent };
|
||||
}
|
||||
|
||||
async function isDocumentIndexed(id: string): Promise<boolean> {
|
||||
const result = await db.select({ id: documents.id }).from(documents).where(eq(documents.id, id));
|
||||
return result.length > 0;
|
||||
}
|
||||
|
||||
async function isAudiobookIndexedForUser(id: string, userId: string): Promise<boolean> {
|
||||
|
|
@ -90,7 +71,6 @@ async function migrateLegacyAudiobooksToUnclaimed(): Promise<number> {
|
|||
|
||||
const sourceDir = path.join(AUDIOBOOKS_V1_DIR, entry.name);
|
||||
const targetDir = path.join(unclaimedDir, entry.name);
|
||||
|
||||
if (existsSync(targetDir)) continue;
|
||||
|
||||
try {
|
||||
|
|
@ -115,133 +95,83 @@ export async function getUnclaimedCounts(): Promise<{ documents: number; audiobo
|
|||
};
|
||||
}
|
||||
|
||||
async function scanAndPopulateDb(): Promise<{ documents: number; audiobooks: number }> {
|
||||
async function scanAndPopulateAudiobookDb(): Promise<void> {
|
||||
const authEnabled = isAuthEnabled();
|
||||
|
||||
console.log('Scanning file system for un-indexed content...');
|
||||
console.log('Scanning file system for un-indexed audiobooks...');
|
||||
|
||||
if (authEnabled) {
|
||||
await migrateLegacyAudiobooksToUnclaimed();
|
||||
}
|
||||
|
||||
if (existsSync(DOCUMENTS_V1_DIR)) {
|
||||
const files = await fs.readdir(DOCUMENTS_V1_DIR);
|
||||
for (const file of files) {
|
||||
const match = /^([a-f0-9]{64})__(.+)$/i.exec(file);
|
||||
if (!match) continue;
|
||||
|
||||
const id = match[1];
|
||||
const encodedName = match[2];
|
||||
if (await isDocumentIndexed(id)) continue;
|
||||
|
||||
let name: string;
|
||||
try {
|
||||
name = decodeURIComponent(encodedName);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const filePath = path.join(DOCUMENTS_V1_DIR, file);
|
||||
const stats = await fs.stat(filePath);
|
||||
const ext = path.extname(name).toLowerCase().replace('.', '');
|
||||
|
||||
await db.insert(documents).values({
|
||||
id,
|
||||
userId: UNCLAIMED_USER_ID,
|
||||
name,
|
||||
type: ext,
|
||||
size: stats.size,
|
||||
lastModified: Math.floor(stats.mtimeMs),
|
||||
filePath: file,
|
||||
});
|
||||
console.log(`Indexed document: ${name} (${id})`);
|
||||
}
|
||||
}
|
||||
|
||||
const audiobookScanDir = authEnabled ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR;
|
||||
if (existsSync(audiobookScanDir)) {
|
||||
const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
if (!existsSync(audiobookScanDir)) return;
|
||||
|
||||
const bookId = entry.name.replace('-audiobook', '');
|
||||
if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue;
|
||||
const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
|
||||
const dirPath = path.join(audiobookScanDir, entry.name);
|
||||
const bookId = entry.name.replace('-audiobook', '');
|
||||
if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue;
|
||||
|
||||
let title = 'Unknown Title';
|
||||
try {
|
||||
const metaPath = path.join(dirPath, 'audiobook.meta.json');
|
||||
const metaContent = await fs.readFile(metaPath, 'utf8');
|
||||
JSON.parse(metaContent);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
const dirPath = path.join(audiobookScanDir, entry.name);
|
||||
|
||||
const chapters = await listStoredChapters(dirPath);
|
||||
const totalDuration = chapters.reduce((acc, c) => acc + (c.durationSec || 0), 0);
|
||||
if (chapters.length > 0) title = chapters[0].title || title;
|
||||
let title = 'Unknown Title';
|
||||
try {
|
||||
const metaPath = path.join(dirPath, 'audiobook.meta.json');
|
||||
const metaContent = await fs.readFile(metaPath, 'utf8');
|
||||
JSON.parse(metaContent);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
await db.insert(audiobooks).values({
|
||||
id: bookId,
|
||||
const chapters = await listStoredChapters(dirPath);
|
||||
const totalDuration = chapters.reduce((acc, chapter) => acc + (chapter.durationSec || 0), 0);
|
||||
if (chapters.length > 0) title = chapters[0].title || title;
|
||||
|
||||
await db.insert(audiobooks).values({
|
||||
id: bookId,
|
||||
userId: UNCLAIMED_USER_ID,
|
||||
title,
|
||||
duration: totalDuration,
|
||||
});
|
||||
console.log(`Indexed audiobook: ${bookId}`);
|
||||
|
||||
for (const chapter of chapters) {
|
||||
await db.insert(audiobookChapters).values({
|
||||
id: `${bookId}-${chapter.index}`,
|
||||
bookId,
|
||||
userId: UNCLAIMED_USER_ID,
|
||||
title,
|
||||
duration: totalDuration,
|
||||
chapterIndex: chapter.index,
|
||||
title: chapter.title,
|
||||
duration: chapter.durationSec || 0,
|
||||
filePath: chapter.filePath,
|
||||
format: chapter.format,
|
||||
});
|
||||
console.log(`Indexed audiobook: ${bookId}`);
|
||||
|
||||
for (const chapter of chapters) {
|
||||
await db.insert(audiobookChapters).values({
|
||||
id: `${bookId}-${chapter.index}`,
|
||||
bookId,
|
||||
userId: UNCLAIMED_USER_ID,
|
||||
chapterIndex: chapter.index,
|
||||
title: chapter.title,
|
||||
duration: chapter.durationSec || 0,
|
||||
filePath: chapter.filePath,
|
||||
format: chapter.format,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return getUnclaimedCounts();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure DB has rows for existing filesystem content (documents/audiobooks).
|
||||
*
|
||||
* This is intentionally safe to call from routes:
|
||||
* - Runs at most once per process (memoryIndexed)
|
||||
* - Uses a persisted state file under docstore/.migrations to avoid rescanning every boot
|
||||
*/
|
||||
export async function ensureDbIndexed(): Promise<void> {
|
||||
export async function ensureAudiobooksIndexed(): Promise<void> {
|
||||
const mode: DbIndexState['mode'] = isAuthEnabled() ? 'auth' : 'noauth';
|
||||
if (memoryIndexedMode === mode) return;
|
||||
|
||||
inflight ??= (async () => {
|
||||
const hasState = existsSync(STATE_PATH) ? await readState() : null;
|
||||
if (hasState && hasState.mode === mode) {
|
||||
// If the DB was reset but the state file survived, don't get stuck "indexed" forever.
|
||||
// Only skip if:
|
||||
// - the DB already has rows for any on-disk content (per category), OR
|
||||
// - there is no content on disk to index (per category).
|
||||
//
|
||||
// This avoids a bad state where (for example) audiobooks are indexed, but documents exist on disk
|
||||
// and aren't counted/claimable because we early-exit based on "some DB rows exist".
|
||||
const [counts, fsHas] = await Promise.all([getUnclaimedCounts(), hasFilesystemContent(mode)]);
|
||||
const docsOk = counts.documents > 0 || !fsHas.documents;
|
||||
const audiobooksOk = counts.audiobooks > 0 || !fsHas.audiobooks;
|
||||
|
||||
if (docsOk && audiobooksOk) {
|
||||
const [counts, fsHasAudiobooks] = await Promise.all([
|
||||
getUnclaimedCounts(),
|
||||
hasAudiobookFilesystemContent(mode),
|
||||
]);
|
||||
const audiobooksOk = counts.audiobooks > 0 || !fsHasAudiobooks;
|
||||
if (audiobooksOk) {
|
||||
memoryIndexedMode = mode;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await fs.mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
await scanAndPopulateDb();
|
||||
await scanAndPopulateAudiobookDb();
|
||||
await writeState();
|
||||
memoryIndexedMode = mode;
|
||||
})().finally(() => {
|
||||
|
|
@ -250,3 +180,7 @@ export async function ensureDbIndexed(): Promise<void> {
|
|||
|
||||
await inflight;
|
||||
}
|
||||
|
||||
export async function ensureDbIndexed(): Promise<void> {
|
||||
await ensureAudiobooksIndexed();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ export const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1');
|
|||
export const AUDIOBOOKS_USERS_DIR = path.join(DOCSTORE_DIR, 'audiobooks_users_v1');
|
||||
|
||||
export const UNCLAIMED_USER_ID = 'unclaimed';
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
/**
|
||||
* Get the audiobook directory for a specific user when auth is enabled.
|
||||
|
|
@ -34,15 +35,29 @@ export function getUnclaimedAudiobookDir(): string {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the full path to a specific audiobook directory.
|
||||
* - When auth is disabled: docstore/audiobooks_v1/{bookId}-audiobook
|
||||
* - When auth is enabled: docstore/audiobooks_users_v1/{userId}/{bookId}-audiobook
|
||||
* Resolve the base audiobooks directory for request context, including optional test namespace.
|
||||
* - When auth is disabled or userId is absent: docstore/audiobooks_v1[/<namespace>]
|
||||
* - When auth is enabled: docstore/audiobooks_users_v1/{userId}[/<namespace>]
|
||||
*/
|
||||
export function getAudiobookPath(bookId: string, userId: string | null, authEnabled: boolean): string {
|
||||
if (!authEnabled || !userId) {
|
||||
return path.join(AUDIOBOOKS_V1_DIR, `${bookId}-audiobook`);
|
||||
}
|
||||
return path.join(getUserAudiobookDir(userId), `${bookId}-audiobook`);
|
||||
export function getAudiobooksRootDir({
|
||||
userId,
|
||||
authEnabled,
|
||||
namespace,
|
||||
}: {
|
||||
userId: string | null;
|
||||
authEnabled: boolean;
|
||||
namespace: string | null;
|
||||
}): string {
|
||||
const baseDir = !authEnabled || !userId ? AUDIOBOOKS_V1_DIR : getUserAudiobookDir(userId);
|
||||
if (!namespace) return baseDir;
|
||||
|
||||
const safe = namespace.trim();
|
||||
if (!safe || safe === '.' || safe === '..' || safe.includes('..')) return baseDir;
|
||||
if (!SAFE_NAMESPACE_REGEX.test(safe)) return baseDir;
|
||||
|
||||
const resolved = path.resolve(baseDir, safe);
|
||||
if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -210,15 +225,6 @@ export function getMigratedDocumentFileName(id: string, name: string): string {
|
|||
return targetFileName;
|
||||
}
|
||||
|
||||
export type FSDocument = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
filePath: string;
|
||||
};
|
||||
|
||||
export async function ensureDocumentsV1Ready(): Promise<boolean> {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
await mkdir(DOCUMENTS_V1_DIR, { recursive: true });
|
||||
|
|
|
|||
222
src/lib/server/documents-blobstore.ts
Normal file
222
src/lib/server/documents-blobstore.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
import {
|
||||
DeleteObjectCommand,
|
||||
DeleteObjectsCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { getS3Client, getS3Config } from '@/lib/server/s3';
|
||||
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
function sanitizeNamespace(namespace: string | null): string | null {
|
||||
if (!namespace) return null;
|
||||
if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null;
|
||||
return namespace;
|
||||
}
|
||||
|
||||
function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream {
|
||||
return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function';
|
||||
}
|
||||
|
||||
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
if (Buffer.isBuffer(chunk)) {
|
||||
chunks.push(chunk);
|
||||
} else if (typeof chunk === 'string') {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
} else {
|
||||
chunks.push(Buffer.from(chunk as Uint8Array));
|
||||
}
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async function bodyToBuffer(body: unknown): Promise<Buffer> {
|
||||
if (!body) return Buffer.alloc(0);
|
||||
|
||||
if (body instanceof Uint8Array) return Buffer.from(body);
|
||||
if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
||||
if (body instanceof ArrayBuffer) return Buffer.from(body);
|
||||
|
||||
if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) {
|
||||
const maybe = body as { transformToByteArray?: () => Promise<Uint8Array> };
|
||||
if (typeof maybe.transformToByteArray === 'function') {
|
||||
return Buffer.from(await maybe.transformToByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
if (isNodeReadableStream(body)) {
|
||||
return streamToBuffer(body);
|
||||
}
|
||||
|
||||
throw new Error('Unsupported S3 response body type');
|
||||
}
|
||||
|
||||
export function isValidDocumentId(id: string): boolean {
|
||||
return DOCUMENT_ID_REGEX.test(id);
|
||||
}
|
||||
|
||||
export function documentKey(id: string, namespace: string | null): string {
|
||||
if (!isValidDocumentId(id)) {
|
||||
throw new Error(`Invalid document id: ${id}`);
|
||||
}
|
||||
|
||||
const cfg = getS3Config();
|
||||
const ns = sanitizeNamespace(namespace);
|
||||
const nsSegment = ns ? `ns/${ns}/` : '';
|
||||
return `${cfg.prefix}/documents_v1/${nsSegment}${id}`;
|
||||
}
|
||||
|
||||
export async function presignPut(
|
||||
id: string,
|
||||
contentType: string,
|
||||
namespace: string | null,
|
||||
): Promise<{ url: string; headers: Record<string, string> }> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = documentKey(id, namespace);
|
||||
const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream';
|
||||
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
ContentType: normalizedType,
|
||||
IfNoneMatch: '*',
|
||||
});
|
||||
const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 });
|
||||
|
||||
return {
|
||||
url,
|
||||
headers: {
|
||||
'Content-Type': normalizedType,
|
||||
'If-None-Match': '*',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function headDocumentBlob(
|
||||
id: string,
|
||||
namespace: string | null,
|
||||
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = documentKey(id, namespace);
|
||||
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
return {
|
||||
contentLength: Number(res.ContentLength ?? 0),
|
||||
contentType: res.ContentType ?? null,
|
||||
eTag: res.ETag ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getDocumentRange(
|
||||
id: string,
|
||||
start: number,
|
||||
endInclusive: number,
|
||||
namespace: string | null,
|
||||
): Promise<Buffer> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = documentKey(id, namespace);
|
||||
const res = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
Range: `bytes=${Math.max(0, start)}-${Math.max(0, endInclusive)}`,
|
||||
}),
|
||||
);
|
||||
return bodyToBuffer(res.Body);
|
||||
}
|
||||
|
||||
export async function getDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = documentKey(id, namespace);
|
||||
const res = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
}),
|
||||
);
|
||||
return bodyToBuffer(res.Body);
|
||||
}
|
||||
|
||||
export async function putDocumentBlob(
|
||||
id: string,
|
||||
body: Buffer,
|
||||
contentType: string,
|
||||
namespace: string | null,
|
||||
): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = documentKey(id, namespace);
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
IfNoneMatch: '*',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function deleteDocumentBlob(id: string, namespace: string | null): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = documentKey(id, namespace);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
}
|
||||
|
||||
export function isMissingBlobError(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
|
||||
if (maybe.$metadata?.httpStatusCode === 404) return true;
|
||||
if (maybe.name === 'NotFound' || maybe.name === 'NoSuchKey') return true;
|
||||
if (maybe.Code === 'NotFound' || maybe.Code === 'NoSuchKey') return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function deleteDocumentPrefix(prefix: string): Promise<number> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const cleanedPrefix = prefix.replace(/^\/+/, '');
|
||||
let deleted = 0;
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const listRes = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: cfg.bucket,
|
||||
Prefix: cleanedPrefix,
|
||||
ContinuationToken: continuationToken,
|
||||
}),
|
||||
);
|
||||
|
||||
const keys = (listRes.Contents ?? [])
|
||||
.map((item) => item.Key)
|
||||
.filter((value): value is string => typeof value === 'string' && value.length > 0);
|
||||
|
||||
if (keys.length > 0) {
|
||||
const deleteRes = await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Delete: {
|
||||
Objects: keys.map((Key) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
deleted += deleteRes.Deleted?.length ?? 0;
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
|
@ -1,11 +1,6 @@
|
|||
import path from 'path';
|
||||
import { utimes } from 'fs/promises';
|
||||
import type { DocumentType } from '@/types/documents';
|
||||
|
||||
export function isEnoent(error: unknown): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && (error as { code?: unknown }).code === 'ENOENT';
|
||||
}
|
||||
|
||||
export function safeDocumentName(rawName: string, fallback: string): string {
|
||||
const baseName = path.basename(rawName || fallback);
|
||||
return baseName.replaceAll('\u0000', '').slice(0, 240) || fallback;
|
||||
|
|
@ -18,16 +13,3 @@ export function toDocumentTypeFromName(name: string): DocumentType {
|
|||
if (ext === '.docx') return 'docx';
|
||||
return 'html';
|
||||
}
|
||||
|
||||
export async function trySetFileMtime(filePath: string, lastModifiedMs: number): Promise<void> {
|
||||
if (!Number.isFinite(lastModifiedMs)) return;
|
||||
const mtime = new Date(lastModifiedMs);
|
||||
if (Number.isNaN(mtime.getTime())) return;
|
||||
|
||||
try {
|
||||
await utimes(filePath, mtime, mtime);
|
||||
} catch (error) {
|
||||
console.warn('Failed to set document mtime:', filePath, error);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -111,6 +111,17 @@ export class RateLimiter {
|
|||
return this.isPostgres() ? new Date() : Date.now();
|
||||
}
|
||||
|
||||
// Use a transaction only when running with Postgres.
|
||||
// better-sqlite3 transactions require sync callbacks and cannot be awaited.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
private async runMutation<T>(fn: (conn: any) => Promise<T>): Promise<T> {
|
||||
if (this.isPostgres()) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return safeDb().transaction(async (tx: any) => fn(tx));
|
||||
}
|
||||
return fn(safeDb());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a user can use TTS and increment their char count if allowed
|
||||
*/
|
||||
|
|
@ -147,16 +158,10 @@ export class RateLimiter {
|
|||
|
||||
try {
|
||||
const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue;
|
||||
|
||||
// Use a DB transaction to avoid partial increments across buckets and to avoid
|
||||
// non-transactional "rollback" logic that can corrupt counts under concurrency.
|
||||
// Note: We intentionally allow a request to push a bucket over its limit; we only
|
||||
// block when the bucket was already exhausted before this request starts updating.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return await safeDb().transaction(async (tx: any) => {
|
||||
return await this.runMutation(async (conn) => {
|
||||
// Ensure records exist for each bucket
|
||||
for (const bucket of buckets) {
|
||||
await tx.insert(userTtsChars)
|
||||
await conn.insert(userTtsChars)
|
||||
.values({
|
||||
userId: bucket.key,
|
||||
date: dateValue,
|
||||
|
|
@ -172,7 +177,7 @@ export class RateLimiter {
|
|||
// that start after the bucket is already exhausted, while still allowing a
|
||||
// request to push the count over the limit.
|
||||
for (const bucket of buckets) {
|
||||
const updateResult = await tx.update(userTtsChars)
|
||||
const updateResult = await conn.update(userTtsChars)
|
||||
.set({
|
||||
charCount: sql`${userTtsChars.charCount} + ${charCount}`,
|
||||
updatedAt,
|
||||
|
|
@ -191,7 +196,7 @@ export class RateLimiter {
|
|||
// Fetch current counts
|
||||
const bucketResults: Array<{ currentCount: number; limit: number }> = [];
|
||||
for (const bucket of buckets) {
|
||||
const result = await tx.select({ currentCount: userTtsChars.charCount })
|
||||
const result = await conn.select({ currentCount: userTtsChars.charCount })
|
||||
.from(userTtsChars)
|
||||
.where(and(eq(userTtsChars.userId, bucket.key), eq(userTtsChars.date, dateValue)));
|
||||
|
||||
|
|
|
|||
82
src/lib/server/s3.ts
Normal file
82
src/lib/server/s3.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
type S3Config = {
|
||||
bucket: string;
|
||||
region: string;
|
||||
endpoint?: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
forcePathStyle: boolean;
|
||||
prefix: string;
|
||||
};
|
||||
|
||||
let cachedClient: S3Client | null = null;
|
||||
let cachedConfig: S3Config | null = null;
|
||||
|
||||
function parseBool(value: string | undefined): boolean {
|
||||
if (!value) return false;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
|
||||
}
|
||||
|
||||
function normalizePrefix(prefix: string | undefined): string {
|
||||
const base = (prefix || 'openreader').trim();
|
||||
if (!base) return 'openreader';
|
||||
return base.replace(/^\/+|\/+$/g, '');
|
||||
}
|
||||
|
||||
function loadS3ConfigFromEnv(): S3Config | null {
|
||||
const bucket = process.env.S3_BUCKET?.trim();
|
||||
const region = process.env.S3_REGION?.trim();
|
||||
const accessKeyId = process.env.S3_ACCESS_KEY_ID?.trim();
|
||||
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY?.trim();
|
||||
const endpoint = process.env.S3_ENDPOINT?.trim();
|
||||
|
||||
if (!bucket || !region || !accessKeyId || !secretAccessKey) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
bucket,
|
||||
region,
|
||||
endpoint: endpoint || undefined,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
forcePathStyle: parseBool(process.env.S3_FORCE_PATH_STYLE),
|
||||
prefix: normalizePrefix(process.env.S3_PREFIX),
|
||||
};
|
||||
}
|
||||
|
||||
export function isS3Configured(): boolean {
|
||||
return loadS3ConfigFromEnv() !== null;
|
||||
}
|
||||
|
||||
export function getS3Config(): S3Config {
|
||||
if (cachedConfig) return cachedConfig;
|
||||
const config = loadS3ConfigFromEnv();
|
||||
if (!config) {
|
||||
throw new Error(
|
||||
'S3 is not configured. Required env vars: S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.',
|
||||
);
|
||||
}
|
||||
cachedConfig = config;
|
||||
return config;
|
||||
}
|
||||
|
||||
export function getS3Client(): S3Client {
|
||||
if (cachedClient) return cachedClient;
|
||||
const config = getS3Config();
|
||||
|
||||
cachedClient = new S3Client({
|
||||
region: config.region,
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
});
|
||||
|
||||
return cachedClient;
|
||||
}
|
||||
|
||||
|
|
@ -251,7 +251,8 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async
|
|||
await startGeneration(page);
|
||||
|
||||
// Progress card should appear with a Cancel button while chapters are being generated
|
||||
const cancelButton = page.getByRole('button', { name: 'Cancel' });
|
||||
const generationCard = page.locator('div', { hasText: 'Generating Audiobook' }).first();
|
||||
const cancelButton = generationCard.getByRole('button', { name: 'Cancel' });
|
||||
await expect(cancelButton).toBeVisible({ timeout: 60_000 });
|
||||
|
||||
await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 });
|
||||
|
|
@ -266,8 +267,10 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async
|
|||
// Now cancel the in-flight generation
|
||||
await cancelButton.click();
|
||||
|
||||
// After cancellation, the inline progress card's Cancel button should be gone
|
||||
await expect(page.getByRole('button', { name: 'Cancel' })).toHaveCount(0);
|
||||
// Cancellation is asynchronous: wait for generation to settle before asserting
|
||||
// that the inline progress card has disappeared.
|
||||
await expect(page.getByRole('button', { name: 'Resume' })).toBeVisible({ timeout: 30_000 });
|
||||
await expect(generationCard).toHaveCount(0, { timeout: 30_000 });
|
||||
|
||||
// After cancellation, wait for the chapter count to stabilize. In-flight TTS
|
||||
// requests may still complete after we click cancel, so we poll until the
|
||||
|
|
|
|||
|
|
@ -1,29 +1,11 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
async function listWorkerNamespaces(documentsRoot: string): Promise<string[]> {
|
||||
let entries: Array<{ name: string; isDirectory: () => boolean }> = [];
|
||||
try {
|
||||
entries = await fs.readdir(documentsRoot, { withFileTypes: true });
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
return entries
|
||||
.filter((d) => d.isDirectory())
|
||||
.map((d) => d.name)
|
||||
.filter((name) => /^(chromium|firefox|webkit)-worker\d+$/.test(name));
|
||||
}
|
||||
import { deleteDocumentPrefix } from '../src/lib/server/documents-blobstore';
|
||||
import { getS3Config, isS3Configured } from '../src/lib/server/s3';
|
||||
|
||||
export default async function globalTeardown(): Promise<void> {
|
||||
const documentsRoot = path.join(process.cwd(), 'docstore', 'documents_v1');
|
||||
const namespaces = await listWorkerNamespaces(documentsRoot);
|
||||
if (!namespaces.length) return;
|
||||
if (!isS3Configured()) return;
|
||||
|
||||
await Promise.all(
|
||||
namespaces.map(async (ns) => {
|
||||
const dir = path.join(documentsRoot, ns);
|
||||
await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
|
||||
}),
|
||||
);
|
||||
const config = getS3Config();
|
||||
const nsRootPrefix = `${config.prefix}/documents_v1/ns/`;
|
||||
await deleteDocumentPrefix(nsRootPrefix);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ import { transferUserDocuments } from '../../src/lib/server/claim-data';
|
|||
|
||||
test.describe('transferUserDocuments', () => {
|
||||
test('moves document rows to new user without PK conflicts', async () => {
|
||||
process.env.BETTER_AUTH_URL = 'http://localhost:3003';
|
||||
process.env.BETTER_AUTH_SECRET = 'test-secret';
|
||||
process.env.BASE_URL = 'http://localhost:3003';
|
||||
process.env.AUTH_SECRET = 'test-secret';
|
||||
|
||||
const sqlite = new Database(':memory:');
|
||||
sqlite.exec(`
|
||||
|
|
@ -72,4 +72,3 @@ test.describe('transferUserDocuments', () => {
|
|||
expect(ids).toEqual(['doc-a', 'doc-b']);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue