diff --git a/.env.example b/.env.example index f0aa30c..8db86f8 100644 --- a/.env.example +++ b/.env.example @@ -9,14 +9,30 @@ API_KEY=api_key_optional # Path to your local whisper.cpp CLI binary for STT timestamp generation WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli +# Auth (recommended for contributors and public instances) +# (Optional) Auth is only enabled when **both** AUTH_SECRET and BASE_URL are set +BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network) +AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32` +AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional trusted origins (BASE_URL is always trusted) +# (Optional) Sign in w/ GitHub Configuration +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= + # Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables. # Defaults to SQLite at docstore/sqlite3.db when not set. POSTGRES_URL= -# Auth (recommended for contributors and public instances) -# (Optional) Auth is only enabled when **both** BETTER_AUTH_SECRET and BETTER_AUTH_URL are set -BETTER_AUTH_URL=http://localhost:3003 # Your externally facing URL for this app -BETTER_AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32` -# (Optional) Sign in w/ GitHub Configuration -GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= +# Embedded SeaweedFS weed mini config +USE_EMBEDDED_WEED_MINI= +WEED_MINI_DIR= +WEED_MINI_WAIT_SEC= +# S3 storage config (use with embedded weed mini or external S3-compatible storage) +# For embedded weed mini, set explicit keys if you want stable credentials across restarts. +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_BUCKET= +S3_REGION= +# If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host. +S3_ENDPOINT= +S3_FORCE_PATH_STYLE= +S3_PREFIX= diff --git a/.gitignore b/.gitignore index 8cbfc60..38f5d2c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # dependencies /node_modules +/.pnpm-store /.pnp .pnp.* .yarn/* @@ -52,4 +53,4 @@ node_modules/ /playwright/.cache/ # vscode -.vscode \ No newline at end of file +.vscode diff --git a/Dockerfile b/Dockerfile index 255275d..6a37bb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,10 @@ RUN git clone --depth 1 https://github.com/ggml-org/whisper.cpp.git && \ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF $( [ "$TARGETARCH" = "arm64" ] && echo "-DGGML_CPU_ARM_ARCH=armv8-a" || true ) && \ cmake --build build -j +# Stage 1b: extract seaweedfs weed binary (for optional embedded weed mini) +FROM chrislusf/seaweedfs:latest AS seaweedfs-builder +RUN cp "$(command -v weed)" /tmp/weed + # Stage 2: build the Next.js app FROM node:lts-alpine AS app-builder @@ -42,7 +46,7 @@ FROM node:lts-alpine AS runner # Add runtime OS dependencies: # - ffmpeg: required for audiobook export and word-by-word alignment (/api/whisper) # - libreoffice-writer: required for DOCX → PDF conversion -RUN apk add --no-cache ffmpeg libreoffice-writer +RUN apk add --no-cache ca-certificates ffmpeg libreoffice-writer # Install pnpm globally for running the app RUN npm install -g pnpm @@ -56,6 +60,9 @@ COPY --from=app-builder /app ./ # Copy the compiled whisper.cpp build output into the runtime image # (includes whisper-cli and its shared libraries, e.g. libwhisper.so, libggml.so) COPY --from=whisper-builder /opt/whisper.cpp/build /opt/whisper.cpp/build +# Copy seaweedfs weed binary for optional embedded local S3. +COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed +RUN chmod +x /usr/local/bin/weed # Point the app at the compiled whisper-cli binary and ensure its libs are discoverable ENV WHISPER_CPP_BIN=/opt/whisper.cpp/build/bin/whisper-cli @@ -65,4 +72,5 @@ ENV LD_LIBRARY_PATH=/opt/whisper.cpp/build EXPOSE 3003 # Start the application -CMD ["pnpm", "start"] +ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"] +CMD ["pnpm", "start:raw"] diff --git a/README.md b/README.md index 040c20c..1ba04ba 100644 --- a/README.md +++ b/README.md @@ -42,63 +42,110 @@ OpenReader WebUI is an open source text to speech document reader web app built ### 1. 🐳 Start the Docker container - Minimal (no persistence, auth disabled unless you set auth env vars): + Minimal (auth disabled, embedded storage is ephemeral, no library import): ```bash docker run --name openreader-webui \ --restart unless-stopped \ -p 3003:3003 \ + -p 8333:8333 \ ghcr.io/richardr1126/openreader-webui:latest ``` - Fully featured (persistent storage + server library import + KokoroFastAPI in Docker + optional auth): + Fully featured (persistent storage, embedded SeaweedFS `weed mini` for documents, optional auth): ```bash docker run --name openreader-webui \ --restart unless-stopped \ -p 3003:3003 \ + -p 8333:8333 \ -v openreader_docstore:/app/docstore \ -v /path/to/your/library:/app/docstore/library:ro \ -e API_BASE=http://host.docker.internal:8880/v1 \ -e API_KEY=none \ - -e BETTER_AUTH_URL=http://localhost:3003 \ - -e BETTER_AUTH_SECRET= \ + -e BASE_URL=http://localhost:3003 \ + -e AUTH_SECRET= \ ghcr.io/richardr1126/openreader-webui:latest ``` You can remove the `/app/docstore/library` mount if you don't need server library import. - You can remove both `BETTER_AUTH_*` env vars to keep auth disabled. + You can remove either `BASE_URL` or `AUTH_SECRET` to keep auth disabled. - > **Notes:** - > - > - `API_BASE` should point to your TTS API server's base URL (if running Kokoro-FastAPI locally in Docker, use `http://host.docker.internal:8880/v1`). - > - `BETTER_AUTH_URL` should be your externally-facing URL for this app (for example `https://reader.example.com` or `http://localhost:3003`). - > - To enable auth, set **both** `BETTER_AUTH_URL` and `BETTER_AUTH_SECRET` generated with `openssl rand -base64 32`. - > - OpenReader always uses a backend DB for server-side metadata. By default it uses SQLite at `/app/docstore/sqlite3.db` (persisted when `/app/docstore` is mounted). - > - If you set `POSTGRES_URL`, the container uses Postgres instead of SQLite and will run migrations against it on startup. Ensure the database is accessible. + Quick notes: + - `API_BASE` should point to your TTS API server's base URL (for local Docker Kokoro, use `http://host.docker.internal:8880/v1`). + - Expose `-p 8333:8333` for direct browser access to embedded SeaweedFS presigned URLs. + - If port `8333` is not exposed, uploads still work via `/api/documents/blob/upload/fallback`. + - To enable auth, set both `BASE_URL` and `AUTH_SECRET`.
- Docker environment variables (Click to expand) + Docker networking and blob behavior (Click to expand) + + **Ports** + + - `3003` serves the OpenReader app and API routes. + - `8333` serves embedded SeaweedFS S3 for browser direct blob access. + + **Upload behavior** + + - Primary upload path: browser uploads to presigned URL from `/api/documents/blob/upload/presign`. + - Fallback upload path: `/api/documents/blob/upload/fallback` if direct upload fails. + - Content serving path: `/api/documents/blob` (server-served bytes/snippets). + +
+ +
+ Which Docker setup should I use? (Click to expand) + + | Goal | Recommended setup | + | --- | --- | + | Fast local setup, no persistence | Minimal `docker run` example | + | Persistent docs/audiobooks and optional auth | Fully featured `docker run` example with `/app/docstore` volume | + | Browser direct blob uploads/downloads | Publish both `3003` and `8333` | + | Private blob endpoint (no `8333` published) | Keep using app APIs; uploads use fallback proxy when direct presigned upload is unreachable | + +
+ +
+ Common Docker environment variables (Click to expand) | Variable | Purpose | Example / Notes | | --- | --- | --- | | `API_BASE` | Default TTS API base URL (server-side) | `http://host.docker.internal:8880/v1` | | `API_KEY` | Default TTS API key | `none` or your provider key | - | `BETTER_AUTH_URL` | Enables auth when set with `BETTER_AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` | - | `BETTER_AUTH_SECRET` | Enables auth when set with `BETTER_AUTH_URL` | Generate with `openssl rand -base64 32` | + | `BASE_URL` | Enables auth when set with `AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` | + | `AUTH_SECRET` | Enables auth when set with `BASE_URL` | Generate with `openssl rand -base64 32` | + | `AUTH_TRUSTED_ORIGINS` | Extra allowed auth request origins | Comma-separated list; leave empty to trust only `BASE_URL` | | `POSTGRES_URL` | Use Postgres for server DB (metadata + auth tables) instead of SQLite | If set, startup migrations target Postgres | | `GITHUB_CLIENT_ID` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_SECRET` | | `GITHUB_CLIENT_SECRET` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_ID` |
+
+ Blob and embedded storage environment variables (Click to expand) + + | Variable | Purpose | Example / Notes | + | --- | --- | --- | + | `USE_EMBEDDED_WEED_MINI` | Start SeaweedFS before app startup | default: `true` when unset in shared entrypoint | + | `WEED_MINI_DIR` | Data directory for embedded `weed mini` | default: `docstore/seaweedfs` | + | `WEED_MINI_WAIT_SEC` | Startup wait timeout for embedded `weed mini` | default: `20` | + | `S3_BUCKET` | S3 bucket used for document blobs | default: `openreader-documents` in embedded mode | + | `S3_REGION` | S3 region used by AWS SDK | default: `us-east-1` in embedded mode | + | `S3_ENDPOINT` | Custom endpoint for S3-compatible providers | default: `http://:8333` when `BASE_URL` is set, otherwise detected LAN host | + | `S3_ACCESS_KEY_ID` | S3 access key | auto-generated in embedded mode if unset | + | `S3_SECRET_ACCESS_KEY` | S3 secret key | auto-generated in embedded mode if unset | + | `S3_FORCE_PATH_STYLE` | Force path-style addressing | default: `true` in embedded mode | + | `S3_PREFIX` | Prefix for stored object keys | default: `openreader` | + +
+
Docker volume mounts (Click to expand) | Mount | Type | Recommended | Purpose | Example | | --- | --- | --- | --- | --- | - | `/app/docstore` | Docker named volume | Yes | Persists server-side storage (documents, audiobook exports, settings, SQLite DB if used) | `-v openreader_docstore:/app/docstore` | - | `/app/docstore/library` | Bind mount (host folder) | Optional + `:ro` | Exposes an existing folder of documents for **Server Library Import** | `-v /path/to/your/library:/app/docstore/library:ro` | + | `/app/docstore` | Docker named volume | Yes (if you want persistence) | Persists embedded SeaweedFS blob data (`docstore/seaweedfs`), SQLite metadata DB (`docstore/sqlite3.db` when `POSTGRES_URL` is unset), audiobook export artifacts, and migration/runtime files | `-v openreader_docstore:/app/docstore` | + | `/app/docstore/library` | Bind mount (host folder) | Optional + `:ro` | Read-only source directory for **Server Library Import**; files are imported/copied into browser storage, not modified in place | `-v /path/to/your/library:/app/docstore/library:ro` | To import from the mounted library: **Settings → Documents → Server Library Import** @@ -117,8 +164,9 @@ OpenReader WebUI is an open source text to speech document reader web app built ### 3. ⬆️ Updating Docker Image ```bash -docker stop openreader-webui && \ -docker rm openreader-webui && \ +docker stop openreader-webui || true && \ +docker rm openreader-webui || true && \ +docker image rm ghcr.io/richardr1126/openreader-webui:latest || true && \ docker pull ghcr.io/richardr1126/openreader-webui:latest ``` @@ -194,7 +242,15 @@ docker run -d \ ``` - A TTS API server (Kokoro-FastAPI, Orpheus-FastAPI, Deepinfra, OpenAI, etc.) running and accessible -Optionally required for different features: +- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) `weed` binary (required) + + ```bash + brew install seaweedfs + ``` + + > **Note:** Verify install with `weed version`. + +#### Optionally required for different features: - [FFmpeg](https://ffmpeg.org) (required for audiobook m4b creation only) ```bash @@ -248,16 +304,52 @@ Optionally required for different features: Auth is recommended for contributors and is enabled when **both** values are set: - - Set `BETTER_AUTH_URL` to your local URL (default: `http://localhost:3003`) - - Generate a `BETTER_AUTH_SECRET` and paste it into `.env`: + - Set `BASE_URL` to your local URL (default: `http://localhost:3003`) + - Generate a `AUTH_SECRET` and paste it into `.env`: ```bash openssl rand -base64 32 ``` + - (Optional) If you use both localhost and LAN URL, set `AUTH_TRUSTED_ORIGINS` (leave empty to trust only `BASE_URL`): - > Note: To disable auth, remove either `BETTER_AUTH_URL` or `BETTER_AUTH_SECRET`. + ```bash + AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://192.168.0.116:3003 + ``` + + The embedded weed mini object store is enabled by default for local development, and started through the shared entrypoint. The ACCESS_KEY_ID and SECRET_ACCESS_KEY are auto-generated if unset, but for stable credentials across restarts, you can generate and set them in `.env`: + - (Optional) Generate `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` and paste them into `.env` for stable credentials: + + ```bash + S3_ACCESS_KEY_ID=<`openssl rand -hex 16`> + S3_SECRET_ACCESS_KEY=<`openssl rand -hex 32`> + ``` + - (Optional) Connect to external S3-compatible storage instead of embedded `weed mini`: + + ```bash + USE_EMBEDDED_WEED_MINI=false + + # Required + S3_BUCKET=your-bucket + S3_REGION=us-east-1 + S3_ACCESS_KEY_ID=your-access-key + S3_SECRET_ACCESS_KEY=your-secret-key + + # Optional / provider-specific + S3_ENDPOINT= + S3_FORCE_PATH_STYLE= + S3_PREFIX=openreader + ``` + + Notes: + - For AWS S3: usually leave `S3_ENDPOINT` empty and `S3_FORCE_PATH_STYLE` empty/false. + - For MinIO/SeaweedFS/R2/B2 S3: set `S3_ENDPOINT` to your endpoint URL and set `S3_FORCE_PATH_STYLE=true` when required by that provider. + + > Notes: + > - The base URL for the TTS API should be accessible and relative to the Next.js server > - > Note: The base URL for the TTS API should be accessible and relative to the Next.js server + > - To disable auth, remove either `BASE_URL` or `AUTH_SECRET`. + > + > - If S3 credentials are unset, they are auto-generated per startup. 4. Run DB migrations: @@ -316,6 +408,10 @@ This project would not be possible without standing on the shoulders of these gi - [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M) model - [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) +- [Better Auth](https://www.better-auth.com/) +- [SQLite](https://www.sqlite.org/) +- [PostgreSQL](https://www.postgresql.org/) +- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`) - [whisper.cpp](https://github.com/ggerganov/whisper.cpp) - [ffmpeg](https://ffmpeg.org) - [react-pdf](https://github.com/wojtekmaj/react-pdf) npm package @@ -328,31 +424,26 @@ This project would not be possible without standing on the shoulders of these gi ## Stack -- **Framework:** Next.js (React) -- **Containerization:** Docker -- **Storage:** - - [Dexie.js](https://dexie.org/) IndexedDB wrapper for client-side storage -- **PDF:** - - [react-pdf](https://github.com/wojtekmaj/react-pdf) - - [pdf.js](https://mozilla.github.io/pdf.js/) -- **EPUB:** - - [react-reader](https://github.com/gerhardsletten/react-reader) - - [epubjs](https://github.com/futurepress/epub.js/) -- **Markdown/Text:** - - [react-markdown](https://github.com/remarkjs/react-markdown) - - [remark-gfm](https://github.com/remarkjs/remark-gfm) -- **UI:** - - [Tailwind CSS](https://tailwindcss.com) - - [Headless UI](https://headlessui.com) - - [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin) -- **TTS:** (tested on) - - [Deepinfra API](https://deepinfra.com) (Kokoro-82M, Orpheus-3B, Sesame-1B) - - [Kokoro FastAPI TTS](https://github.com/remsky/Kokoro-FastAPI/tree/v0.0.5post1-stable) - - [Orpheus FastAPI TTS](https://github.com/Lex-au/Orpheus-FastAPI) -- **NLP:** - - [compromise](https://github.com/spencermountain/compromise) NLP library for sentence splitting - - [cmpstr](https://github.com/remsky/cmpstr) String comparison library - - [whisper.cpp](https://github.com/ggerganov/whisper.cpp) for TTS timestamps (word-by-word highlighting) +- **Framework:** [Next.js](https://nextjs.org/) 15 (App Router), [React](https://react.dev/) 19, [TypeScript](https://www.typescriptlang.org/) +- **Containerization / Runtime:** [Docker](https://www.docker.com/) (linux/amd64 + linux/arm64), with a shared entrypoint that can bootstrap embedded SeaweedFS before app startup +- **Next.js Client:** + - **UI:** [Tailwind CSS](https://tailwindcss.com), [Headless UI](https://headlessui.com), [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin) + - **Interactions:** `react-dnd`, `react-dropzone` + - **Authentication:** [Better Auth](https://www.better-auth.com/) client SDK (`better-auth/react`, anonymous client plugin) + - **Local storage/cache:** [Dexie.js](https://dexie.org/) (IndexedDB) for documents, cache, and app settings + - **Document rendering:** + - **PDF:** [react-pdf](https://github.com/wojtekmaj/react-pdf), [pdf.js](https://mozilla.github.io/pdf.js/) + - **EPUB:** [react-reader](https://github.com/gerhardsletten/react-reader), [epubjs](https://github.com/futurepress/epub.js/) + - **Markdown/Text:** [react-markdown](https://github.com/remarkjs/react-markdown), [remark-gfm](https://github.com/remarkjs/remark-gfm) + - **Text preprocessing / matching:** [compromise](https://github.com/spencermountain/compromise), [cmpstr](https://github.com/remsky/cmpstr) +- **Next.js Server:** + - **APIs:** Next.js Route Handlers for document sync, blob/content access, migration flows, audiobook export, and TTS/Whisper proxying + - **Authentication:** [Better Auth](https://www.better-auth.com/) server handlers/adapters for session and auth routing + - **Text preprocessing / NLP utilities:** [compromise](https://github.com/spencermountain/compromise) via shared `lib/nlp` helpers used in server processing paths + - **Metadata database:** [Drizzle ORM](https://orm.drizzle.team/), [SQLite](https://www.sqlite.org/) (`better-sqlite3`) by default, optional [PostgreSQL](https://www.postgresql.org/) (`pg`) + - **Blob/object storage:** embedded [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`) by default, or external S3-compatible storage via AWS SDK v3 (`@aws-sdk/client-s3`, presigned URLs, upload fallback proxy) + - **Audio/processing pipeline:** OpenAI-compatible TTS providers (OpenAI, DeepInfra, Kokoro, Orpheus, custom), [ffmpeg](https://ffmpeg.org/) for audiobook assembly, optional [whisper.cpp](https://github.com/ggerganov/whisper.cpp) for word-level timestamps +- **Tooling / Testing:** ESLint, TypeScript, [Playwright](https://playwright.dev/) end-to-end tests, and Drizzle migrations/generation scripts ## License diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml deleted file mode 100644 index 218398c..0000000 --- a/examples/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ -services: - kokoro-tts: - container_name: kokoro-tts - image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 - ports: - - "8880:8880" - environment: - # ONNX Optimization Settings for vectorized operations - - ONNX_NUM_THREADS=8 # Maximize core usage for vectorized ops - - ONNX_INTER_OP_THREADS=4 # Higher inter-op for parallel matrix operations - - ONNX_EXECUTION_MODE=parallel - - ONNX_OPTIMIZATION_LEVEL=all - - ONNX_MEMORY_PATTERN=true - - ONNX_ARENA_EXTEND_STRATEGY=kNextPowerOfTwo - - API_LOG_LEVEL=DEBUG - restart: unless-stopped - - openreader-webui: - container_name: openreader-webui - image: ghcr.io/richardr1126/openreader-webui:latest - environment: - - API_BASE=http://host.docker.internal:8880/v1 - ports: - - "3003:3003" - volumes: - - docstore:/app/docstore - restart: unless-stopped - -volumes: - docstore: \ No newline at end of file diff --git a/package.json b/package.json index d376aa1..dd95738 100644 --- a/package.json +++ b/package.json @@ -3,15 +3,19 @@ "version": "v1.3.0", "private": true, "scripts": { - "dev": "next dev --turbopack -p 3003", + "dev": "node scripts/openreader-entrypoint.mjs -- pnpm dev:raw", + "dev:raw": "next dev --turbopack -p 3003", "build": "next build", - "start": "node drizzle/scripts/migrate.mjs && next start -p 3003", + "start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw", + "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", "migrate": "node drizzle/scripts/migrate.mjs", "generate": "node drizzle/scripts/generate.mjs" }, "dependencies": { + "@aws-sdk/client-s3": "^3.985.0", + "@aws-sdk/s3-request-presigner": "^3.985.0", "@headlessui/react": "^2.2.9", "@types/howler": "^2.2.12", "@types/uuid": "^10.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8cc42a0..f41538b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,12 @@ importers: .: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.985.0 + version: 3.985.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.985.0 + version: 3.985.0 '@headlessui/react': specifier: ^2.2.9 version: 2.2.9(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -156,6 +162,177 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@aws-crypto/crc32@5.2.0': + resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/crc32c@5.2.0': + resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==} + + '@aws-crypto/sha1-browser@5.2.0': + resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} + + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + + '@aws-sdk/client-s3@3.985.0': + resolution: {integrity: sha512-S9TqjzzZEEIKBnC7yFpvqM7CG9ALpY5qhQ5BnDBJtdG20NoGpjKLGUUfD2wmZItuhbrcM4Z8c6m6Fg0XYIOVvw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-sso@3.985.0': + resolution: {integrity: sha512-81J8iE8MuXhdbMfIz4sWFj64Pe41bFi/uqqmqOC5SlGv+kwoyLsyKS/rH2tW2t5buih4vTUxskRjxlqikTD4oQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.973.7': + resolution: {integrity: sha512-wNZZQQNlJ+hzD49cKdo+PY6rsTDElO8yDImnrI69p2PLBa7QomeUKAJWYp9xnaR38nlHqWhMHZuYLCQ3oSX+xg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/crc64-nvme@3.972.0': + resolution: {integrity: sha512-ThlLhTqX68jvoIVv+pryOdb5coP1cX1/MaTbB9xkGDCbWbsqQcLqzPxuSoW1DCnAAIacmXCWpzUNOB9pv+xXQw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.5': + resolution: {integrity: sha512-LxJ9PEO4gKPXzkufvIESUysykPIdrV7+Ocb9yAhbhJLE4TiAYqbCVUE+VuKP1leGR1bBfjWjYgSV5MxprlX3mQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.7': + resolution: {integrity: sha512-L2uOGtvp2x3bTcxFTpSM+GkwFIPd8pHfGWO1764icMbo7e5xJh0nfhx1UwkXLnwvocTNEf8A7jISZLYjUSNaTg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.972.5': + resolution: {integrity: sha512-SdDTYE6jkARzOeL7+kudMIM4DaFnP5dZVeatzw849k4bSXDdErDS188bgeNzc/RA2WGrlEpsqHUKP6G7sVXhZg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.5': + resolution: {integrity: sha512-uYq1ILyTSI6ZDCMY5+vUsRM0SOCVI7kaW4wBrehVVkhAxC6y+e9rvGtnoZqCOWL1gKjTMouvsf4Ilhc5NCg1Aw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.6': + resolution: {integrity: sha512-DZ3CnAAtSVtVz+G+ogqecaErMLgzph4JH5nYbHoBMgBkwTUV+SUcjsjOJwdBJTHu3Dm6l5LBYekZoU2nDqQk2A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.5': + resolution: {integrity: sha512-HDKF3mVbLnuqGg6dMnzBf1VUOywE12/N286msI9YaK9mEIzdsGCtLTvrDhe3Up0R9/hGFbB+9l21/TwF5L1C6g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.972.5': + resolution: {integrity: sha512-8urj3AoeNeQisjMmMBhFeiY2gxt6/7wQQbEGun0YV/OaOOiXrIudTIEYF8ZfD+NQI6X1FY5AkRsx6O/CaGiybA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.5': + resolution: {integrity: sha512-OK3cULuJl6c+RcDZfPpaK5o3deTOnKZbxm7pzhFNGA3fI2hF9yDih17fGRazJzGGWaDVlR9ejZrpDef4DJCEsw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-bucket-endpoint@3.972.3': + resolution: {integrity: sha512-fmbgWYirF67YF1GfD7cg5N6HHQ96EyRNx/rDIrTF277/zTWVuPI2qS/ZHgofwR1NZPe/NWvoppflQY01LrbVLg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-expect-continue@3.972.3': + resolution: {integrity: sha512-4msC33RZsXQpUKR5QR4HnvBSNCPLGHmB55oDiROqqgyOc+TOfVu2xgi5goA7ms6MdZLeEh2905UfWMnMMF4mRg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-flexible-checksums@3.972.5': + resolution: {integrity: sha512-SF/1MYWx67OyCrLA4icIpWUfCkdlOi8Y1KecQ9xYxkL10GMjVdPTGPnYhAg0dw5U43Y9PVUWhAV2ezOaG+0BLg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-host-header@3.972.3': + resolution: {integrity: sha512-aknPTb2M+G3s+0qLCx4Li/qGZH8IIYjugHMv15JTYMe6mgZO8VBpYgeGYsNMGCqCZOcWzuf900jFBG5bopfzmA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-location-constraint@3.972.3': + resolution: {integrity: sha512-nIg64CVrsXp67vbK0U1/Is8rik3huS3QkRHn2DRDx4NldrEFMgdkZGI/+cZMKD9k4YOS110Dfu21KZLHrFA/1g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-logger@3.972.3': + resolution: {integrity: sha512-Ftg09xNNRqaz9QNzlfdQWfpqMCJbsQdnZVJP55jfhbKi1+FTWxGuvfPoBhDHIovqWKjqbuiew3HuhxbJ0+OjgA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-recursion-detection@3.972.3': + resolution: {integrity: sha512-PY57QhzNuXHnwbJgbWYTrqIDHYSeOlhfYERTAuc16LKZpTZRJUjzBFokp9hF7u1fuGeE3D70ERXzdbMBOqQz7Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.7': + resolution: {integrity: sha512-VtZ7tMIw18VzjG+I6D6rh2eLkJfTtByiFoCIauGDtTTPBEUMQUiGaJ/zZrPlCY6BsvLLeFKz3+E5mntgiOWmIg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-ssec@3.972.3': + resolution: {integrity: sha512-dU6kDuULN3o3jEHcjm0c4zWJlY1zWVkjG9NPe9qxYLLpcbdj5kRYBS2DdWYD+1B9f910DezRuws7xDEqKkHQIg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-user-agent@3.972.7': + resolution: {integrity: sha512-HUD+geASjXSCyL/DHPQc/Ua7JhldTcIglVAoCV8kiVm99IaFSlAbTvEnyhZwdE6bdFyTL+uIaWLaCFSRsglZBQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.985.0': + resolution: {integrity: sha512-TsWwKzb/2WHafAY0CE7uXgLj0FmnkBTgfioG9HO+7z/zCPcl1+YU+i7dW4o0y+aFxFgxTMG+ExBQpqT/k2ao8g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/region-config-resolver@3.972.3': + resolution: {integrity: sha512-v4J8qYAWfOMcZ4MJUyatntOicTzEMaU7j3OpkRCGGFSL2NgXQ5VbxauIyORA+pxdKZ0qQG2tCQjQjZDlXEC3Ow==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/s3-request-presigner@3.985.0': + resolution: {integrity: sha512-lPnf977GFM4cMLJ7X+ThktKMe/0CXIfX+wz1z+sUT7yagPL2IRyiNUPFZ0VTEGBo1gRhHEDPWy6yzk8WWRFsvg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.985.0': + resolution: {integrity: sha512-W6hTSOPiSbh4IdTYVxN7xHjpCh0qvfQU1GKGBzGQm0ZEIOaMmWqiDEvFfyGYKmfBvumT8vHKxQRTX0av9omtIg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.985.0': + resolution: {integrity: sha512-+hwpHZyEq8k+9JL2PkE60V93v2kNhUIv7STFt+EAez1UJsJOQDhc5LpzEX66pNjclI5OTwBROs/DhJjC/BtMjQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.1': + resolution: {integrity: sha512-DwHBiMNOB468JiX6+i34c+THsKHErYUdNQ3HexeXZvVn4zouLjgaS4FejiGSi2HyBuzuyHg7SuOPmjSvoU9NRg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-arn-parser@3.972.2': + resolution: {integrity: sha512-VkykWbqMjlSgBFDyrY3nOSqupMc6ivXuGmvci6Q3NnLq5kC+mKQe2QBZ4nrWRE/jqOxeFP2uYzLtwncYYcvQDg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-endpoints@3.985.0': + resolution: {integrity: sha512-vth7UfGSUR3ljvaq8V4Rc62FsM7GUTH/myxPWkaEgOrprz1/Pc72EgTXxj+cPPPDAfHFIpjhkB7T7Td0RJx+BA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-format-url@3.972.3': + resolution: {integrity: sha512-n7F2ycckcKFXa01vAsT/SJdjFHfKH9s96QHcs5gn8AaaigASICeME8WdUL9uBp8XV/OVwEt8+6gzn6KFUgQa8g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-locate-window@3.965.4': + resolution: {integrity: sha512-H1onv5SkgPBK2P6JR2MjGgbOnttoNzSPIRoeZTNPZYyaplwGg50zS3amXvXqF0/qfXpWEC9rLWU564QTB9bSog==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-user-agent-browser@3.972.3': + resolution: {integrity: sha512-JurOwkRUcXD/5MTDBcqdyQ9eVedtAsZgw5rBwktsPTN7QtPiS2Ld1jkJepNgYoCufz1Wcut9iup7GJDoIHp8Fw==} + + '@aws-sdk/util-user-agent-node@3.972.5': + resolution: {integrity: sha512-GsUDF+rXyxDZkkJxUsDxnA67FG+kc5W1dnloCFLl6fWzceevsCYzJpASBzT+BPjwUgREE6FngfJYYYMQUY5fZQ==} + engines: {node: '>=20.0.0'} + peerDependencies: + aws-crt: '>=1.0.0' + peerDependenciesMeta: + aws-crt: + optional: true + + '@aws-sdk/xml-builder@3.972.4': + resolution: {integrity: sha512-0zJ05ANfYqI6+rGqj8samZBFod0dPPousBjLEqg8WdxSgbMAkRgLyn81lP215Do0rFJ/17LIXwr7q0yK24mP6Q==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.2.3': + resolution: {integrity: sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==} + engines: {node: '>=18.0.0'} + '@babel/runtime@7.28.6': resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} engines: {node: '>=6.9.0'} @@ -868,6 +1045,222 @@ packages: '@rushstack/eslint-patch@1.15.0': resolution: {integrity: sha512-ojSshQPKwVvSMR8yT2L/QtUkV5SXi/IfDiJ4/8d6UbTPjiHVmxZzUAzGD8Tzks1b9+qQkZa0isUOvYObedITaw==} + '@smithy/abort-controller@4.2.8': + resolution: {integrity: sha512-peuVfkYHAmS5ybKxWcfraK7WBBP0J+rkfUcbHJJKQ4ir3UAUNQI+Y4Vt/PqSzGqgloJ5O1dk7+WzNL8wcCSXbw==} + engines: {node: '>=18.0.0'} + + '@smithy/chunked-blob-reader-native@4.2.1': + resolution: {integrity: sha512-lX9Ay+6LisTfpLid2zZtIhSEjHMZoAR5hHCR4H7tBz/Zkfr5ea8RcQ7Tk4mi0P76p4cN+Btz16Ffno7YHpKXnQ==} + engines: {node: '>=18.0.0'} + + '@smithy/chunked-blob-reader@5.2.0': + resolution: {integrity: sha512-WmU0TnhEAJLWvfSeMxBNe5xtbselEO8+4wG0NtZeL8oR21WgH1xiO37El+/Y+H/Ie4SCwBy3MxYWmOYaGgZueA==} + engines: {node: '>=18.0.0'} + + '@smithy/config-resolver@4.4.6': + resolution: {integrity: sha512-qJpzYC64kaj3S0fueiu3kXm8xPrR3PcXDPEgnaNMRn0EjNSZFoFjvbUp0YUDsRhN1CB90EnHJtbxWKevnH99UQ==} + engines: {node: '>=18.0.0'} + + '@smithy/core@3.22.1': + resolution: {integrity: sha512-x3ie6Crr58MWrm4viHqqy2Du2rHYZjwu8BekasrQx4ca+Y24dzVAwq3yErdqIbc2G3I0kLQA13PQ+/rde+u65g==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.2.8': + resolution: {integrity: sha512-FNT0xHS1c/CPN8upqbMFP83+ul5YgdisfCfkZ86Jh2NSmnqw/AJ6x5pEogVCTVvSm7j9MopRU89bmDelxuDMYw==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-codec@4.2.8': + resolution: {integrity: sha512-jS/O5Q14UsufqoGhov7dHLOPCzkYJl9QDzusI2Psh4wyYx/izhzvX9P4D69aTxcdfVhEPhjK+wYyn/PzLjKbbw==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-browser@4.2.8': + resolution: {integrity: sha512-MTfQT/CRQz5g24ayXdjg53V0mhucZth4PESoA5IhvaWVDTOQLfo8qI9vzqHcPsdd2v6sqfTYqF5L/l+pea5Uyw==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-config-resolver@4.3.8': + resolution: {integrity: sha512-ah12+luBiDGzBruhu3efNy1IlbwSEdNiw8fOZksoKoWW1ZHvO/04MQsdnws/9Aj+5b0YXSSN2JXKy/ClIsW8MQ==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-node@4.2.8': + resolution: {integrity: sha512-cYpCpp29z6EJHa5T9WL0KAlq3SOKUQkcgSoeRfRVwjGgSFl7Uh32eYGt7IDYCX20skiEdRffyDpvF2efEZPC0A==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-universal@4.2.8': + resolution: {integrity: sha512-iJ6YNJd0bntJYnX6s52NC4WFYcZeKrPUr1Kmmr5AwZcwCSzVpS7oavAmxMR7pMq7V+D1G4s9F5NJK0xwOsKAlQ==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.3.9': + resolution: {integrity: sha512-I4UhmcTYXBrct03rwzQX1Y/iqQlzVQaPxWjCjula++5EmWq9YGBrx6bbGqluGc1f0XEfhSkiY4jhLgbsJUMKRA==} + engines: {node: '>=18.0.0'} + + '@smithy/hash-blob-browser@4.2.9': + resolution: {integrity: sha512-m80d/iicI7DlBDxyQP6Th7BW/ejDGiF0bgI754+tiwK0lgMkcaIBgvwwVc7OFbY4eUzpGtnig52MhPAEJ7iNYg==} + engines: {node: '>=18.0.0'} + + '@smithy/hash-node@4.2.8': + resolution: {integrity: sha512-7ZIlPbmaDGxVoxErDZnuFG18WekhbA/g2/i97wGj+wUBeS6pcUeAym8u4BXh/75RXWhgIJhyC11hBzig6MljwA==} + engines: {node: '>=18.0.0'} + + '@smithy/hash-stream-node@4.2.8': + resolution: {integrity: sha512-v0FLTXgHrTeheYZFGhR+ehX5qUm4IQsjAiL9qehad2cyjMWcN2QG6/4mSwbSgEQzI7jwfoXj7z4fxZUx/Mhj2w==} + engines: {node: '>=18.0.0'} + + '@smithy/invalid-dependency@4.2.8': + resolution: {integrity: sha512-N9iozRybwAQ2dn9Fot9kI6/w9vos2oTXLhtK7ovGqwZjlOcxu6XhPlpLpC+INsxktqHinn5gS2DXDjDF2kG5sQ==} + engines: {node: '>=18.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/is-array-buffer@4.2.0': + resolution: {integrity: sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==} + engines: {node: '>=18.0.0'} + + '@smithy/md5-js@4.2.8': + resolution: {integrity: sha512-oGMaLj4tVZzLi3itBa9TCswgMBr7k9b+qKYowQ6x1rTyTuO1IU2YHdHUa+891OsOH+wCsH7aTPRsTJO3RMQmjQ==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-content-length@4.2.8': + resolution: {integrity: sha512-RO0jeoaYAB1qBRhfVyq0pMgBoUK34YEJxVxyjOWYZiOKOq2yMZ4MnVXMZCUDenpozHue207+9P5ilTV1zeda0A==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-endpoint@4.4.13': + resolution: {integrity: sha512-x6vn0PjYmGdNuKh/juUJJewZh7MoQ46jYaJ2mvekF4EesMuFfrl4LaW/k97Zjf8PTCPQmPgMvwewg7eNoH9n5w==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-retry@4.4.30': + resolution: {integrity: sha512-CBGyFvN0f8hlnqKH/jckRDz78Snrp345+PVk8Ux7pnkUCW97Iinse59lY78hBt04h1GZ6hjBN94BRwZy1xC8Bg==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-serde@4.2.9': + resolution: {integrity: sha512-eMNiej0u/snzDvlqRGSN3Vl0ESn3838+nKyVfF2FKNXFbi4SERYT6PR392D39iczngbqqGG0Jl1DlCnp7tBbXQ==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-stack@4.2.8': + resolution: {integrity: sha512-w6LCfOviTYQjBctOKSwy6A8FIkQy7ICvglrZFl6Bw4FmcQ1Z420fUtIhxaUZZshRe0VCq4kvDiPiXrPZAe8oRA==} + engines: {node: '>=18.0.0'} + + '@smithy/node-config-provider@4.3.8': + resolution: {integrity: sha512-aFP1ai4lrbVlWjfpAfRSL8KFcnJQYfTl5QxLJXY32vghJrDuFyPZ6LtUL+JEGYiFRG1PfPLHLoxj107ulncLIg==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.4.9': + resolution: {integrity: sha512-KX5Wml5mF+luxm1szW4QDz32e3NObgJ4Fyw+irhph4I/2geXwUy4jkIMUs5ZPGflRBeR6BUkC2wqIab4Llgm3w==} + engines: {node: '>=18.0.0'} + + '@smithy/property-provider@4.2.8': + resolution: {integrity: sha512-EtCTbyIveCKeOXDSWSdze3k612yCPq1YbXsbqX3UHhkOSW8zKsM9NOJG5gTIya0vbY2DIaieG8pKo1rITHYL0w==} + engines: {node: '>=18.0.0'} + + '@smithy/protocol-http@5.3.8': + resolution: {integrity: sha512-QNINVDhxpZ5QnP3aviNHQFlRogQZDfYlCkQT+7tJnErPQbDhysondEjhikuANxgMsZrkGeiAxXy4jguEGsDrWQ==} + engines: {node: '>=18.0.0'} + + '@smithy/querystring-builder@4.2.8': + resolution: {integrity: sha512-Xr83r31+DrE8CP3MqPgMJl+pQlLLmOfiEUnoyAlGzzJIrEsbKsPy1hqH0qySaQm4oWrCBlUqRt+idEgunKB+iw==} + engines: {node: '>=18.0.0'} + + '@smithy/querystring-parser@4.2.8': + resolution: {integrity: sha512-vUurovluVy50CUlazOiXkPq40KGvGWSdmusa3130MwrR1UNnNgKAlj58wlOe61XSHRpUfIIh6cE0zZ8mzKaDPA==} + engines: {node: '>=18.0.0'} + + '@smithy/service-error-classification@4.2.8': + resolution: {integrity: sha512-mZ5xddodpJhEt3RkCjbmUQuXUOaPNTkbMGR0bcS8FE0bJDLMZlhmpgrvPNCYglVw5rsYTpSnv19womw9WWXKQQ==} + engines: {node: '>=18.0.0'} + + '@smithy/shared-ini-file-loader@4.4.3': + resolution: {integrity: sha512-DfQjxXQnzC5UbCUPeC3Ie8u+rIWZTvuDPAGU/BxzrOGhRvgUanaP68kDZA+jaT3ZI+djOf+4dERGlm9mWfFDrg==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.3.8': + resolution: {integrity: sha512-6A4vdGj7qKNRF16UIcO8HhHjKW27thsxYci+5r/uVRkdcBEkOEiY8OMPuydLX4QHSrJqGHPJzPRwwVTqbLZJhg==} + engines: {node: '>=18.0.0'} + + '@smithy/smithy-client@4.11.2': + resolution: {integrity: sha512-SCkGmFak/xC1n7hKRsUr6wOnBTJ3L22Qd4e8H1fQIuKTAjntwgU8lrdMe7uHdiT2mJAOWA/60qaW9tiMu69n1A==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.12.0': + resolution: {integrity: sha512-9YcuJVTOBDjg9LWo23Qp0lTQ3D7fQsQtwle0jVfpbUHy9qBwCEgKuVH4FqFB3VYu0nwdHKiEMA+oXz7oV8X1kw==} + engines: {node: '>=18.0.0'} + + '@smithy/url-parser@4.2.8': + resolution: {integrity: sha512-NQho9U68TGMEU639YkXnVMV3GEFFULmmaWdlu1E9qzyIePOHsoSnagTGSDv1Zi8DCNN6btxOSdgmy5E/hsZwhA==} + engines: {node: '>=18.0.0'} + + '@smithy/util-base64@4.3.0': + resolution: {integrity: sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-body-length-browser@4.2.0': + resolution: {integrity: sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-body-length-node@4.2.1': + resolution: {integrity: sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==} + engines: {node: '>=18.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-buffer-from@4.2.0': + resolution: {integrity: sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==} + engines: {node: '>=18.0.0'} + + '@smithy/util-config-provider@4.2.0': + resolution: {integrity: sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==} + engines: {node: '>=18.0.0'} + + '@smithy/util-defaults-mode-browser@4.3.29': + resolution: {integrity: sha512-nIGy3DNRmOjaYaaKcQDzmWsro9uxlaqUOhZDHQed9MW/GmkBZPtnU70Pu1+GT9IBmUXwRdDuiyaeiy9Xtpn3+Q==} + engines: {node: '>=18.0.0'} + + '@smithy/util-defaults-mode-node@4.2.32': + resolution: {integrity: sha512-7dtFff6pu5fsjqrVve0YMhrnzJtccCWDacNKOkiZjJ++fmjGExmmSu341x+WU6Oc1IccL7lDuaUj7SfrHpWc5Q==} + engines: {node: '>=18.0.0'} + + '@smithy/util-endpoints@3.2.8': + resolution: {integrity: sha512-8JaVTn3pBDkhZgHQ8R0epwWt+BqPSLCjdjXXusK1onwJlRuN69fbvSK66aIKKO7SwVFM6x2J2ox5X8pOaWcUEw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-hex-encoding@4.2.0': + resolution: {integrity: sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-middleware@4.2.8': + resolution: {integrity: sha512-PMqfeJxLcNPMDgvPbbLl/2Vpin+luxqTGPpW3NAQVLbRrFRzTa4rNAASYeIGjRV9Ytuhzny39SpyU04EQreF+A==} + engines: {node: '>=18.0.0'} + + '@smithy/util-retry@4.2.8': + resolution: {integrity: sha512-CfJqwvoRY0kTGe5AkQokpURNCT1u/MkRzMTASWMPPo2hNSnKtF1D45dQl3DE2LKLr4m+PW9mCeBMJr5mCAVThg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-stream@4.5.11': + resolution: {integrity: sha512-lKmZ0S/3Qj2OF5H1+VzvDLb6kRxGzZHq6f3rAsoSu5cTLGsn3v3VQBA8czkNNXlLjoFEtVu3OQT2jEeOtOE2CA==} + engines: {node: '>=18.0.0'} + + '@smithy/util-uri-escape@4.2.0': + resolution: {integrity: sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==} + engines: {node: '>=18.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@4.2.0': + resolution: {integrity: sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-waiter@4.2.8': + resolution: {integrity: sha512-n+lahlMWk+aejGuax7DPWtqav8HYnWxQwR+LCG2BgCUmaGcTe9qZCFsmw8TMg9iG75HOwhrJCX9TCJRLH+Yzqg==} + engines: {node: '>=18.0.0'} + + '@smithy/uuid@1.1.0': + resolution: {integrity: sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -1316,6 +1709,9 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + bowser@2.13.1: + resolution: {integrity: sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -1869,6 +2265,10 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-xml-parser@5.3.4: + resolution: {integrity: sha512-EFd6afGmXlCx8H8WTZHhAoDaWaGyuIBoZJ2mknrNxug+aZKjkp0a0dlars9Izl+jF+7Gu1/5f/2h68cQpe0IiA==} + hasBin: true + fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -3076,6 +3476,9 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strnum@2.1.2: + resolution: {integrity: sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==} + style-to-js@1.1.21: resolution: {integrity: sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==} @@ -3294,6 +3697,509 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@aws-crypto/crc32@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.1 + tslib: 2.8.1 + + '@aws-crypto/crc32c@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.1 + tslib: 2.8.1 + + '@aws-crypto/sha1-browser@5.2.0': + dependencies: + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-locate-window': 3.965.4 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-locate-window': 3.965.4 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.1 + tslib: 2.8.1 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.8.1 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.985.0': + dependencies: + '@aws-crypto/sha1-browser': 5.2.0 + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.973.7 + '@aws-sdk/credential-provider-node': 3.972.6 + '@aws-sdk/middleware-bucket-endpoint': 3.972.3 + '@aws-sdk/middleware-expect-continue': 3.972.3 + '@aws-sdk/middleware-flexible-checksums': 3.972.5 + '@aws-sdk/middleware-host-header': 3.972.3 + '@aws-sdk/middleware-location-constraint': 3.972.3 + '@aws-sdk/middleware-logger': 3.972.3 + '@aws-sdk/middleware-recursion-detection': 3.972.3 + '@aws-sdk/middleware-sdk-s3': 3.972.7 + '@aws-sdk/middleware-ssec': 3.972.3 + '@aws-sdk/middleware-user-agent': 3.972.7 + '@aws-sdk/region-config-resolver': 3.972.3 + '@aws-sdk/signature-v4-multi-region': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-endpoints': 3.985.0 + '@aws-sdk/util-user-agent-browser': 3.972.3 + '@aws-sdk/util-user-agent-node': 3.972.5 + '@smithy/config-resolver': 4.4.6 + '@smithy/core': 3.22.1 + '@smithy/eventstream-serde-browser': 4.2.8 + '@smithy/eventstream-serde-config-resolver': 4.3.8 + '@smithy/eventstream-serde-node': 4.2.8 + '@smithy/fetch-http-handler': 5.3.9 + '@smithy/hash-blob-browser': 4.2.9 + '@smithy/hash-node': 4.2.8 + '@smithy/hash-stream-node': 4.2.8 + '@smithy/invalid-dependency': 4.2.8 + '@smithy/md5-js': 4.2.8 + '@smithy/middleware-content-length': 4.2.8 + '@smithy/middleware-endpoint': 4.4.13 + '@smithy/middleware-retry': 4.4.30 + '@smithy/middleware-serde': 4.2.9 + '@smithy/middleware-stack': 4.2.8 + '@smithy/node-config-provider': 4.3.8 + '@smithy/node-http-handler': 4.4.9 + '@smithy/protocol-http': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/url-parser': 4.2.8 + '@smithy/util-base64': 4.3.0 + '@smithy/util-body-length-browser': 4.2.0 + '@smithy/util-body-length-node': 4.2.1 + '@smithy/util-defaults-mode-browser': 4.3.29 + '@smithy/util-defaults-mode-node': 4.2.32 + '@smithy/util-endpoints': 3.2.8 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-retry': 4.2.8 + '@smithy/util-stream': 4.5.11 + '@smithy/util-utf8': 4.2.0 + '@smithy/util-waiter': 4.2.8 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/client-sso@3.985.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.973.7 + '@aws-sdk/middleware-host-header': 3.972.3 + '@aws-sdk/middleware-logger': 3.972.3 + '@aws-sdk/middleware-recursion-detection': 3.972.3 + '@aws-sdk/middleware-user-agent': 3.972.7 + '@aws-sdk/region-config-resolver': 3.972.3 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-endpoints': 3.985.0 + '@aws-sdk/util-user-agent-browser': 3.972.3 + '@aws-sdk/util-user-agent-node': 3.972.5 + '@smithy/config-resolver': 4.4.6 + '@smithy/core': 3.22.1 + '@smithy/fetch-http-handler': 5.3.9 + '@smithy/hash-node': 4.2.8 + '@smithy/invalid-dependency': 4.2.8 + '@smithy/middleware-content-length': 4.2.8 + '@smithy/middleware-endpoint': 4.4.13 + '@smithy/middleware-retry': 4.4.30 + '@smithy/middleware-serde': 4.2.9 + '@smithy/middleware-stack': 4.2.8 + '@smithy/node-config-provider': 4.3.8 + '@smithy/node-http-handler': 4.4.9 + '@smithy/protocol-http': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/url-parser': 4.2.8 + '@smithy/util-base64': 4.3.0 + '@smithy/util-body-length-browser': 4.2.0 + '@smithy/util-body-length-node': 4.2.1 + '@smithy/util-defaults-mode-browser': 4.3.29 + '@smithy/util-defaults-mode-node': 4.2.32 + '@smithy/util-endpoints': 3.2.8 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-retry': 4.2.8 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/core@3.973.7': + dependencies: + '@aws-sdk/types': 3.973.1 + '@aws-sdk/xml-builder': 3.972.4 + '@smithy/core': 3.22.1 + '@smithy/node-config-provider': 4.3.8 + '@smithy/property-provider': 4.2.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/signature-v4': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/util-base64': 4.3.0 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/crc64-nvme@3.972.0': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.5': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/types': 3.973.1 + '@smithy/property-provider': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.7': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/types': 3.973.1 + '@smithy/fetch-http-handler': 5.3.9 + '@smithy/node-http-handler': 4.4.9 + '@smithy/property-provider': 4.2.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/util-stream': 4.5.11 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.972.5': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/credential-provider-env': 3.972.5 + '@aws-sdk/credential-provider-http': 3.972.7 + '@aws-sdk/credential-provider-login': 3.972.5 + '@aws-sdk/credential-provider-process': 3.972.5 + '@aws-sdk/credential-provider-sso': 3.972.5 + '@aws-sdk/credential-provider-web-identity': 3.972.5 + '@aws-sdk/nested-clients': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@smithy/credential-provider-imds': 4.2.8 + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-login@3.972.5': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/nested-clients': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@smithy/property-provider': 4.2.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-node@3.972.6': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.5 + '@aws-sdk/credential-provider-http': 3.972.7 + '@aws-sdk/credential-provider-ini': 3.972.5 + '@aws-sdk/credential-provider-process': 3.972.5 + '@aws-sdk/credential-provider-sso': 3.972.5 + '@aws-sdk/credential-provider-web-identity': 3.972.5 + '@aws-sdk/types': 3.973.1 + '@smithy/credential-provider-imds': 4.2.8 + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-process@3.972.5': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/types': 3.973.1 + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.972.5': + dependencies: + '@aws-sdk/client-sso': 3.985.0 + '@aws-sdk/core': 3.973.7 + '@aws-sdk/token-providers': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-web-identity@3.972.5': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/nested-clients': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/middleware-bucket-endpoint@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-arn-parser': 3.972.2 + '@smithy/node-config-provider': 4.3.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + '@smithy/util-config-provider': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-expect-continue@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-flexible-checksums@3.972.5': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@aws-crypto/crc32c': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/core': 3.973.7 + '@aws-sdk/crc64-nvme': 3.972.0 + '@aws-sdk/types': 3.973.1 + '@smithy/is-array-buffer': 4.2.0 + '@smithy/node-config-provider': 4.3.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-stream': 4.5.11 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-host-header@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-location-constraint@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-logger@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-recursion-detection@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@aws/lambda-invoke-store': 0.2.3 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.7': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-arn-parser': 3.972.2 + '@smithy/core': 3.22.1 + '@smithy/node-config-provider': 4.3.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/signature-v4': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/util-config-provider': 4.2.0 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-stream': 4.5.11 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-ssec@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-user-agent@3.972.7': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-endpoints': 3.985.0 + '@smithy/core': 3.22.1 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.985.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.973.7 + '@aws-sdk/middleware-host-header': 3.972.3 + '@aws-sdk/middleware-logger': 3.972.3 + '@aws-sdk/middleware-recursion-detection': 3.972.3 + '@aws-sdk/middleware-user-agent': 3.972.7 + '@aws-sdk/region-config-resolver': 3.972.3 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-endpoints': 3.985.0 + '@aws-sdk/util-user-agent-browser': 3.972.3 + '@aws-sdk/util-user-agent-node': 3.972.5 + '@smithy/config-resolver': 4.4.6 + '@smithy/core': 3.22.1 + '@smithy/fetch-http-handler': 5.3.9 + '@smithy/hash-node': 4.2.8 + '@smithy/invalid-dependency': 4.2.8 + '@smithy/middleware-content-length': 4.2.8 + '@smithy/middleware-endpoint': 4.4.13 + '@smithy/middleware-retry': 4.4.30 + '@smithy/middleware-serde': 4.2.9 + '@smithy/middleware-stack': 4.2.8 + '@smithy/node-config-provider': 4.3.8 + '@smithy/node-http-handler': 4.4.9 + '@smithy/protocol-http': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/url-parser': 4.2.8 + '@smithy/util-base64': 4.3.0 + '@smithy/util-body-length-browser': 4.2.0 + '@smithy/util-body-length-node': 4.2.1 + '@smithy/util-defaults-mode-browser': 4.3.29 + '@smithy/util-defaults-mode-node': 4.2.32 + '@smithy/util-endpoints': 3.2.8 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-retry': 4.2.8 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/region-config-resolver@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/config-resolver': 4.4.6 + '@smithy/node-config-provider': 4.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/s3-request-presigner@3.985.0': + dependencies: + '@aws-sdk/signature-v4-multi-region': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@aws-sdk/util-format-url': 3.972.3 + '@smithy/middleware-endpoint': 4.4.13 + '@smithy/protocol-http': 5.3.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.985.0': + dependencies: + '@aws-sdk/middleware-sdk-s3': 3.972.7 + '@aws-sdk/types': 3.973.1 + '@smithy/protocol-http': 5.3.8 + '@smithy/signature-v4': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.985.0': + dependencies: + '@aws-sdk/core': 3.973.7 + '@aws-sdk/nested-clients': 3.985.0 + '@aws-sdk/types': 3.973.1 + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/types@3.973.1': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/util-arn-parser@3.972.2': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/util-endpoints@3.985.0': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/types': 4.12.0 + '@smithy/url-parser': 4.2.8 + '@smithy/util-endpoints': 3.2.8 + tslib: 2.8.1 + + '@aws-sdk/util-format-url@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/querystring-builder': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/util-locate-window@3.965.4': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/util-user-agent-browser@3.972.3': + dependencies: + '@aws-sdk/types': 3.973.1 + '@smithy/types': 4.12.0 + bowser: 2.13.1 + tslib: 2.8.1 + + '@aws-sdk/util-user-agent-node@3.972.5': + dependencies: + '@aws-sdk/middleware-user-agent': 3.972.7 + '@aws-sdk/types': 3.973.1 + '@smithy/node-config-provider': 4.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.4': + dependencies: + '@smithy/types': 4.12.0 + fast-xml-parser: 5.3.4 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.2.3': {} + '@babel/runtime@7.28.6': {} '@better-auth/core@1.4.17(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.1.3)(kysely@0.28.10)(nanostores@1.1.0)': @@ -3813,6 +4719,344 @@ snapshots: '@rushstack/eslint-patch@1.15.0': {} + '@smithy/abort-controller@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/chunked-blob-reader-native@4.2.1': + dependencies: + '@smithy/util-base64': 4.3.0 + tslib: 2.8.1 + + '@smithy/chunked-blob-reader@5.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/config-resolver@4.4.6': + dependencies: + '@smithy/node-config-provider': 4.3.8 + '@smithy/types': 4.12.0 + '@smithy/util-config-provider': 4.2.0 + '@smithy/util-endpoints': 3.2.8 + '@smithy/util-middleware': 4.2.8 + tslib: 2.8.1 + + '@smithy/core@3.22.1': + dependencies: + '@smithy/middleware-serde': 4.2.9 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + '@smithy/util-base64': 4.3.0 + '@smithy/util-body-length-browser': 4.2.0 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-stream': 4.5.11 + '@smithy/util-utf8': 4.2.0 + '@smithy/uuid': 1.1.0 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.2.8': + dependencies: + '@smithy/node-config-provider': 4.3.8 + '@smithy/property-provider': 4.2.8 + '@smithy/types': 4.12.0 + '@smithy/url-parser': 4.2.8 + tslib: 2.8.1 + + '@smithy/eventstream-codec@4.2.8': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.12.0 + '@smithy/util-hex-encoding': 4.2.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-browser@4.2.8': + dependencies: + '@smithy/eventstream-serde-universal': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-config-resolver@4.3.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-node@4.2.8': + dependencies: + '@smithy/eventstream-serde-universal': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-universal@4.2.8': + dependencies: + '@smithy/eventstream-codec': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.3.9': + dependencies: + '@smithy/protocol-http': 5.3.8 + '@smithy/querystring-builder': 4.2.8 + '@smithy/types': 4.12.0 + '@smithy/util-base64': 4.3.0 + tslib: 2.8.1 + + '@smithy/hash-blob-browser@4.2.9': + dependencies: + '@smithy/chunked-blob-reader': 5.2.0 + '@smithy/chunked-blob-reader-native': 4.2.1 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/hash-node@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + '@smithy/util-buffer-from': 4.2.0 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@smithy/hash-stream-node@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@smithy/invalid-dependency@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/is-array-buffer@4.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/md5-js@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@smithy/middleware-content-length@4.2.8': + dependencies: + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/middleware-endpoint@4.4.13': + dependencies: + '@smithy/core': 3.22.1 + '@smithy/middleware-serde': 4.2.9 + '@smithy/node-config-provider': 4.3.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + '@smithy/url-parser': 4.2.8 + '@smithy/util-middleware': 4.2.8 + tslib: 2.8.1 + + '@smithy/middleware-retry@4.4.30': + dependencies: + '@smithy/node-config-provider': 4.3.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/service-error-classification': 4.2.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-retry': 4.2.8 + '@smithy/uuid': 1.1.0 + tslib: 2.8.1 + + '@smithy/middleware-serde@4.2.9': + dependencies: + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/middleware-stack@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/node-config-provider@4.3.8': + dependencies: + '@smithy/property-provider': 4.2.8 + '@smithy/shared-ini-file-loader': 4.4.3 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.4.9': + dependencies: + '@smithy/abort-controller': 4.2.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/querystring-builder': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/property-provider@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/protocol-http@5.3.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/querystring-builder@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + '@smithy/util-uri-escape': 4.2.0 + tslib: 2.8.1 + + '@smithy/querystring-parser@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/service-error-classification@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + + '@smithy/shared-ini-file-loader@4.4.3': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/signature-v4@5.3.8': + dependencies: + '@smithy/is-array-buffer': 4.2.0 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + '@smithy/util-hex-encoding': 4.2.0 + '@smithy/util-middleware': 4.2.8 + '@smithy/util-uri-escape': 4.2.0 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@smithy/smithy-client@4.11.2': + dependencies: + '@smithy/core': 3.22.1 + '@smithy/middleware-endpoint': 4.4.13 + '@smithy/middleware-stack': 4.2.8 + '@smithy/protocol-http': 5.3.8 + '@smithy/types': 4.12.0 + '@smithy/util-stream': 4.5.11 + tslib: 2.8.1 + + '@smithy/types@4.12.0': + dependencies: + tslib: 2.8.1 + + '@smithy/url-parser@4.2.8': + dependencies: + '@smithy/querystring-parser': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/util-base64@4.3.0': + dependencies: + '@smithy/util-buffer-from': 4.2.0 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-body-length-browser@4.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-body-length-node@4.2.1': + dependencies: + tslib: 2.8.1 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-buffer-from@4.2.0': + dependencies: + '@smithy/is-array-buffer': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-config-provider@4.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-defaults-mode-browser@4.3.29': + dependencies: + '@smithy/property-provider': 4.2.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/util-defaults-mode-node@4.2.32': + dependencies: + '@smithy/config-resolver': 4.4.6 + '@smithy/credential-provider-imds': 4.2.8 + '@smithy/node-config-provider': 4.3.8 + '@smithy/property-provider': 4.2.8 + '@smithy/smithy-client': 4.11.2 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/util-endpoints@3.2.8': + dependencies: + '@smithy/node-config-provider': 4.3.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/util-hex-encoding@4.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-middleware@4.2.8': + dependencies: + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/util-retry@4.2.8': + dependencies: + '@smithy/service-error-classification': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/util-stream@4.5.11': + dependencies: + '@smithy/fetch-http-handler': 5.3.9 + '@smithy/node-http-handler': 4.4.9 + '@smithy/types': 4.12.0 + '@smithy/util-base64': 4.3.0 + '@smithy/util-buffer-from': 4.2.0 + '@smithy/util-hex-encoding': 4.2.0 + '@smithy/util-utf8': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-uri-escape@4.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-utf8@4.2.0': + dependencies: + '@smithy/util-buffer-from': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-waiter@4.2.8': + dependencies: + '@smithy/abort-controller': 4.2.8 + '@smithy/types': 4.12.0 + tslib: 2.8.1 + + '@smithy/uuid@1.1.0': + dependencies: + tslib: 2.8.1 + '@standard-schema/spec@1.1.0': {} '@swc/helpers@0.5.15': @@ -4221,6 +5465,8 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + bowser@2.13.1: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -4898,6 +6144,10 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-xml-parser@5.3.4: + dependencies: + strnum: 2.1.2 + fastq@1.20.1: dependencies: reusify: 1.1.0 @@ -6422,6 +7672,8 @@ snapshots: strip-json-comments@3.1.1: {} + strnum@2.1.2: {} + style-to-js@1.1.21: dependencies: style-to-object: 1.0.14 diff --git a/scripts/openreader-entrypoint.mjs b/scripts/openreader-entrypoint.mjs new file mode 100644 index 0000000..a39df05 --- /dev/null +++ b/scripts/openreader-entrypoint.mjs @@ -0,0 +1,367 @@ +#!/usr/bin/env node +import { spawn, spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { once } from 'node:events'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { setTimeout as delay } from 'node:timers/promises'; +import * as dotenv from 'dotenv'; + +function loadEnvFiles() { + const cwd = process.cwd(); + const envPath = path.join(cwd, '.env'); + const envLocalPath = path.join(cwd, '.env.local'); + + if (fs.existsSync(envPath)) { + dotenv.config({ path: envPath }); + } + if (fs.existsSync(envLocalPath)) { + dotenv.config({ path: envLocalPath, override: true }); + } +} + +function isTrue(value, defaultValue) { + if (value == null || value.trim() === '') return defaultValue; + const normalized = value.trim().toLowerCase(); + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; +} + +function withDefault(value, fallback) { + return value && value.trim() ? value.trim() : fallback; +} + +function isPrivateIPv4(address) { + if (!address) return false; + if (address.startsWith('10.')) return true; + if (address.startsWith('192.168.')) return true; + const m = /^172\.(\d+)\./.exec(address); + if (m) { + const second = Number.parseInt(m[1], 10); + if (second >= 16 && second <= 31) return true; + } + return false; +} + +function detectHostForDefaultEndpoint() { + const interfaces = os.networkInterfaces(); + const ipv4 = []; + + for (const entries of Object.values(interfaces)) { + for (const entry of entries || []) { + if (!entry) continue; + const family = typeof entry.family === 'string' ? entry.family : String(entry.family); + if (family !== 'IPv4') continue; + if (entry.internal) continue; + ipv4.push(entry.address); + } + } + + const privateAddr = ipv4.find(isPrivateIPv4); + if (privateAddr) return privateAddr; + if (ipv4[0]) return ipv4[0]; + return '127.0.0.1'; +} + +function parseS3Endpoint(endpoint) { + let url; + try { + url = new URL(endpoint); + } catch { + throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`); + } + + if (!url.hostname) { + throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`); + } + + const port = Number.parseInt(url.port || '8333', 10); + if (!Number.isFinite(port) || port < 1 || port > 65535) { + throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`); + } + + return { + hostname: url.hostname, + port, + normalized: `${url.protocol}//${url.hostname}:${port}`, + }; +} + +function parseUrlHost(urlValue, fieldName) { + let url; + try { + url = new URL(urlValue); + } catch { + throw new Error(`Invalid ${fieldName}: ${urlValue}`); + } + + if (!url.hostname) { + throw new Error(`Invalid ${fieldName} host: ${urlValue}`); + } + + return url.hostname; +} + +function parseCommandFromArgs(argv) { + const marker = argv.indexOf('--'); + if (marker >= 0) return argv.slice(marker + 1); + return argv; +} + +function forwardChildStream(stream, target) { + if (!stream) return () => {}; + const onData = (chunk) => { + target.write(chunk); + }; + stream.on('data', onData); + return () => { + stream.off('data', onData); + }; +} + +function hasWeedBinary() { + const probe = spawnSync('weed', ['version'], { stdio: 'ignore' }); + if (probe.error) return false; + return true; +} + +async function waitForEndpoint(url, timeoutSeconds) { + const waitMs = Math.max(1, timeoutSeconds) * 1000; + const deadline = Date.now() + waitMs; + + while (Date.now() < deadline) { + try { + const res = await fetch(url, { method: 'GET' }); + if (res) return; + } catch { + // retry + } + await delay(1000); + } + + throw new Error(`Embedded weed mini did not become ready at ${url} within ${timeoutSeconds}s.`); +} + +function spawnMainCommand(command, env) { + const [cmd, ...args] = command; + const child = spawn(cmd, args, { + env, + stdio: 'inherit', + shell: process.platform === 'win32', + }); + + const exitPromise = new Promise((resolve) => { + child.on('error', (error) => { + console.error('Failed to launch command:', error); + resolve(1); + }); + + child.on('exit', (code, signal) => { + if (typeof code === 'number') { + resolve(code); + return; + } + if (signal) { + resolve(1); + return; + } + resolve(0); + }); + }); + + return { child, exitPromise }; +} + +function runDbMigrations(env) { + const migrateScript = path.join(process.cwd(), 'drizzle', 'scripts', 'migrate.mjs'); + if (!fs.existsSync(migrateScript)) { + throw new Error(`Could not find migration script at ${migrateScript}`); + } + + console.log('Running database migrations...'); + const migration = spawnSync(process.execPath, [migrateScript], { + env, + stdio: 'inherit', + }); + + if (migration.error) { + throw migration.error; + } + if (typeof migration.status === 'number' && migration.status !== 0) { + throw new Error(`Database migrations failed with exit code ${migration.status}.`); + } +} + +function sendSignal(child, signal, useProcessGroup) { + if (!child) return false; + if (useProcessGroup && process.platform !== 'win32' && typeof child.pid === 'number' && child.pid > 0) { + try { + process.kill(-child.pid, signal); + return true; + } catch { + return false; + } + } + + try { + child.kill(signal); + return true; + } catch { + return false; + } +} + +async function terminateChild(child, signal = 'SIGTERM', graceMs = 3000, useProcessGroup = false) { + if (!child) return; + if (child.exitCode != null) return; + + if (!sendSignal(child, signal, useProcessGroup)) return; + + const exited = await Promise.race([ + once(child, 'exit').then(() => true).catch(() => true), + delay(graceMs).then(() => false), + ]); + + if (exited) return; + + if (!sendSignal(child, 'SIGKILL', useProcessGroup)) return; + + await Promise.race([ + once(child, 'exit').then(() => true).catch(() => true), + delay(1000).then(() => false), + ]); +} + +async function main() { + loadEnvFiles(); + + const command = parseCommandFromArgs(process.argv.slice(2)); + if (command.length === 0) { + console.error('Usage: node scripts/openreader-entrypoint.mjs -- [args]'); + process.exit(2); + } + + const embeddedEnvRaw = process.env.USE_EMBEDDED_WEED_MINI; + let useEmbeddedWeed = isTrue(embeddedEnvRaw, true); + + if (useEmbeddedWeed && !hasWeedBinary()) { + if (embeddedEnvRaw && isTrue(embeddedEnvRaw, true)) { + console.error('USE_EMBEDDED_WEED_MINI=true but `weed` binary is not available in PATH.'); + process.exit(1); + } + useEmbeddedWeed = false; + console.warn('`weed` binary not found; skipping embedded SeaweedFS startup.'); + } + + const runtimeEnv = { ...process.env }; + let weedProc = null; + let weedExitPromise = Promise.resolve(); + let appProc = null; + let shutdownPromise = null; + let stopWeedStdoutForward = () => {}; + let stopWeedStderrForward = () => {}; + let didExit = false; + + const exitOnce = (code) => { + if (didExit) return; + didExit = true; + process.exit(code); + }; + + const shutdown = async (signal = 'SIGTERM') => { + if (shutdownPromise) return shutdownPromise; + shutdownPromise = (async () => { + await Promise.all([ + terminateChild(appProc, signal, 4000), + terminateChild(weedProc, 'SIGTERM', 4000), + ]); + await weedExitPromise; + stopWeedStdoutForward(); + stopWeedStderrForward(); + })(); + return shutdownPromise; + }; + + process.once('SIGINT', () => { + void shutdown('SIGINT').finally(() => exitOnce(130)); + }); + process.once('SIGTERM', () => { + void shutdown('SIGTERM').finally(() => exitOnce(143)); + }); + + try { + const shouldRunDbMigrations = isTrue(runtimeEnv.RUN_DB_MIGRATIONS, true); + if (shouldRunDbMigrations) { + runDbMigrations(runtimeEnv); + } + + if (useEmbeddedWeed) { + runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, 'docstore/seaweedfs'); + runtimeEnv.WEED_MINI_WAIT_SEC = withDefault(runtimeEnv.WEED_MINI_WAIT_SEC, '20'); + runtimeEnv.S3_BUCKET = withDefault(runtimeEnv.S3_BUCKET, 'openreader-documents'); + runtimeEnv.S3_REGION = withDefault(runtimeEnv.S3_REGION, 'us-east-1'); + const configuredBaseUrl = runtimeEnv.BASE_URL?.trim() || ''; + const baseUrlHost = configuredBaseUrl ? parseUrlHost(configuredBaseUrl, 'BASE_URL') : ''; + const configuredS3Endpoint = runtimeEnv.S3_ENDPOINT?.trim() || ''; + const defaultS3Host = baseUrlHost || detectHostForDefaultEndpoint(); + runtimeEnv.S3_ENDPOINT = configuredS3Endpoint || `http://${defaultS3Host}:8333`; + runtimeEnv.S3_FORCE_PATH_STYLE = withDefault(runtimeEnv.S3_FORCE_PATH_STYLE, 'true'); + runtimeEnv.S3_PREFIX = withDefault(runtimeEnv.S3_PREFIX, 'openreader'); + runtimeEnv.S3_ACCESS_KEY_ID = withDefault(runtimeEnv.S3_ACCESS_KEY_ID, randomBytes(16).toString('hex')); + runtimeEnv.S3_SECRET_ACCESS_KEY = withDefault(runtimeEnv.S3_SECRET_ACCESS_KEY, randomBytes(32).toString('hex')); + runtimeEnv.AWS_ACCESS_KEY_ID = runtimeEnv.S3_ACCESS_KEY_ID; + runtimeEnv.AWS_SECRET_ACCESS_KEY = runtimeEnv.S3_SECRET_ACCESS_KEY; + + if (!runtimeEnv.S3_ACCESS_KEY_ID || !runtimeEnv.S3_SECRET_ACCESS_KEY) { + throw new Error('Failed to initialize embedded S3 credentials.'); + } + + fs.mkdirSync(runtimeEnv.WEED_MINI_DIR, { recursive: true }); + + console.log('Starting embedded SeaweedFS weed mini...'); + const weedArgs = ['mini', `-dir=${runtimeEnv.WEED_MINI_DIR}`]; + if (configuredS3Endpoint || baseUrlHost) { + const endpoint = parseS3Endpoint(runtimeEnv.S3_ENDPOINT); + weedArgs.push(`-ip=${endpoint.hostname}`); + weedArgs.push(`-s3.port=${endpoint.port}`); + } + + weedProc = spawn('weed', weedArgs, { + env: runtimeEnv, + stdio: ['ignore', 'pipe', 'pipe'], + }); + stopWeedStdoutForward = forwardChildStream(weedProc.stdout, process.stdout); + stopWeedStderrForward = forwardChildStream(weedProc.stderr, process.stderr); + weedExitPromise = once(weedProc, 'exit').then(() => undefined).catch(() => undefined); + + weedProc.on('exit', (code, signal) => { + if (typeof code === 'number' && code !== 0) { + console.error(`Embedded weed mini exited with code ${code}.`); + return; + } + if (signal) { + console.error(`Embedded weed mini exited due to signal ${signal}.`); + } + }); + + const endpoint = runtimeEnv.S3_ENDPOINT; + const waitSec = Number.parseInt(runtimeEnv.WEED_MINI_WAIT_SEC || '20', 10); + await waitForEndpoint(endpoint, Number.isFinite(waitSec) ? waitSec : 20); + console.log(`Embedded SeaweedFS is ready at ${endpoint}`); + } + + const { child, exitPromise } = spawnMainCommand(command, runtimeEnv); + appProc = child; + const exitCode = await exitPromise; + + await shutdown('SIGTERM'); + exitOnce(exitCode); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + await shutdown('SIGTERM'); + exitOnce(1); + } +} + +await main(); diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 4da59ee..f52924d 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -2,34 +2,18 @@ import { NextRequest, NextResponse } from 'next/server'; import { createReadStream, existsSync } from 'fs'; import { readdir, unlink } from 'fs/promises'; import { join } from 'path'; -import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore'; +import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore'; import { findStoredChapterByIndex } from '@/lib/server/audiobook'; import { db } from '@/db'; import { audiobooks, audiobookChapters } from '@/db/schema'; import { and, eq, inArray } from 'drizzle-orm'; import { requireAuthContext } from '@/lib/server/auth'; import { ensureDbIndexed } from '@/lib/server/db-indexing'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; import { pruneAudiobookChapterIfMissingFile, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune'; export const dynamic = 'force-dynamic'; -/** - * Get the base audiobooks directory, accounting for test namespaces. - * When auth is disabled, returns AUDIOBOOKS_V1_DIR. - * When auth is enabled, returns the user-specific directory. - */ -function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string { - const namespace = getOpenReaderTestNamespace(request.headers); - - if (!authEnabled || !userId) { - return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace); - } - - const userDir = getUserAudiobookDir(userId); - return applyOpenReaderTestNamespacePath(userDir, namespace); -} - export async function GET(request: NextRequest) { try { const bookId = request.nextUrl.searchParams.get('bookId'); @@ -77,7 +61,14 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Book not found' }, { status: 404 }); } - const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`); + const intermediateDir = join( + getAudiobooksRootDir({ + userId: existingBook.userId, + authEnabled, + namespace: testNamespace, + }), + `${bookId}-audiobook`, + ); const dirExists = existsSync(intermediateDir); if (!dirExists) { await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false); @@ -185,7 +176,14 @@ export async function DELETE(request: NextRequest) { ), ); - const intermediateDir = join(getAudiobooksRootDir(request, storageUserId, authEnabled), `${bookId}-audiobook`); + const intermediateDir = join( + getAudiobooksRootDir({ + userId: storageUserId, + authEnabled, + namespace: testNamespace, + }), + `${bookId}-audiobook`, + ); const chapterPrefix = `${String(chapterIndex + 1).padStart(4, '0')}__`; const files = await readdir(intermediateDir).catch(() => []); for (const file of files) { diff --git a/src/app/api/audiobook/route.ts b/src/app/api/audiobook/route.ts index 254f704..7ed21d1 100644 --- a/src/app/api/audiobook/route.ts +++ b/src/app/api/audiobook/route.ts @@ -4,7 +4,7 @@ import { readFile, writeFile, mkdir, unlink, rm, rename, readdir } from 'fs/prom import { existsSync, createReadStream } from 'fs'; import { basename, join } from 'path'; import { randomUUID } from 'crypto'; -import { AUDIOBOOKS_V1_DIR, ensureAudiobooksV1Ready, isAudiobooksV1Ready, getUserAudiobookDir } from '@/lib/server/docstore'; +import { ensureAudiobooksV1Ready, isAudiobooksV1Ready, getAudiobooksRootDir } from '@/lib/server/docstore'; import { encodeChapterFileName, encodeChapterTitleTag, listStoredChapters, ffprobeAudio, escapeFFMetadata } from '@/lib/server/audiobook'; import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts'; import type { AudiobookGenerationSettings } from '@/types/client'; @@ -13,35 +13,11 @@ import { audiobooks, audiobookChapters } from '@/db/schema'; import { eq, and, inArray } from 'drizzle-orm'; import { requireAuthContext } from '@/lib/server/auth'; import { ensureDbIndexed } from '@/lib/server/db-indexing'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; import { pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune'; export const dynamic = 'force-dynamic'; -/** - * Apply test namespace to a directory path if present in request headers. - */ -function applyTestNamespace(baseDir: string, request: NextRequest): string { - const namespace = getOpenReaderTestNamespace(request.headers); - return applyOpenReaderTestNamespacePath(baseDir, namespace); -} - -/** - * Get the base audiobooks directory, accounting for test namespaces. - * When auth is disabled, returns AUDIOBOOKS_V1_DIR (possibly with test namespace). - * When auth is enabled, returns the user-specific directory under AUDIOBOOKS_USERS_DIR. - */ -function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string { - // When auth is disabled, use the flat audiobooks_v1 directory - if (!authEnabled || !userId) { - return applyTestNamespace(AUDIOBOOKS_V1_DIR, request); - } - - // When auth is enabled, use user-specific directory - const userDir = getUserAudiobookDir(userId); - return applyTestNamespace(userDir, request); -} - interface ConversionRequest { chapterTitle: string; buffer: TTSAudioBytes; @@ -207,7 +183,14 @@ export async function POST(request: NextRequest) { }) .onConflictDoNothing(); - const intermediateDir = join(getAudiobooksRootDir(request, userId, ctxOrRes.authEnabled), `${bookId}-audiobook`); + const intermediateDir = join( + getAudiobooksRootDir({ + userId, + authEnabled: ctxOrRes.authEnabled, + namespace: testNamespace, + }), + `${bookId}-audiobook`, + ); // Create intermediate directory await mkdir(intermediateDir, { recursive: true }); @@ -434,7 +417,11 @@ export async function GET(request: NextRequest) { } const intermediateDir = join( - getAudiobooksRootDir(request, existingBook.userId, authEnabled), + getAudiobooksRootDir({ + userId: existingBook.userId, + authEnabled, + namespace: testNamespace, + }), `${bookId}-audiobook`, ); @@ -657,7 +644,14 @@ export async function DELETE(request: NextRequest) { await db.delete(audiobooks).where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId))); - const intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`); + const intermediateDir = join( + getAudiobooksRootDir({ + userId, + authEnabled, + namespace: testNamespace, + }), + `${bookId}-audiobook`, + ); // If directory doesn't exist, consider it already reset if (!existsSync(intermediateDir)) { diff --git a/src/app/api/audiobook/status/route.ts b/src/app/api/audiobook/status/route.ts index 78d558a..1f502ca 100644 --- a/src/app/api/audiobook/status/route.ts +++ b/src/app/api/audiobook/status/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { existsSync } from 'fs'; import { join } from 'path'; -import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore'; +import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore'; import { listStoredChapters } from '@/lib/server/audiobook'; import type { AudiobookGenerationSettings } from '@/types/client'; import type { TTSAudiobookFormat, TTSAudiobookChapter } from '@/types/tts'; @@ -11,27 +11,11 @@ import { db } from '@/db'; import { audiobooks } from '@/db/schema'; import { eq, and, inArray } from 'drizzle-orm'; import { ensureDbIndexed } from '@/lib/server/db-indexing'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; import { pruneAudiobookChaptersNotOnDisk, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune'; export const dynamic = 'force-dynamic'; -/** - * Get the base audiobooks directory, accounting for test namespaces. - * When auth is disabled, returns AUDIOBOOKS_V1_DIR. - * When auth is enabled, returns the user-specific directory. - */ -function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string { - const namespace = getOpenReaderTestNamespace(request.headers); - - if (!authEnabled || !userId) { - return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace); - } - - const userDir = getUserAudiobookDir(userId); - return applyOpenReaderTestNamespacePath(userDir, namespace); -} - const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; function isSafeId(value: string): boolean { @@ -80,7 +64,14 @@ export async function GET(request: NextRequest) { }); } - const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`); + const intermediateDir = join( + getAudiobooksRootDir({ + userId: existingBook.userId, + authEnabled, + namespace: testNamespace, + }), + `${bookId}-audiobook`, + ); if (!existsSync(intermediateDir)) { await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false); diff --git a/src/app/api/documents/content/route.ts b/src/app/api/documents/blob/route.ts similarity index 55% rename from src/app/api/documents/content/route.ts rename to src/app/api/documents/blob/route.ts index ed6c0fe..04c183d 100644 --- a/src/app/api/documents/content/route.ts +++ b/src/app/api/documents/blob/route.ts @@ -1,16 +1,13 @@ -import { open, readFile } from 'fs/promises'; -import path from 'path'; import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; -import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore'; import { requireAuthContext } from '@/lib/server/auth'; -import { ensureDbIndexed } from '@/lib/server/db-indexing'; import { contentTypeForName } from '@/lib/server/library'; import { extractRawTextSnippet } from '@/lib/text-snippets'; -import { isEnoent } from '@/lib/server/documents-utils'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { getDocumentBlob, getDocumentRange, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; export const dynamic = 'force-dynamic'; @@ -19,67 +16,55 @@ function clampInt(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, Math.trunc(value))); } -async function readHeadBuffer(filePath: string, maxBytes: number): Promise { - const handle = await open(filePath, 'r'); - try { - const buf = Buffer.allocUnsafe(maxBytes); - const result = await handle.read(buf, 0, maxBytes, 0); - return buf.subarray(0, result.bytesRead); - } finally { - await handle.close(); - } +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); } export async function GET(req: NextRequest) { try { - await ensureDocumentsV1Ready(); - if (!(await isDocumentsV1Ready())) { - return NextResponse.json( - { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, - { status: 409 }, - ); - } + if (!isS3Configured()) return s3NotConfiguredResponse(); const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; const testNamespace = getOpenReaderTestNamespace(req.headers); - const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); - const storageUserId = ctxOrRes.userId ?? unclaimedUserId; const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; - await ensureDbIndexed(); - const url = new URL(req.url); - const id = url.searchParams.get('id'); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); const format = (url.searchParams.get('format') || '').toLowerCase().trim(); - if (!id) { - return NextResponse.json({ error: 'Missing id' }, { status: 400 }); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid id' }, { status: 400 }); } - const docs = await db + const rows = (await db .select({ id: documents.id, userId: documents.userId, name: documents.name, filePath: documents.filePath }) .from(documents) - .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); + .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ + id: string; + userId: string; + name: string; + filePath: string; + }>; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const doc = docs.find((d: any) => d.userId === storageUserId) ?? docs[0]; - - if (!doc?.filePath) { + const doc = rows.find((row) => row.userId === storageUserId) ?? rows[0]; + if (!doc) { return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - const filePath = path.join(documentsDir, doc.filePath); - const filename = doc.name || doc.filePath; + const filename = doc.name || `${id}.bin`; + const responseType = contentTypeForName(filename); if (format === 'snippet') { const maxChars = clampInt(Number.parseInt(url.searchParams.get('maxChars') || '1600', 10), 100, 8000); const maxBytes = clampInt(Number.parseInt(url.searchParams.get('maxBytes') || '131072', 10), 4096, 1024 * 1024); - try { - const head = await readHeadBuffer(filePath, maxBytes); + const head = await getDocumentRange(id, 0, maxBytes - 1, testNamespace); const decoded = new TextDecoder().decode(new Uint8Array(head)); const snippet = extractRawTextSnippet(decoded, maxChars); return NextResponse.json( @@ -87,7 +72,7 @@ export async function GET(req: NextRequest) { { headers: { 'Cache-Control': 'no-store' } }, ); } catch (error) { - if (isEnoent(error)) { + if (isMissingBlobError(error)) { await db .delete(documents) .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); @@ -97,14 +82,22 @@ export async function GET(req: NextRequest) { } } - let content: ArrayBuffer; try { - const buf = await readFile(filePath); - content = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + const content = await getDocumentBlob(id, testNamespace); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(content)); + controller.close(); + }, + }); + return new NextResponse(stream, { + headers: { + 'Content-Type': responseType, + 'Cache-Control': 'no-store', + }, + }); } catch (error) { - if (isEnoent(error)) { - // The DB can become stale if a file is deleted manually from the docstore. - // Prune rows so the client stops showing ghost documents. + if (isMissingBlobError(error)) { await db .delete(documents) .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); @@ -112,13 +105,6 @@ export async function GET(req: NextRequest) { } throw error; } - - return new NextResponse(content, { - headers: { - 'Content-Type': contentTypeForName(filename), - 'Cache-Control': 'no-store', - }, - }); } catch (error) { console.error('Error loading document content:', error); return NextResponse.json({ error: 'Failed to load document content' }, { status: 500 }); diff --git a/src/app/api/documents/blob/upload/fallback/route.ts b/src/app/api/documents/blob/upload/fallback/route.ts new file mode 100644 index 0000000..16ada73 --- /dev/null +++ b/src/app/api/documents/blob/upload/fallback/route.ts @@ -0,0 +1,51 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAuthContext } from '@/lib/server/auth'; +import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore'; +import { isS3Configured } from '@/lib/server/s3'; +import { getOpenReaderTestNamespace } from '@/lib/server/test-namespace'; + +export const dynamic = 'force-dynamic'; + +function isPreconditionFailed(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } }; + return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed'; +} + +export async function PUT(req: NextRequest) { + try { + if (!isS3Configured()) { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); + } + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const url = new URL(req.url); + const id = (url.searchParams.get('id') || '').trim().toLowerCase(); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid document id' }, { status: 400 }); + } + + const contentType = (req.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream'; + const body = Buffer.from(await req.arrayBuffer()); + const namespace = getOpenReaderTestNamespace(req.headers); + + try { + await putDocumentBlob(id, body, contentType, namespace); + } catch (error) { + if (!isPreconditionFailed(error)) { + throw error; + } + } + + return NextResponse.json({ success: true, id }); + } catch (error) { + console.error('Error proxy-uploading document blob:', error); + return NextResponse.json({ error: 'Failed to upload document blob' }, { status: 500 }); + } +} + diff --git a/src/app/api/documents/blob/upload/presign/route.ts b/src/app/api/documents/blob/upload/presign/route.ts new file mode 100644 index 0000000..c1c803a --- /dev/null +++ b/src/app/api/documents/blob/upload/presign/route.ts @@ -0,0 +1,72 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAuthContext } from '@/lib/server/auth'; +import { isValidDocumentId, presignPut } from '@/lib/server/documents-blobstore'; +import { getOpenReaderTestNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; + +export const dynamic = 'force-dynamic'; + +type PresignUpload = { + id: string; + contentType: string; + size: number; +}; + +function parseUploads(body: unknown): PresignUpload[] { + if (!body || typeof body !== 'object') return []; + const rawUploads = (body as { uploads?: unknown }).uploads; + if (!Array.isArray(rawUploads)) return []; + + const uploads: PresignUpload[] = []; + for (const raw of rawUploads) { + if (!raw || typeof raw !== 'object') continue; + const rec = raw as Record; + const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : ''; + if (!isValidDocumentId(id)) continue; + const contentType = + typeof rec.contentType === 'string' && rec.contentType.trim() + ? rec.contentType.trim() + : 'application/octet-stream'; + const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0; + uploads.push({ id, contentType, size }); + } + return uploads; +} + +export async function POST(req: NextRequest) { + try { + if (!isS3Configured()) { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); + } + + const ctxOrRes = await requireAuthContext(req); + if (ctxOrRes instanceof Response) return ctxOrRes; + + const body = await req.json().catch(() => null); + const uploads = parseUploads(body); + if (uploads.length === 0) { + return NextResponse.json({ error: 'No valid uploads provided' }, { status: 400 }); + } + + const namespace = getOpenReaderTestNamespace(req.headers); + const signed = await Promise.all( + uploads.map(async (upload) => { + const res = await presignPut(upload.id, upload.contentType, namespace); + return { + id: upload.id, + url: res.url, + headers: res.headers, + }; + }), + ); + + return NextResponse.json({ uploads: signed }); + } catch (error) { + console.error('Error creating document upload signatures:', error); + return NextResponse.json({ error: 'Failed to presign uploads' }, { status: 500 }); + } +} + diff --git a/src/app/api/documents/docx-to-pdf/upload/route.ts b/src/app/api/documents/docx-to-pdf/upload/route.ts index 2075b90..0b4a3bd 100644 --- a/src/app/api/documents/docx-to-pdf/upload/route.ts +++ b/src/app/api/documents/docx-to-pdf/upload/route.ts @@ -6,11 +6,12 @@ import { existsSync } from 'fs'; import { randomUUID, createHash } from 'crypto'; import { pathToFileURL } from 'url'; import { requireAuthContext } from '@/lib/server/auth'; -import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore'; import { db } from '@/db'; import { documents } from '@/db/schema'; -import { safeDocumentName, trySetFileMtime } from '@/lib/server/documents-utils'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { safeDocumentName } from '@/lib/server/documents-utils'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; +import { putDocumentBlob } from '@/lib/server/documents-blobstore'; const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp'); @@ -66,18 +67,15 @@ async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100) export async function POST(req: NextRequest) { try { - await ensureDocumentsV1Ready(); - if (!(await isDocumentsV1Ready())) { + if (!isS3Configured()) { return NextResponse.json( - { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, - { status: 409 }, + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, ); } const testNamespace = getOpenReaderTestNamespace(req.headers); - const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); - await mkdir(documentsDir, { recursive: true }); const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; @@ -96,7 +94,7 @@ export async function POST(req: NextRequest) { } const docxBytes = Buffer.from(await file.arrayBuffer()); - // IMPORTANT: use sha of the source DOCX bytes for a stable ID across conversions. + // Keep stable IDs tied to source bytes. const id = createHash('sha256').update(docxBytes).digest('hex'); const tempId = randomUUID(); @@ -113,18 +111,19 @@ export async function POST(req: NextRequest) { const pdfPath = await waitForPdfReady(jobDir); const pdfContent = await readFile(pdfPath); - const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`); - const targetFileName = `${id}__${encodeURIComponent(derivedName)}`; - const targetPath = path.join(documentsDir, targetFileName); - try { - await stat(targetPath); - } catch { - await writeFile(targetPath, pdfContent); + await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace); + } catch (error) { + // Idempotent behavior: if blob already exists for this sha, continue. + const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; + const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed'; + if (!isPreconditionFailed) { + throw error; + } } + const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`); const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now(); - await trySetFileMtime(targetPath, lastModified); await db .insert(documents) @@ -135,9 +134,18 @@ export async function POST(req: NextRequest) { type: 'pdf', size: pdfContent.length, lastModified, - filePath: targetFileName, + filePath: id, }) - .onConflictDoNothing(); + .onConflictDoUpdate({ + target: [documents.id, documents.userId], + set: { + name: derivedName, + type: 'pdf', + size: pdfContent.length, + lastModified, + filePath: id, + }, + }); return NextResponse.json({ stored: { diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 5d3d7ef..023afdf 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,212 +1,207 @@ -import { createHash } from 'crypto'; -import { mkdir, readFile, stat, unlink, writeFile } from 'fs/promises'; -import { existsSync } from 'fs'; import { NextRequest, NextResponse } from 'next/server'; -import path from 'path'; -import { DOCUMENTS_V1_DIR, ensureDocumentsV1Ready, isDocumentsV1Ready } from '@/lib/server/docstore'; -import type { BaseDocument, DocumentType, SyncedDocument } from '@/types/documents'; +import { and, count, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; -import { eq, and, inArray, count } from 'drizzle-orm'; import { requireAuthContext } from '@/lib/server/auth'; -import { ensureDbIndexed } from '@/lib/server/db-indexing'; -import { isEnoent, toDocumentTypeFromName, trySetFileMtime } from '@/lib/server/documents-utils'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents-utils'; +import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents-blobstore'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; +import { isS3Configured } from '@/lib/server/s3'; +import type { BaseDocument, DocumentType } from '@/types/documents'; export const dynamic = 'force-dynamic'; +type RegisterDocument = { + id: string; + name: string; + type: DocumentType; + size: number; + lastModified: number; +}; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function normalizeDocumentType(rawType: unknown, safeName: string): DocumentType { + if (rawType === 'pdf' || rawType === 'epub' || rawType === 'docx' || rawType === 'html') { + return rawType; + } + return toDocumentTypeFromName(safeName); +} + +function normalizeLastModified(value: unknown): number { + return Number.isFinite(value) && Number(value) > 0 ? Number(value) : Date.now(); +} + +function parseDocumentPayload(body: unknown): RegisterDocument[] { + if (!body || typeof body !== 'object') return []; + const rawDocs = (body as { documents?: unknown }).documents; + if (!Array.isArray(rawDocs)) return []; + + const docs: RegisterDocument[] = []; + for (const rawDoc of rawDocs) { + if (!rawDoc || typeof rawDoc !== 'object') continue; + const rec = rawDoc as Record; + const id = typeof rec.id === 'string' ? rec.id.trim().toLowerCase() : ''; + if (!isValidDocumentId(id)) continue; + const fallbackName = `${id}.${typeof rec.type === 'string' ? rec.type : 'txt'}`; + const name = safeDocumentName(typeof rec.name === 'string' ? rec.name : '', fallbackName); + const type = normalizeDocumentType(rec.type, name); + const lastModified = normalizeLastModified(rec.lastModified); + const size = Number.isFinite(rec.size) && Number(rec.size) >= 0 ? Number(rec.size) : 0; + docs.push({ id, name, type, size, lastModified }); + } + return docs; +} + export async function POST(req: NextRequest) { try { - await ensureDocumentsV1Ready(); - if (!(await isDocumentsV1Ready())) { - return NextResponse.json( - { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, - { status: 409 }, - ); - } - - const testNamespace = getOpenReaderTestNamespace(req.headers); - const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); - const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); - await mkdir(syncDir, { recursive: true }); + if (!isS3Configured()) return s3NotConfiguredResponse(); const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); const storageUserId = ctxOrRes.userId ?? unclaimedUserId; - const data = await req.json(); - const documentsData = data.documents as SyncedDocument[]; - const stored: Array<{ oldId: string; id: string; name: string }> = []; + const body = await req.json().catch(() => null); + const documentsData = parseDocumentPayload(body); + if (documentsData.length === 0) { + return NextResponse.json({ error: 'No valid documents provided' }, { status: 400 }); + } - // Ensure directory exists (redundant with isDocumentsV1Ready but safe) - // const SYNC_DIR = DOCUMENTS_V1_DIR; + const stored: BaseDocument[] = []; for (const doc of documentsData) { - const content = Buffer.from(new Uint8Array(doc.data)); - const id = createHash('sha256').update(content).digest('hex'); - - const baseName = path.basename(doc.name || `${id}.${doc.type}`); - const safeName = baseName.replaceAll('\u0000', '').slice(0, 240) || `${id}.${doc.type}`; - - const targetFileName = `${id}__${encodeURIComponent(safeName)}`; - const targetPath = path.join(syncDir, targetFileName); - - // Write file if not exists + let headSize = doc.size; try { - await stat(targetPath); - } catch { - await writeFile(targetPath, content); + const head = await headDocumentBlob(doc.id, testNamespace); + if (head.contentLength > 0) headSize = head.contentLength; + } catch (error) { + if (isMissingBlobError(error)) { + return NextResponse.json( + { + error: `Blob missing for document ${doc.id}. Upload bytes first using /api/documents/blob/upload/presign.`, + }, + { status: 409 }, + ); + } + throw error; } - await trySetFileMtime(targetPath, doc.lastModified); - // DB Upsert - // With composite PK (id, userId), we check if THIS user already has this document await db .insert(documents) .values({ - id, + id: doc.id, userId: storageUserId, - name: safeName, + name: doc.name, type: doc.type, - size: content.length, + size: headSize, lastModified: doc.lastModified, - filePath: targetFileName, + filePath: doc.id, }) - .onConflictDoNothing(); + .onConflictDoUpdate({ + target: [documents.id, documents.userId], + set: { + name: doc.name, + type: doc.type, + size: headSize, + lastModified: doc.lastModified, + filePath: doc.id, + }, + }); - stored.push({ oldId: doc.id, id, name: safeName }); + stored.push({ + id: doc.id, + name: doc.name, + type: doc.type, + size: headSize, + lastModified: doc.lastModified, + scope: storageUserId === unclaimedUserId ? 'unclaimed' : 'user', + }); } return NextResponse.json({ success: true, stored }); } catch (error) { - console.error('Error saving documents:', error); - return NextResponse.json({ error: 'Failed to save documents' }, { status: 500 }); + console.error('Error registering documents:', error); + return NextResponse.json({ error: 'Failed to register documents' }, { status: 500 }); } } export async function GET(req: NextRequest) { try { - await ensureDocumentsV1Ready(); - if (!(await isDocumentsV1Ready())) { - return NextResponse.json( - { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, - { status: 409 }, - ); - } + if (!isS3Configured()) return s3NotConfiguredResponse(); const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; const testNamespace = getOpenReaderTestNamespace(req.headers); - const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); - const storageUserId = ctxOrRes.userId ?? unclaimedUserId; const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; - await ensureDbIndexed(); - const url = new URL(req.url); - const list = url.searchParams.get('list') === 'true'; - const format = url.searchParams.get('format'); const idsParam = url.searchParams.get('ids'); + const targetIds = idsParam + ? idsParam + .split(',') + .map((id) => id.trim().toLowerCase()) + .filter((id) => isValidDocumentId(id)) + : null; - // If list=true, force metadata only. - // If format=metadata, force metadata only. - // Otherwise include data. - const includeData = !list && format !== 'metadata'; - - const targetIds = idsParam ? idsParam.split(',').filter(Boolean) : null; - - // Query database for documents the user is allowed to access - let allowedDocs: { id: string; userId: string; name: string; type: string; size: number; lastModified: number; filePath: string }[] = []; + if (idsParam && (!targetIds || targetIds.length === 0)) { + return NextResponse.json({ documents: [] }); + } const conditions = [ inArray(documents.userId, allowedUserIds), - ...(targetIds ? [inArray(documents.id, targetIds)] : []), + ...(targetIds && targetIds.length > 0 ? [inArray(documents.id, targetIds)] : []), ]; - const rows = await db.select().from(documents).where(and(...conditions)); - allowedDocs = rows as unknown as { id: string; userId: string; name: string; type: string; size: number; lastModified: number; filePath: string }[]; + const rows = (await db.select().from(documents).where(and(...conditions))) as Array<{ + id: string; + userId: string; + name: string; + type: string; + size: number; + lastModified: number; + filePath: string; + }>; - const results: (BaseDocument | SyncedDocument)[] = []; - - for (const doc of allowedDocs) { - const type: DocumentType = - doc.type === 'pdf' || doc.type === 'epub' || doc.type === 'docx' || doc.type === 'html' - ? (doc.type as DocumentType) - : toDocumentTypeFromName(doc.name); - - // If the underlying file was deleted manually, keep the API self-healing: - // prune the DB row so clients stop listing ghost documents. - const absolutePath = doc.filePath ? path.join(syncDir, doc.filePath) : ''; - if (!absolutePath || !existsSync(absolutePath)) { - await db - .delete(documents) - .where(and(eq(documents.id, doc.id), eq(documents.userId, doc.userId))); - continue; - } - - const metadata: BaseDocument = { - id: doc.id!, + const results: BaseDocument[] = rows.map((doc) => { + const type = normalizeDocumentType(doc.type, doc.name); + return { + id: doc.id, name: doc.name, - size: doc.size, - lastModified: doc.lastModified, + size: Number(doc.size), + lastModified: Number(doc.lastModified), type, scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user', }; - - if (!includeData) { - results.push(metadata); - continue; - } - - try { - const content = await readFile(absolutePath); - results.push({ - ...metadata, - data: Array.from(new Uint8Array(content)), - }); - } catch (err) { - if (isEnoent(err)) { - await db - .delete(documents) - .where(and(eq(documents.id, doc.id), eq(documents.userId, doc.userId))); - continue; - } - console.warn(`Failed to read content for document ${doc.id} at ${doc.filePath}`, err); - } - } + }); return NextResponse.json({ documents: results }); } catch (error) { - console.error('Error loading documents:', error); + console.error('Error loading document metadata:', error); return NextResponse.json({ error: 'Failed to load documents' }, { status: 500 }); } } - export async function DELETE(req: NextRequest) { try { - await ensureDocumentsV1Ready(); - if (!(await isDocumentsV1Ready())) { - return NextResponse.json( - { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, - { status: 409 }, - ); - } + if (!isS3Configured()) return s3NotConfiguredResponse(); - // Auth check - require session const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; const testNamespace = getOpenReaderTestNamespace(req.headers); - const syncDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); - const storageUserId = ctxOrRes.userId ?? unclaimedUserId; - await ensureDbIndexed(); - const url = new URL(req.url); const idsParam = url.searchParams.get('ids'); const scopeParam = (url.searchParams.get('scope') || '').toLowerCase().trim(); @@ -221,7 +216,6 @@ export async function DELETE(req: NextRequest) { ); } - // Deleting the global unclaimed pool is a privileged operation when auth is enabled. if (ctxOrRes.authEnabled && wantsUnclaimed && ctxOrRes.user?.isAnonymous) { return NextResponse.json({ error: 'Forbidden' }, { status: 403 }); } @@ -239,58 +233,45 @@ export async function DELETE(req: NextRequest) { return NextResponse.json({ success: true, deleted: 0 }); } - // Determine which IDs to try to delete let targetIds: string[] = []; - if (idsParam) { - targetIds = idsParam.split(',').filter(Boolean); + targetIds = idsParam + .split(',') + .map((id) => id.trim().toLowerCase()) + .filter((id) => isValidDocumentId(id)); } else { - // Existing behavior was "nuke everything"; keep it scoped to the selected user buckets. - const rows = await db + const rows = (await db .select({ id: documents.id }) .from(documents) - .where(inArray(documents.userId, targetUserIds)); - targetIds = rows.map((d: { id: string | null }) => d.id!).filter(Boolean) as string[]; + .where(inArray(documents.userId, targetUserIds))) as Array<{ id: string }>; + targetIds = rows.map((row) => row.id); } if (targetIds.length === 0) { return NextResponse.json({ success: true, deleted: 0 }); } - const deletedRows: { id: string; filePath: string }[] = []; - - const rows = await db + const deletedRows = (await db .delete(documents) .where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds))) - .returning({ id: documents.id, filePath: documents.filePath }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - rows.forEach((r: any) => deletedRows.push({ id: r.id!, filePath: r.filePath })); + .returning({ id: documents.id })) as Array<{ id: string }>; - // If driver doesn't support returning (e.g. older SQLite without properly configured returning), we might fallback. - // But Drizzle usually handles this. + const uniqueIds = Array.from(new Set(deletedRows.map((row) => row.id))); + for (const id of uniqueIds) { + const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, id)); + const refCount = Number(ref?.count ?? 0); + if (refCount > 0) continue; - let deletedCount = 0; - - for (const row of deletedRows) { - deletedCount++; - // Chech reference count for this ID - // If 0 remaining, delete file - let refCount = 0; - const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, row.id!)); - refCount = Number(ref?.count ?? 0); - - if (refCount === 0) { - const filePath = path.join(syncDir, row.filePath); - try { - await unlink(filePath); - } catch { - // Ignore if missing + try { + await deleteDocumentBlob(id, testNamespace); + } catch (error) { + if (!isMissingBlobError(error)) { + throw error; } } } - return NextResponse.json({ success: true, deleted: deletedCount }); - + return NextResponse.json({ success: true, deleted: deletedRows.length }); } catch (error) { console.error('Error deleting documents:', error); return NextResponse.json({ error: 'Failed to delete documents' }, { status: 500 }); diff --git a/src/app/api/documents/upload/route.ts b/src/app/api/documents/upload/route.ts deleted file mode 100644 index 4491a33..0000000 --- a/src/app/api/documents/upload/route.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { createHash } from 'crypto'; -import { mkdir, stat, writeFile } from 'fs/promises'; -import path from 'path'; -import { NextRequest, NextResponse } from 'next/server'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; -import { ensureDocumentsV1Ready, isDocumentsV1Ready, DOCUMENTS_V1_DIR } from '@/lib/server/docstore'; -import { requireAuthContext } from '@/lib/server/auth'; -import { safeDocumentName, toDocumentTypeFromName, trySetFileMtime } from '@/lib/server/documents-utils'; -import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; - -export const dynamic = 'force-dynamic'; - -export async function POST(req: NextRequest) { - try { - await ensureDocumentsV1Ready(); - if (!(await isDocumentsV1Ready())) { - return NextResponse.json( - { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, - { status: 409 }, - ); - } - - const testNamespace = getOpenReaderTestNamespace(req.headers); - const documentsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); - const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); - await mkdir(documentsDir, { recursive: true }); - - const ctxOrRes = await requireAuthContext(req); - if (ctxOrRes instanceof Response) return ctxOrRes; - const storageUserId = ctxOrRes.userId ?? unclaimedUserId; - - const form = await req.formData(); - const files = form.getAll('files').filter((value): value is File => value instanceof File); - - if (files.length === 0) { - return NextResponse.json({ error: 'Missing files' }, { status: 400 }); - } - - const stored: Array<{ - id: string; - name: string; - type: 'pdf' | 'epub' | 'docx' | 'html'; - size: number; - lastModified: number; - }> = []; - - for (const file of files) { - const arrayBuffer = await file.arrayBuffer(); - const content = Buffer.from(new Uint8Array(arrayBuffer)); - const id = createHash('sha256').update(content).digest('hex'); - - const safeName = safeDocumentName(file.name, `${id}.${toDocumentTypeFromName(file.name)}`); - const targetFileName = `${id}__${encodeURIComponent(safeName)}`; - const targetPath = path.join(documentsDir, targetFileName); - - try { - await stat(targetPath); - } catch { - await writeFile(targetPath, content); - } - - const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now(); - await trySetFileMtime(targetPath, lastModified); - - const type = toDocumentTypeFromName(safeName); - - await db - .insert(documents) - .values({ - id, - userId: storageUserId, - name: safeName, - type, - size: content.length, - lastModified, - filePath: targetFileName, - }) - .onConflictDoNothing(); - - stored.push({ - id, - name: safeName, - type, - size: content.length, - lastModified, - }); - } - - return NextResponse.json({ stored }); - } catch (error) { - console.error('Error uploading documents:', error); - return NextResponse.json({ error: 'Failed to upload documents' }, { status: 500 }); - } -} diff --git a/src/app/api/migrations/v2/route.ts b/src/app/api/migrations/v2/route.ts new file mode 100644 index 0000000..d289f03 --- /dev/null +++ b/src/app/api/migrations/v2/route.ts @@ -0,0 +1,275 @@ +import { createHash } from 'crypto'; +import { existsSync } from 'fs'; +import { readdir, readFile, stat, unlink } from 'fs/promises'; +import path from 'path'; +import { and, eq } from 'drizzle-orm'; +import { NextRequest, NextResponse } from 'next/server'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { auth } from '@/lib/server/auth'; +import { DOCUMENTS_V1_DIR } from '@/lib/server/docstore'; +import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore'; +import { toDocumentTypeFromName } from '@/lib/server/documents-utils'; +import { contentTypeForName } from '@/lib/server/library'; +import { isS3Configured } from '@/lib/server/s3'; +import type { DocumentType } from '@/types/documents'; +import { + applyOpenReaderTestNamespacePath, + getOpenReaderTestNamespace, + getUnclaimedUserIdForNamespace, +} from '@/lib/server/test-namespace'; + +export const dynamic = 'force-dynamic'; + +type V2Body = { + deleteLocal?: boolean; + dryRun?: boolean; +}; + +type LegacyDocumentCandidate = { + id: string; + name: string; + type: string; + size: number; + lastModified: number; +}; + +function isPreconditionFailed(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } }; + return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed'; +} + +function extractIdFromFileName(fileName: string): string | null { + const match = /^([a-f0-9]{64})__/i.exec(fileName); + if (!match) return null; + const id = match[1].toLowerCase(); + return isValidDocumentId(id) ? id : null; +} + +function decodeNameFromFileName(fileName: string, id: string): string { + const prefix = `${id}__`; + if (!fileName.startsWith(prefix)) return `${id}.bin`; + const encoded = fileName.slice(prefix.length); + try { + return decodeURIComponent(encoded); + } catch { + return `${id}.bin`; + } +} + +function sniffBinaryDocumentType(bytes: Buffer): Exclude | null { + // PDF signature: "%PDF-" + if (bytes.length >= 5 && bytes.subarray(0, 5).toString('ascii') === '%PDF-') { + return 'pdf'; + } + + // ZIP signatures: PK.. + const isZip = + bytes.length >= 4 && + bytes[0] === 0x50 && + bytes[1] === 0x4b && + (bytes[2] === 0x03 || bytes[2] === 0x05 || bytes[2] === 0x07) && + (bytes[3] === 0x04 || bytes[3] === 0x06 || bytes[3] === 0x08); + if (!isZip) return null; + + // EPUB/DOCX markers usually appear in ZIP local headers near the start. + const probe = bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString('latin1'); + if (probe.includes('application/epub+zip') || probe.includes('META-INF/container.xml')) { + return 'epub'; + } + if (probe.includes('[Content_Types].xml') && probe.includes('word/')) { + return 'docx'; + } + + return null; +} + +function normalizeNameForType(name: string, id: string, type: DocumentType): string { + if (type === 'html') return name; + const expectedExt = type === 'pdf' ? '.pdf' : type === 'epub' ? '.epub' : '.docx'; + if (name.toLowerCase().endsWith(expectedExt)) return name; + const base = name.replace(/\.bin$/i, ''); + return `${base || id}${expectedExt}`; +} + +function contentTypeForDocument(type: DocumentType, name: string): string { + if (type === 'pdf') return 'application/pdf'; + if (type === 'epub') return 'application/epub+zip'; + if (type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + return contentTypeForName(name); +} + +async function migrateDocumentRowsToBlobHandle(dryRun: boolean): Promise { + const rows = (await db.select().from(documents)) as Array<{ + id: string; + userId: string; + filePath: string; + }>; + + let updated = 0; + for (const row of rows) { + if (!row.id || !row.userId || row.filePath === row.id) continue; + updated++; + if (dryRun) continue; + await db + .update(documents) + .set({ filePath: row.id }) + .where(and(eq(documents.id, row.id), eq(documents.userId, row.userId))); + } + return updated; +} + +async function seedMissingDocumentRows( + userId: string, + candidates: LegacyDocumentCandidate[], + dryRun: boolean, +): Promise { + if (candidates.length === 0) return 0; + + const existingRows = (await db.select().from(documents).where(eq(documents.userId, userId))) as Array<{ + id: string; + }>; + const existingIds = new Set(existingRows.map((row) => row.id)); + + const seen = new Set(); + const toInsert: LegacyDocumentCandidate[] = []; + for (const candidate of candidates) { + if (seen.has(candidate.id)) continue; + seen.add(candidate.id); + if (existingIds.has(candidate.id)) continue; + toInsert.push(candidate); + } + + if (toInsert.length === 0) return 0; + if (dryRun) return toInsert.length; + + await db.insert(documents).values( + toInsert.map((candidate) => ({ + id: candidate.id, + userId, + name: candidate.name, + type: candidate.type, + size: candidate.size, + lastModified: candidate.lastModified, + filePath: candidate.id, + })), + ).onConflictDoNothing(); + + return toInsert.length; +} + +export async function POST(request: NextRequest) { + try { + const session = await auth?.api.getSession({ headers: request.headers }); + if (auth && !session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + if (!isS3Configured()) { + return NextResponse.json( + { error: 'S3 is not configured. Set S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.' }, + { status: 409 }, + ); + } + + const raw = (await request.json().catch(() => ({}))) as V2Body; + const dryRun = raw.dryRun === true; + const deleteLocal = raw.deleteLocal === true; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const docsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace); + + if (!existsSync(docsDir)) { + const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun); + return NextResponse.json({ + success: true, + dryRun, + deleteLocal, + docsDir, + filesScanned: 0, + uploaded: 0, + alreadyPresent: 0, + skippedInvalid: 0, + deletedLocal: 0, + dbRowsUpdated: rowsUpdated, + dbRowsSeeded: 0, + }); + } + + const entries = await readdir(docsDir, { withFileTypes: true }); + const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name); + + let uploaded = 0; + let alreadyPresent = 0; + let skippedInvalid = 0; + let deletedLocal = 0; + const candidates: LegacyDocumentCandidate[] = []; + + for (const fileName of files) { + const fullPath = path.join(docsDir, fileName); + const bytes = await readFile(fullPath); + const fileStats = await stat(fullPath); + + const extractedId = extractIdFromFileName(fileName); + const id = extractedId ?? createHash('sha256').update(bytes).digest('hex'); + if (!isValidDocumentId(id)) { + skippedInvalid++; + continue; + } + + const inferredName = decodeNameFromFileName(fileName, id); + const inferredType = toDocumentTypeFromName(inferredName); + const type = inferredType === 'html' ? (sniffBinaryDocumentType(bytes) ?? inferredType) : inferredType; + const normalizedName = normalizeNameForType(inferredName, id, type); + const contentType = contentTypeForDocument(type, normalizedName); + const lastModified = Number.isFinite(fileStats.mtimeMs) ? Math.floor(fileStats.mtimeMs) : Date.now(); + + candidates.push({ + id, + name: normalizedName, + type, + size: bytes.length, + lastModified, + }); + + if (!dryRun) { + try { + await putDocumentBlob(id, bytes, contentType, testNamespace); + uploaded++; + } catch (error) { + if (isPreconditionFailed(error)) { + alreadyPresent++; + } else { + throw error; + } + } + } + + if (deleteLocal && !dryRun) { + await unlink(fullPath).catch(() => {}); + deletedLocal++; + } + } + + const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun); + const rowsSeeded = await seedMissingDocumentRows(unclaimedUserId, candidates, dryRun); + + return NextResponse.json({ + success: true, + dryRun, + deleteLocal, + docsDir, + filesScanned: files.length, + uploaded, + alreadyPresent, + skippedInvalid, + deletedLocal, + dbRowsUpdated: rowsUpdated, + dbRowsSeeded: rowsSeeded, + }); + } catch (error) { + console.error('Error running v2 migrations:', error); + return NextResponse.json({ error: 'Failed to run v2 migrations' }, { status: 500 }); + } +} diff --git a/src/app/api/user/claim/route.ts b/src/app/api/user/claim/route.ts index 1011282..592cbfa 100644 --- a/src/app/api/user/claim/route.ts +++ b/src/app/api/user/claim/route.ts @@ -2,6 +2,34 @@ import { NextRequest, NextResponse } from 'next/server'; import { claimAnonymousData } from '@/lib/server/claim-data'; import { auth } from '@/lib/server/auth'; import { ensureDbIndexed, getUnclaimedCounts } from '@/lib/server/db-indexing'; +import { isDocumentsV1Ready } from '@/lib/server/docstore'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { count, ne } from 'drizzle-orm'; + +async function checkClaimMigrationReadiness(): Promise { + const documentsV1Ready = await isDocumentsV1Ready(); + if (!documentsV1Ready) { + return NextResponse.json( + { error: 'Document migration is not ready. Run startup migrations first.' }, + { status: 409 }, + ); + } + + const [legacyRows] = await db + .select({ count: count() }) + .from(documents) + .where(ne(documents.filePath, documents.id)); + + if (Number(legacyRows?.count ?? 0) > 0) { + return NextResponse.json( + { error: 'Document metadata migration is still pending. Wait for startup migrations to complete.' }, + { status: 409 }, + ); + } + + return null; +} export async function GET(req: NextRequest) { try { @@ -10,6 +38,9 @@ export async function GET(req: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + const readiness = await checkClaimMigrationReadiness(); + if (readiness) return readiness; + await ensureDbIndexed(); const counts = await getUnclaimedCounts(); return NextResponse.json({ success: true, ...counts }); @@ -26,6 +57,9 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + const readiness = await checkClaimMigrationReadiness(); + if (readiness) return readiness; + const userId = session.user.id; await ensureDbIndexed(); diff --git a/src/app/html/[id]/page.tsx b/src/app/html/[id]/page.tsx index f70381b..a3d0811 100644 --- a/src/app/html/[id]/page.tsx +++ b/src/app/html/[id]/page.tsx @@ -57,8 +57,9 @@ export default function HTMLPage() { }, [isLoading, id, router, setCurrentDocument, stop]); useEffect(() => { + if (!isLoading) return; loadDocument(); - }, [loadDocument]); + }, [loadDocument, isLoading]); // Compute available height = viewport - (header height + tts bar height) useEffect(() => { diff --git a/src/components/doclist/DocumentPreview.tsx b/src/components/doclist/DocumentPreview.tsx index 1a17429..d6bcab9 100644 --- a/src/components/doclist/DocumentPreview.tsx +++ b/src/components/doclist/DocumentPreview.tsx @@ -5,7 +5,8 @@ import { extractEpubCoverToDataUrl, renderPdfFirstPageToDataUrl, } from '@/lib/documentPreview'; -import { downloadDocumentContent, getDocumentContentSnippet } from '@/lib/client-documents'; +import { getDocumentContentSnippet } from '@/lib/client-documents'; +import { ensureCachedDocument } from '@/lib/document-cache'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; @@ -36,6 +37,16 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { const [isGenerating, setIsGenerating] = useState(false); const previewKey = useMemo(() => `${doc.type}:${doc.id}`, [doc.id, doc.type]); + const cacheMeta = useMemo( + () => ({ + id: doc.id, + name: doc.name, + type: doc.type, + size: doc.size, + lastModified: doc.lastModified, + }), + [doc.id, doc.lastModified, doc.name, doc.size, doc.type], + ); useEffect(() => { const el = containerRef.current; @@ -81,7 +92,9 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { const targetWidth = 240; if (doc.type === 'pdf') { - const data = await downloadDocumentContent(doc.id, { signal: controller.signal }); + const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal }); + if (cached.type !== 'pdf') return; + const data = cached.data; if (cancelled) return; const dataUrl = await renderPdfFirstPageToDataUrl(data, targetWidth); if (cancelled) return; @@ -92,7 +105,9 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { } if (doc.type === 'epub') { - const data = await downloadDocumentContent(doc.id, { signal: controller.signal }); + const cached = await ensureCachedDocument(cacheMeta, { signal: controller.signal }); + if (cached.type !== 'epub') return; + const data = cached.data; if (cancelled) return; const cover = await extractEpubCoverToDataUrl(data, targetWidth); if (cancelled) return; @@ -130,7 +145,7 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) { cancelled = true; controller.abort(); }; - }, [doc.id, doc.type, isVisible, previewKey]); + }, [cacheMeta, doc.id, doc.type, isVisible, previewKey]); const gradientClass = isPDF ? 'from-red-500/80 via-red-400/60 to-red-600/80' diff --git a/src/contexts/ConfigContext.tsx b/src/contexts/ConfigContext.tsx index 3622679..9773e20 100644 --- a/src/contexts/ConfigContext.tsx +++ b/src/contexts/ConfigContext.tsx @@ -111,13 +111,40 @@ export function ConfigProvider({ children }: { children: ReactNode }) { if (response?.ok) { const data = await response.json(); - const didMigrate = + const v2ApplyResponse = await fetch('/api/migrations/v2', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ dryRun: false, deleteLocal: false }), + }).catch(() => null); + const v2ApplyData = v2ApplyResponse?.ok ? await v2ApplyResponse.json().catch(() => null) : null; + + const didMigrateV1 = data.documentsMigrated || data.audiobooksMigrated || (data.rekey?.renamed ?? 0) > 0 || (data.rekey?.merged ?? 0) > 0; - if (didMigrate) { + if (v2ApplyData) { + const uploaded = Number(v2ApplyData.uploaded ?? 0); + const alreadyPresent = Number(v2ApplyData.alreadyPresent ?? 0); + const dbRowsUpdated = Number(v2ApplyData.dbRowsUpdated ?? 0); + const dbRowsSeeded = Number(v2ApplyData.dbRowsSeeded ?? 0); + const deletedLocal = Number(v2ApplyData.deletedLocal ?? 0); + const dbRowsMigrated = dbRowsUpdated + dbRowsSeeded; + const didMigrateV2 = uploaded > 0 || dbRowsMigrated > 0 || deletedLocal > 0; + + if (didMigrateV2) { + toast.success( + `Legacy document migration complete: ${uploaded} uploaded, ${alreadyPresent} already in S3, ${dbRowsMigrated} DB row(s) migrated.`, + { duration: 6000, icon: '📦' }, + ); + window.dispatchEvent(new CustomEvent('openreader:documentsChanged', { + detail: { reason: 'migration-v2-complete' }, + })); + } + } + + if (didMigrateV1) { toast.success('Library migration complete', { duration: 5000, icon: '📦', diff --git a/src/contexts/DocumentContext.tsx b/src/contexts/DocumentContext.tsx index 126f918..d9f7390 100644 --- a/src/contexts/DocumentContext.tsx +++ b/src/contexts/DocumentContext.tsx @@ -50,6 +50,19 @@ export function DocumentProvider({ children }: { children: ReactNode }) { }); }, [refreshDocuments]); + useEffect(() => { + const handler = () => { + refreshDocuments().catch((err) => { + console.error('Failed to refresh documents after change event:', err); + }); + }; + + window.addEventListener('openreader:documentsChanged', handler as EventListener); + return () => { + window.removeEventListener('openreader:documentsChanged', handler as EventListener); + }; + }, [refreshDocuments]); + const docsByType = useMemo(() => { const pdfDocs = (docs ?? []).filter((d) => d.type === 'pdf') as Array; const epubDocs = (docs ?? []).filter((d) => d.type === 'epub') as Array; diff --git a/src/contexts/HTMLContext.tsx b/src/contexts/HTMLContext.tsx index 401ed89..caf8e26 100644 --- a/src/contexts/HTMLContext.tsx +++ b/src/contexts/HTMLContext.tsx @@ -7,6 +7,8 @@ import { ReactNode, useCallback, useMemo, + useEffect, + useRef, } from 'react'; import { getDocumentMetadata } from '@/lib/client-documents'; import { ensureCachedDocument } from '@/lib/document-cache'; @@ -30,12 +32,16 @@ const HTMLContext = createContext(undefined); */ export function HTMLProvider({ children }: { children: ReactNode }) { const { setText: setTTSText, stop } = useTTS(); + const setTTSTextRef = useRef(setTTSText); // Current document state const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + useEffect(() => { + setTTSTextRef.current = setTTSText; + }, [setTTSText]); /** * Clears all current document state and stops any active TTS @@ -69,12 +75,12 @@ export function HTMLProvider({ children }: { children: ReactNode }) { setCurrDocName(doc.name); setCurrDocData(doc.data); setCurrDocText(doc.data); // Use the same text for TTS - setTTSText(doc.data); + setTTSTextRef.current(doc.data); } catch (error) { console.error('Failed to get HTML document:', error); clearCurrDoc(); } - }, [clearCurrDoc, setTTSText]); + }, [clearCurrDoc]); diff --git a/src/lib/auth-client.ts b/src/lib/auth-client.ts index 067bda2..a7e8c9e 100644 --- a/src/lib/auth-client.ts +++ b/src/lib/auth-client.ts @@ -12,22 +12,32 @@ function createAuthClientWithUrl(baseUrl: string) { // Cache for auth client instances by baseUrl const clientCache = new Map>(); +function resolveAuthClientBaseUrl(baseUrl: string | null): string { + if (typeof window !== 'undefined' && window.location?.origin) { + // Always use same-origin in the browser so local hostname variants + // (localhost vs LAN IP) do not break cookie/session bootstrap. + return window.location.origin; + } + + if (baseUrl) return baseUrl; + + throw new Error( + 'Cannot create auth client without baseUrl in a non-browser context. ' + + 'Use useAuthConfig() in components to get the properly configured baseUrl.' + ); +} + /** * Factory function to get auth client with specific baseUrl. * In components, prefer reading `baseUrl` from `useAuthConfig()` and then calling `getAuthClient(baseUrl)`. - * @param baseUrl - The auth server base URL. If null, will throw an error. + * @param baseUrl - Server-provided auth URL; in the browser we use same-origin automatically. */ export function getAuthClient(baseUrl: string | null) { - if (!baseUrl) { - throw new Error( - 'Cannot create auth client without baseUrl. ' + - 'Use useAuthConfig() in components to get the properly configured baseUrl.' - ); + const resolvedBaseUrl = resolveAuthClientBaseUrl(baseUrl); + + if (!clientCache.has(resolvedBaseUrl)) { + clientCache.set(resolvedBaseUrl, createAuthClientWithUrl(resolvedBaseUrl)); } - if (!clientCache.has(baseUrl)) { - clientCache.set(baseUrl, createAuthClientWithUrl(baseUrl)); - } - - return clientCache.get(baseUrl)!; + return clientCache.get(resolvedBaseUrl)!; } diff --git a/src/lib/client-documents.ts b/src/lib/client-documents.ts index df58c33..194524b 100644 --- a/src/lib/client-documents.ts +++ b/src/lib/client-documents.ts @@ -1,8 +1,52 @@ -import type { BaseDocument } from '@/types/documents'; +import { sha256HexFromArrayBuffer } from '@/lib/sha256'; +import type { BaseDocument, DocumentType } from '@/types/documents'; + +export type UploadSource = { + id: string; + name: string; + type: DocumentType; + size: number; + lastModified: number; + contentType: string; + body: Blob | ArrayBuffer | Uint8Array; +}; + +type UploadOptions = { + signal?: AbortSignal; +}; + +function toUploadBody(body: UploadSource['body']): BodyInit { + if (body instanceof Blob) return body; + if (body instanceof ArrayBuffer) return body; + return body as unknown as BodyInit; +} + +async function uploadDocumentSourceViaProxy(source: UploadSource, options?: UploadOptions): Promise { + const res = await fetch(`/api/documents/blob/upload/fallback?id=${encodeURIComponent(source.id)}`, { + method: 'PUT', + headers: { 'Content-Type': source.contentType || 'application/octet-stream' }, + body: toUploadBody(source.body), + signal: options?.signal, + }); + + if (!res.ok) { + const data = (await res.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || `Proxy upload failed (status ${res.status})`); + } +} + +function documentTypeForName(name: string): DocumentType { + const lower = name.toLowerCase(); + if (lower.endsWith('.pdf')) return 'pdf'; + if (lower.endsWith('.epub')) return 'epub'; + if (lower.endsWith('.docx')) return 'docx'; + return 'html'; +} export function mimeTypeForDoc(doc: Pick): string { if (doc.type === 'pdf') return 'application/pdf'; if (doc.type === 'epub') return 'application/epub+zip'; + if (doc.type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; const lower = doc.name.toLowerCase(); if (lower.endsWith('.md') || lower.endsWith('.markdown') || lower.endsWith('.mdown') || lower.endsWith('.mkd')) { @@ -13,7 +57,6 @@ export function mimeTypeForDoc(doc: Pick): string export async function listDocuments(options?: { ids?: string[]; signal?: AbortSignal }): Promise { const params = new URLSearchParams(); - params.set('list', 'true'); if (options?.ids?.length) { params.set('ids', options.ids.join(',')); } @@ -33,27 +76,114 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort return docs[0] ?? null; } -export async function uploadDocuments(files: File[], options?: { signal?: AbortSignal }): Promise { - const form = new FormData(); - for (const file of files) { - form.append('files', file); - } +export async function uploadDocumentSources(sources: UploadSource[], options?: UploadOptions): Promise { + if (sources.length === 0) return []; - const res = await fetch('/api/documents/upload', { + const presignRes = await fetch('/api/documents/blob/upload/presign', { method: 'POST', - body: form, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + uploads: sources.map((source) => ({ + id: source.id, + contentType: source.contentType, + size: source.size, + })), + }), signal: options?.signal, }); - if (!res.ok) { - const data = (await res.json().catch(() => null)) as { error?: string } | null; - throw new Error(data?.error || 'Failed to upload documents'); + if (!presignRes.ok) { + const data = (await presignRes.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || 'Failed to prepare uploads'); } - const data = (await res.json()) as { stored: BaseDocument[] }; + const presigned = (await presignRes.json()) as { + uploads?: Array<{ id: string; url: string; headers?: Record }>; + }; + const byId = new Map((presigned.uploads || []).map((upload) => [upload.id, upload])); + + for (const source of sources) { + const upload = byId.get(source.id); + if (!upload?.url) { + throw new Error(`Missing presigned upload for document ${source.id}`); + } + + let putError: unknown = null; + try { + const putRes = await fetch(upload.url, { + method: 'PUT', + headers: new Headers(upload.headers || {}), + body: toUploadBody(source.body), + signal: options?.signal, + }); + + // 412 means the content-hash object already exists (idempotent upload). + if (putRes.ok || putRes.status === 412) { + continue; + } + putError = new Error(`Direct upload failed with status ${putRes.status}`); + } catch (error) { + if (options?.signal?.aborted) throw error; + putError = error; + } + + try { + await uploadDocumentSourceViaProxy(source, options); + } catch (proxyError) { + const directMessage = putError instanceof Error ? putError.message : 'unknown direct upload error'; + const proxyMessage = proxyError instanceof Error ? proxyError.message : 'unknown proxy upload error'; + throw new Error(`Failed to upload document ${source.name}: ${directMessage}; fallback failed: ${proxyMessage}`); + } + } + + const registerRes = await fetch('/api/documents', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + documents: sources.map((source) => ({ + id: source.id, + name: source.name, + type: source.type, + size: source.size, + lastModified: source.lastModified, + })), + }), + signal: options?.signal, + }); + + if (!registerRes.ok) { + const data = (await registerRes.json().catch(() => null)) as { error?: string } | null; + throw new Error(data?.error || 'Failed to register uploaded documents'); + } + + const data = (await registerRes.json()) as { stored: BaseDocument[] }; return data.stored || []; } +export async function uploadDocuments(files: File[], options?: UploadOptions): Promise { + if (files.length === 0) return []; + + const sources: UploadSource[] = []; + for (const file of files) { + const bytes = await file.arrayBuffer(); + const id = await sha256HexFromArrayBuffer(bytes); + const type = documentTypeForName(file.name); + const name = file.name || `${id}.${type}`; + const contentType = file.type || mimeTypeForDoc({ name, type }); + sources.push({ + id, + name, + type, + size: file.size, + lastModified: Number.isFinite(file.lastModified) ? file.lastModified : Date.now(), + contentType, + body: file, + }); + } + + return uploadDocumentSources(sources, options); +} + export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user' | 'unclaimed'; signal?: AbortSignal }): Promise { const params = new URLSearchParams(); if (options?.ids?.length) { @@ -72,7 +202,7 @@ export async function deleteDocuments(options?: { ids?: string[]; scope?: 'user' } export async function downloadDocumentContent(id: string, options?: { signal?: AbortSignal }): Promise { - const res = await fetch(`/api/documents/content?id=${encodeURIComponent(id)}`, { signal: options?.signal }); + const res = await fetch(`/api/documents/blob?id=${encodeURIComponent(id)}`, { signal: options?.signal }); if (!res.ok) { const contentType = res.headers.get('content-type') || ''; if (contentType.includes('application/json')) { @@ -95,7 +225,7 @@ export async function getDocumentContentSnippet( if (typeof options?.maxChars === 'number') params.set('maxChars', String(options.maxChars)); if (typeof options?.maxBytes === 'number') params.set('maxBytes', String(options.maxBytes)); - const res = await fetch(`/api/documents/content?${params.toString()}`, { signal: options?.signal }); + const res = await fetch(`/api/documents/blob?${params.toString()}`, { signal: options?.signal }); if (!res.ok) { const data = (await res.json().catch(() => null)) as { error?: string } | null; throw new Error(data?.error || `Failed to load content snippet (status ${res.status})`); diff --git a/src/lib/dexie.ts b/src/lib/dexie.ts index e43d39e..5392b18 100644 --- a/src/lib/dexie.ts +++ b/src/lib/dexie.ts @@ -5,11 +5,12 @@ import { EPUBDocument, HTMLDocument, DocumentListState, - SyncedDocument, BaseDocument, DocumentListDocument, } from '@/types/documents'; import { sha256HexFromBytes, sha256HexFromString } from '@/lib/sha256'; +import { downloadDocumentContent, listDocuments, uploadDocumentSources, type UploadSource } from '@/lib/client-documents'; +import { cacheStoredDocumentFromBytes } from '@/lib/document-cache'; const DB_NAME = 'openreader-db'; // Managed via Dexie (version bumped from the original manual IndexedDB) @@ -657,15 +658,26 @@ export async function syncDocumentsToServer( const epubDocs = await getAllEpubDocuments(); const htmlDocs = await getAllHtmlDocuments(); - const documents: SyncedDocument[] = []; + const uploads: Array<{ oldId: string; source: UploadSource }> = []; const totalDocs = pdfDocs.length + epubDocs.length + htmlDocs.length; let processedDocs = 0; + const textEncoder = new TextEncoder(); + for (const doc of pdfDocs) { - documents.push({ - ...doc, - type: 'pdf', - data: Array.from(new Uint8Array(doc.data)), + const bytes = new Uint8Array(doc.data); + const id = await sha256HexFromBytes(bytes); + uploads.push({ + oldId: doc.id, + source: { + id, + name: doc.name, + type: 'pdf', + size: bytes.byteLength, + lastModified: doc.lastModified, + contentType: 'application/pdf', + body: bytes, + }, }); processedDocs++; if (onProgress) { @@ -674,10 +686,19 @@ export async function syncDocumentsToServer( } for (const doc of epubDocs) { - documents.push({ - ...doc, - type: 'epub', - data: Array.from(new Uint8Array(doc.data)), + const bytes = new Uint8Array(doc.data); + const id = await sha256HexFromBytes(bytes); + uploads.push({ + oldId: doc.id, + source: { + id, + name: doc.name, + type: 'epub', + size: bytes.byteLength, + lastModified: doc.lastModified, + contentType: 'application/epub+zip', + body: bytes, + }, }); processedDocs++; if (onProgress) { @@ -685,13 +706,20 @@ export async function syncDocumentsToServer( } } - const encoder = new TextEncoder(); for (const doc of htmlDocs) { - const encoded = encoder.encode(doc.data); - documents.push({ - ...doc, - type: 'html', - data: Array.from(encoded), + const encoded = textEncoder.encode(doc.data); + const id = await sha256HexFromBytes(encoded); + uploads.push({ + oldId: doc.id, + source: { + id, + name: doc.name, + type: 'html', + size: encoded.byteLength, + lastModified: doc.lastModified, + contentType: 'text/plain; charset=utf-8', + body: encoded, + }, }); processedDocs++; if (onProgress) { @@ -703,25 +731,11 @@ export async function syncDocumentsToServer( onProgress(50, 'Uploading to server...'); } - const response = await fetch('/api/documents', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ documents }), - signal, - }); + await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); - if (!response.ok) { - throw new Error('Failed to sync documents to server'); - } - - const payload = (await response.json().catch(() => null)) as - | { stored?: Array<{ oldId: string; id: string }> } - | null; - const stored = payload?.stored ?? []; - for (const mapping of stored) { - if (!mapping || typeof mapping.oldId !== 'string' || typeof mapping.id !== 'string') continue; - if (mapping.oldId === mapping.id) continue; - await applyDocumentIdMapping(mapping.oldId, mapping.id); + for (const entry of uploads) { + if (entry.oldId === entry.source.id) continue; + await applyDocumentIdMapping(entry.oldId, entry.source.id); } if (onProgress) { @@ -736,52 +750,77 @@ export async function syncSelectedDocumentsToServer( onProgress?: (progress: number, status?: string) => void, signal?: AbortSignal, ): Promise<{ lastSync: number }> { - // Re-use logic from syncDocumentsToServer but only for specific documents - // Actually, syncDocumentsToServer fetches all docs from DB. - // We need to fetch the *full content* of the selected docs from DB. - - const fullDocs: SyncedDocument[] = []; - let processed = 0; - - for (const doc of documents) { - if (doc.type === 'pdf') { - const data = await getPdfDocument(doc.id); - if (data) fullDocs.push({ ...data, type: 'pdf', data: Array.from(new Uint8Array(data.data)) }); - } else if (doc.type === 'epub') { - const data = await getEpubDocument(doc.id); - if (data) fullDocs.push({ ...data, type: 'epub', data: Array.from(new Uint8Array(data.data)) }); - } else { - const data = await getHtmlDocument(doc.id); - if (data) { - const encoder = new TextEncoder(); - fullDocs.push({ ...data, type: 'html', data: Array.from(encoder.encode(data.data)) }); - } - } - processed++; - if (onProgress) onProgress((processed / documents.length) * 50, `Preparing ${processed}/${documents.length}...`); + const uploads: Array<{ oldId: string; source: UploadSource }> = []; + const textEncoder = new TextEncoder(); + let processed = 0; + + for (const doc of documents) { + if (doc.type === 'pdf') { + const data = await getPdfDocument(doc.id); + if (data) { + const bytes = new Uint8Array(data.data); + const id = await sha256HexFromBytes(bytes); + uploads.push({ + oldId: data.id, + source: { + id, + name: data.name, + type: 'pdf', + size: bytes.byteLength, + lastModified: data.lastModified, + contentType: 'application/pdf', + body: bytes, + }, + }); + } + } else if (doc.type === 'epub') { + const data = await getEpubDocument(doc.id); + if (data) { + const bytes = new Uint8Array(data.data); + const id = await sha256HexFromBytes(bytes); + uploads.push({ + oldId: data.id, + source: { + id, + name: data.name, + type: 'epub', + size: bytes.byteLength, + lastModified: data.lastModified, + contentType: 'application/epub+zip', + body: bytes, + }, + }); + } + } else { + const data = await getHtmlDocument(doc.id); + if (data) { + const bytes = textEncoder.encode(data.data); + const id = await sha256HexFromBytes(bytes); + uploads.push({ + oldId: data.id, + source: { + id, + name: data.name, + type: 'html', + size: bytes.byteLength, + lastModified: data.lastModified, + contentType: 'text/plain; charset=utf-8', + body: bytes, + }, + }); + } } - - if (onProgress) onProgress(50, 'Uploading to server...'); - const response = await fetch('/api/documents', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ documents: fullDocs }), - signal, - }); - - if (!response.ok) { - throw new Error('Failed to sync documents to server'); + processed++; + if (onProgress) onProgress((processed / documents.length) * 50, `Preparing ${processed}/${documents.length}...`); } - const payload = (await response.json().catch(() => null)) as - | { stored?: Array<{ oldId: string; id: string }> } - | null; - const stored = payload?.stored ?? []; - for (const mapping of stored) { - if (!mapping || typeof mapping.oldId !== 'string' || typeof mapping.id !== 'string') continue; - if (mapping.oldId === mapping.id) continue; - await applyDocumentIdMapping(mapping.oldId, mapping.id); + if (onProgress) onProgress(50, 'Uploading to server...'); + await uploadDocumentSources(uploads.map((entry) => entry.source), { signal }); + + for (const entry of uploads) { + if (entry.oldId === entry.source.id) continue; + await applyDocumentIdMapping(entry.oldId, entry.source.id); } if (onProgress) { @@ -800,22 +839,8 @@ export async function loadDocumentsFromServer( onProgress(10, 'Starting download...'); } - const response = await fetch('/api/documents', { signal }); - if (!response.ok) { - throw new Error('Failed to fetch documents from server'); - } - - if (onProgress) { - onProgress(30, 'Download complete'); - } - - const { documents } = (await response.json()) as { documents: SyncedDocument[] }; - - if (onProgress) { - onProgress(40, 'Parsing documents...'); - } - - await saveSyncedDocumentsLocally(documents, onProgress); + const documents = await listDocuments({ signal }); + await downloadAndCacheServerDocuments(documents, onProgress, signal); if (onProgress) { onProgress(100, 'Load complete!'); @@ -832,26 +857,9 @@ export async function loadSelectedDocumentsFromServer( if (onProgress) { onProgress(10, 'Starting download...'); } - - // Use new filtered API - const idsParam = selectedIds.join(','); - const response = await fetch(`/api/documents?ids=${encodeURIComponent(idsParam)}`, { signal }); - - if (!response.ok) { - throw new Error('Failed to fetch documents from server'); - } - if (onProgress) { - onProgress(30, 'Download complete'); - } - - const { documents } = (await response.json()) as { documents: SyncedDocument[] }; - - if (onProgress) { - onProgress(40, 'Parsing documents...'); - } - - await saveSyncedDocumentsLocally(documents, onProgress); + const documents = await listDocuments({ ids: selectedIds, signal }); + await downloadAndCacheServerDocuments(documents, onProgress, signal); if (onProgress) { onProgress(100, 'Load complete!'); @@ -860,52 +868,23 @@ export async function loadSelectedDocumentsFromServer( return { lastSync: Date.now() }; } -async function saveSyncedDocumentsLocally(documents: SyncedDocument[], onProgress?: (progress: number, status?: string) => void) { - const textDecoder = new TextDecoder(); +async function downloadAndCacheServerDocuments( + documents: BaseDocument[], + onProgress?: (progress: number, status?: string) => void, + signal?: AbortSignal, +) { + if (onProgress) onProgress(30, 'List complete'); + if (documents.length === 0) { + if (onProgress) onProgress(95, 'No documents to import'); + return; + } for (let i = 0; i < documents.length; i++) { const doc = documents[i]; - - if (doc.type === 'pdf') { - const uint8Array = new Uint8Array(doc.data); - const documentData: PDFDocument = { - id: doc.id, - type: 'pdf', - name: doc.name, - size: doc.size, - lastModified: doc.lastModified, - data: uint8Array.buffer, - }; - await addPdfDocument(documentData); - } else if (doc.type === 'epub') { - const uint8Array = new Uint8Array(doc.data); - const documentData: EPUBDocument = { - id: doc.id, - type: 'epub', - name: doc.name, - size: doc.size, - lastModified: doc.lastModified, - data: uint8Array.buffer, - }; - await addEpubDocument(documentData); - } else if (doc.type === 'html') { - const uint8Array = new Uint8Array(doc.data); - const decoded = textDecoder.decode(uint8Array); - const documentData: HTMLDocument = { - id: doc.id, - type: 'html', - name: doc.name, - size: doc.size, - lastModified: doc.lastModified, - data: decoded, - }; - await addHtmlDocument(documentData); - } else { - console.warn(`Unknown document type: ${doc.type}`); - } - + const bytes = await downloadDocumentContent(doc.id, { signal }); + await cacheStoredDocumentFromBytes(doc, bytes); if (onProgress) { - onProgress(40 + ((i + 1) / documents.length) * 50, `Processing document ${i + 1}/${documents.length}...`); + onProgress(30 + ((i + 1) / documents.length) * 65, `Downloading ${i + 1}/${documents.length}: ${doc.name}`); } } } @@ -1007,5 +986,3 @@ export async function importDocumentsFromLibrary( onProgress(100, 'Library import complete!'); } } - - diff --git a/src/lib/server/auth-config.ts b/src/lib/server/auth-config.ts index 2279e9e..6acea79 100644 --- a/src/lib/server/auth-config.ts +++ b/src/lib/server/auth-config.ts @@ -1,9 +1,9 @@ /** * Centralized auth configuration check. - * Auth is only enabled when BOTH BETTER_AUTH_SECRET and BETTER_AUTH_URL are set. + * Auth is only enabled when BOTH AUTH_SECRET and BASE_URL are set. */ export function isAuthEnabled(): boolean { - return !!(process.env.BETTER_AUTH_SECRET && process.env.BETTER_AUTH_URL); + return !!(process.env.AUTH_SECRET && process.env.BASE_URL); } /** @@ -13,5 +13,5 @@ export function getAuthBaseUrl(): string | null { if (!isAuthEnabled()) { return null; } - return process.env.BETTER_AUTH_URL || null; + return process.env.BASE_URL || null; } diff --git a/src/lib/server/auth.ts b/src/lib/server/auth.ts index 1339dc4..e483ff1 100644 --- a/src/lib/server/auth.ts +++ b/src/lib/server/auth.ts @@ -13,6 +13,34 @@ import * as schema from "@/db/schema"; // Import the dynamic schema // ... +function tryGetOrigin(url: string | undefined): string | null { + if (!url) return null; + try { + return new URL(url).origin; + } catch { + return null; + } +} + +function getTrustedOrigins(): string[] { + const origins = new Set(); + const baseOrigin = tryGetOrigin(process.env.BASE_URL); + if (baseOrigin) origins.add(baseOrigin); + + // Comma-separated list for local multi-host setups (e.g., localhost + LAN IP). + const extra = (process.env.AUTH_TRUSTED_ORIGINS || '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean); + + for (const candidate of extra) { + const origin = tryGetOrigin(candidate); + if (origin) origins.add(origin); + } + + return Array.from(origins); +} + const createAuth = () => betterAuth({ // eslint-disable-next-line @typescript-eslint/no-explicit-any database: drizzleAdapter(db as any, { @@ -25,8 +53,9 @@ const createAuth = () => betterAuth({ verification: schema.verification, } }), - secret: process.env.BETTER_AUTH_SECRET!, - baseURL: process.env.BETTER_AUTH_URL || "http://localhost:3003", + secret: process.env.AUTH_SECRET!, + baseURL: process.env.BASE_URL!, + trustedOrigins: getTrustedOrigins(), emailAndPassword: { enabled: true, requireEmailVerification: false, // Set to true in production diff --git a/src/lib/server/db-indexing.ts b/src/lib/server/db-indexing.ts index ab8c1c3..1572fcd 100644 --- a/src/lib/server/db-indexing.ts +++ b/src/lib/server/db-indexing.ts @@ -6,7 +6,7 @@ import { db } from '@/db'; import { audiobookChapters, audiobooks, documents } from '@/db/schema'; import { and, count, eq } from 'drizzle-orm'; import { listStoredChapters } from '@/lib/server/audiobook'; -import { AUDIOBOOKS_V1_DIR, DOCUMENTS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore'; +import { AUDIOBOOKS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore'; const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); const MIGRATIONS_DIR = path.join(DOCSTORE_DIR, '.migrations'); @@ -38,33 +38,14 @@ async function writeState(): Promise { await fs.writeFile(STATE_PATH, JSON.stringify(state, null, 2)); } -async function hasFilesystemContent(mode: DbIndexState['mode']): Promise<{ documents: boolean; audiobooks: boolean }> { - let documentsPresent = false; - let audiobooksPresent = false; - - // Documents are always stored under documents_v1. - try { - const entries = await fs.readdir(DOCUMENTS_V1_DIR); - documentsPresent = entries.some((name) => /^[a-f0-9]{64}__.+$/i.test(name)); - } catch { - // ignore - } - - // Audiobooks differ by auth-mode layout. +async function hasAudiobookFilesystemContent(mode: DbIndexState['mode']): Promise { const audiobookDir = mode === 'auth' ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR; try { const entries = await fs.readdir(audiobookDir, { withFileTypes: true }); - audiobooksPresent = entries.some((e) => e.isDirectory() && e.name.endsWith('-audiobook')); + return entries.some((entry) => entry.isDirectory() && entry.name.endsWith('-audiobook')); } catch { - // ignore + return false; } - - return { documents: documentsPresent, audiobooks: audiobooksPresent }; -} - -async function isDocumentIndexed(id: string): Promise { - const result = await db.select({ id: documents.id }).from(documents).where(eq(documents.id, id)); - return result.length > 0; } async function isAudiobookIndexedForUser(id: string, userId: string): Promise { @@ -90,7 +71,6 @@ async function migrateLegacyAudiobooksToUnclaimed(): Promise { const sourceDir = path.join(AUDIOBOOKS_V1_DIR, entry.name); const targetDir = path.join(unclaimedDir, entry.name); - if (existsSync(targetDir)) continue; try { @@ -115,133 +95,83 @@ export async function getUnclaimedCounts(): Promise<{ documents: number; audiobo }; } -async function scanAndPopulateDb(): Promise<{ documents: number; audiobooks: number }> { +async function scanAndPopulateAudiobookDb(): Promise { const authEnabled = isAuthEnabled(); - - console.log('Scanning file system for un-indexed content...'); + console.log('Scanning file system for un-indexed audiobooks...'); if (authEnabled) { await migrateLegacyAudiobooksToUnclaimed(); } - if (existsSync(DOCUMENTS_V1_DIR)) { - const files = await fs.readdir(DOCUMENTS_V1_DIR); - for (const file of files) { - const match = /^([a-f0-9]{64})__(.+)$/i.exec(file); - if (!match) continue; - - const id = match[1]; - const encodedName = match[2]; - if (await isDocumentIndexed(id)) continue; - - let name: string; - try { - name = decodeURIComponent(encodedName); - } catch { - continue; - } - - const filePath = path.join(DOCUMENTS_V1_DIR, file); - const stats = await fs.stat(filePath); - const ext = path.extname(name).toLowerCase().replace('.', ''); - - await db.insert(documents).values({ - id, - userId: UNCLAIMED_USER_ID, - name, - type: ext, - size: stats.size, - lastModified: Math.floor(stats.mtimeMs), - filePath: file, - }); - console.log(`Indexed document: ${name} (${id})`); - } - } - const audiobookScanDir = authEnabled ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR; - if (existsSync(audiobookScanDir)) { - const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true }); - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if (!entry.name.endsWith('-audiobook')) continue; + if (!existsSync(audiobookScanDir)) return; - const bookId = entry.name.replace('-audiobook', ''); - if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue; + const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (!entry.name.endsWith('-audiobook')) continue; - const dirPath = path.join(audiobookScanDir, entry.name); + const bookId = entry.name.replace('-audiobook', ''); + if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue; - let title = 'Unknown Title'; - try { - const metaPath = path.join(dirPath, 'audiobook.meta.json'); - const metaContent = await fs.readFile(metaPath, 'utf8'); - JSON.parse(metaContent); - } catch { - // ignore - } + const dirPath = path.join(audiobookScanDir, entry.name); - const chapters = await listStoredChapters(dirPath); - const totalDuration = chapters.reduce((acc, c) => acc + (c.durationSec || 0), 0); - if (chapters.length > 0) title = chapters[0].title || title; + let title = 'Unknown Title'; + try { + const metaPath = path.join(dirPath, 'audiobook.meta.json'); + const metaContent = await fs.readFile(metaPath, 'utf8'); + JSON.parse(metaContent); + } catch { + // ignore + } - await db.insert(audiobooks).values({ - id: bookId, + const chapters = await listStoredChapters(dirPath); + const totalDuration = chapters.reduce((acc, chapter) => acc + (chapter.durationSec || 0), 0); + if (chapters.length > 0) title = chapters[0].title || title; + + await db.insert(audiobooks).values({ + id: bookId, + userId: UNCLAIMED_USER_ID, + title, + duration: totalDuration, + }); + console.log(`Indexed audiobook: ${bookId}`); + + for (const chapter of chapters) { + await db.insert(audiobookChapters).values({ + id: `${bookId}-${chapter.index}`, + bookId, userId: UNCLAIMED_USER_ID, - title, - duration: totalDuration, + chapterIndex: chapter.index, + title: chapter.title, + duration: chapter.durationSec || 0, + filePath: chapter.filePath, + format: chapter.format, }); - console.log(`Indexed audiobook: ${bookId}`); - - for (const chapter of chapters) { - await db.insert(audiobookChapters).values({ - id: `${bookId}-${chapter.index}`, - bookId, - userId: UNCLAIMED_USER_ID, - chapterIndex: chapter.index, - title: chapter.title, - duration: chapter.durationSec || 0, - filePath: chapter.filePath, - format: chapter.format, - }); - } } } - - return getUnclaimedCounts(); } -/** - * Ensure DB has rows for existing filesystem content (documents/audiobooks). - * - * This is intentionally safe to call from routes: - * - Runs at most once per process (memoryIndexed) - * - Uses a persisted state file under docstore/.migrations to avoid rescanning every boot - */ -export async function ensureDbIndexed(): Promise { +export async function ensureAudiobooksIndexed(): Promise { const mode: DbIndexState['mode'] = isAuthEnabled() ? 'auth' : 'noauth'; if (memoryIndexedMode === mode) return; inflight ??= (async () => { const hasState = existsSync(STATE_PATH) ? await readState() : null; if (hasState && hasState.mode === mode) { - // If the DB was reset but the state file survived, don't get stuck "indexed" forever. - // Only skip if: - // - the DB already has rows for any on-disk content (per category), OR - // - there is no content on disk to index (per category). - // - // This avoids a bad state where (for example) audiobooks are indexed, but documents exist on disk - // and aren't counted/claimable because we early-exit based on "some DB rows exist". - const [counts, fsHas] = await Promise.all([getUnclaimedCounts(), hasFilesystemContent(mode)]); - const docsOk = counts.documents > 0 || !fsHas.documents; - const audiobooksOk = counts.audiobooks > 0 || !fsHas.audiobooks; - - if (docsOk && audiobooksOk) { + const [counts, fsHasAudiobooks] = await Promise.all([ + getUnclaimedCounts(), + hasAudiobookFilesystemContent(mode), + ]); + const audiobooksOk = counts.audiobooks > 0 || !fsHasAudiobooks; + if (audiobooksOk) { memoryIndexedMode = mode; return; } } await fs.mkdir(DOCSTORE_DIR, { recursive: true }); - await scanAndPopulateDb(); + await scanAndPopulateAudiobookDb(); await writeState(); memoryIndexedMode = mode; })().finally(() => { @@ -250,3 +180,7 @@ export async function ensureDbIndexed(): Promise { await inflight; } + +export async function ensureDbIndexed(): Promise { + await ensureAudiobooksIndexed(); +} diff --git a/src/lib/server/docstore.ts b/src/lib/server/docstore.ts index 8ade6a3..79e76a7 100644 --- a/src/lib/server/docstore.ts +++ b/src/lib/server/docstore.ts @@ -11,6 +11,7 @@ export const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1'); export const AUDIOBOOKS_USERS_DIR = path.join(DOCSTORE_DIR, 'audiobooks_users_v1'); export const UNCLAIMED_USER_ID = 'unclaimed'; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; /** * Get the audiobook directory for a specific user when auth is enabled. @@ -34,15 +35,29 @@ export function getUnclaimedAudiobookDir(): string { } /** - * Get the full path to a specific audiobook directory. - * - When auth is disabled: docstore/audiobooks_v1/{bookId}-audiobook - * - When auth is enabled: docstore/audiobooks_users_v1/{userId}/{bookId}-audiobook + * Resolve the base audiobooks directory for request context, including optional test namespace. + * - When auth is disabled or userId is absent: docstore/audiobooks_v1[/] + * - When auth is enabled: docstore/audiobooks_users_v1/{userId}[/] */ -export function getAudiobookPath(bookId: string, userId: string | null, authEnabled: boolean): string { - if (!authEnabled || !userId) { - return path.join(AUDIOBOOKS_V1_DIR, `${bookId}-audiobook`); - } - return path.join(getUserAudiobookDir(userId), `${bookId}-audiobook`); +export function getAudiobooksRootDir({ + userId, + authEnabled, + namespace, +}: { + userId: string | null; + authEnabled: boolean; + namespace: string | null; +}): string { + const baseDir = !authEnabled || !userId ? AUDIOBOOKS_V1_DIR : getUserAudiobookDir(userId); + if (!namespace) return baseDir; + + const safe = namespace.trim(); + if (!safe || safe === '.' || safe === '..' || safe.includes('..')) return baseDir; + if (!SAFE_NAMESPACE_REGEX.test(safe)) return baseDir; + + const resolved = path.resolve(baseDir, safe); + if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir; + return resolved; } /** @@ -210,15 +225,6 @@ export function getMigratedDocumentFileName(id: string, name: string): string { return targetFileName; } -export type FSDocument = { - id: string; - name: string; - type: string; - size: number; - lastModified: number; - filePath: string; -}; - export async function ensureDocumentsV1Ready(): Promise { await mkdir(DOCSTORE_DIR, { recursive: true }); await mkdir(DOCUMENTS_V1_DIR, { recursive: true }); diff --git a/src/lib/server/documents-blobstore.ts b/src/lib/server/documents-blobstore.ts new file mode 100644 index 0000000..dad8092 --- /dev/null +++ b/src/lib/server/documents-blobstore.ts @@ -0,0 +1,222 @@ +import { + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { getS3Client, getS3Config } from '@/lib/server/s3'; + +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; + +function sanitizeNamespace(namespace: string | null): string | null { + if (!namespace) return null; + if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null; + return namespace; +} + +function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream { + return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function'; +} + +async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) { + chunks.push(chunk); + } else if (typeof chunk === 'string') { + chunks.push(Buffer.from(chunk)); + } else { + chunks.push(Buffer.from(chunk as Uint8Array)); + } + } + return Buffer.concat(chunks); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + + if (isNodeReadableStream(body)) { + return streamToBuffer(body); + } + + throw new Error('Unsupported S3 response body type'); +} + +export function isValidDocumentId(id: string): boolean { + return DOCUMENT_ID_REGEX.test(id); +} + +export function documentKey(id: string, namespace: string | null): string { + if (!isValidDocumentId(id)) { + throw new Error(`Invalid document id: ${id}`); + } + + const cfg = getS3Config(); + const ns = sanitizeNamespace(namespace); + const nsSegment = ns ? `ns/${ns}/` : ''; + return `${cfg.prefix}/documents_v1/${nsSegment}${id}`; +} + +export async function presignPut( + id: string, + contentType: string, + namespace: string | null, +): Promise<{ url: string; headers: Record }> { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; + + const command = new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + ContentType: normalizedType, + IfNoneMatch: '*', + }); + const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); + + return { + url, + headers: { + 'Content-Type': normalizedType, + 'If-None-Match': '*', + }, + }; +} + +export async function headDocumentBlob( + id: string, + namespace: string | null, +): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key })); + return { + contentLength: Number(res.ContentLength ?? 0), + contentType: res.ContentType ?? null, + eTag: res.ETag ?? null, + }; +} + +export async function getDocumentRange( + id: string, + start: number, + endInclusive: number, + namespace: string | null, +): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + const res = await client.send( + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Range: `bytes=${Math.max(0, start)}-${Math.max(0, endInclusive)}`, + }), + ); + return bodyToBuffer(res.Body); +} + +export async function getDocumentBlob(id: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + const res = await client.send( + new GetObjectCommand({ + Bucket: cfg.bucket, + Key: key, + }), + ); + return bodyToBuffer(res.Body); +} + +export async function putDocumentBlob( + id: string, + body: Buffer, + contentType: string, + namespace: string | null, +): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: contentType, + IfNoneMatch: '*', + }), + ); +} + +export async function deleteDocumentBlob(id: string, namespace: string | null): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const key = documentKey(id, namespace); + await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key })); +} + +export function isMissingBlobError(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; + if (maybe.$metadata?.httpStatusCode === 404) return true; + if (maybe.name === 'NotFound' || maybe.name === 'NoSuchKey') return true; + if (maybe.Code === 'NotFound' || maybe.Code === 'NoSuchKey') return true; + return false; +} + +export async function deleteDocumentPrefix(prefix: string): Promise { + const cfg = getS3Config(); + const client = getS3Client(); + const cleanedPrefix = prefix.replace(/^\/+/, ''); + let deleted = 0; + let continuationToken: string | undefined; + + do { + const listRes = await client.send( + new ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: cleanedPrefix, + ContinuationToken: continuationToken, + }), + ); + + const keys = (listRes.Contents ?? []) + .map((item) => item.Key) + .filter((value): value is string => typeof value === 'string' && value.length > 0); + + if (keys.length > 0) { + const deleteRes = await client.send( + new DeleteObjectsCommand({ + Bucket: cfg.bucket, + Delete: { + Objects: keys.map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + deleted += deleteRes.Deleted?.length ?? 0; + } + + continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; + } while (continuationToken); + + return deleted; +} diff --git a/src/lib/server/documents-utils.ts b/src/lib/server/documents-utils.ts index 1d4bf1f..982c5a9 100644 --- a/src/lib/server/documents-utils.ts +++ b/src/lib/server/documents-utils.ts @@ -1,11 +1,6 @@ import path from 'path'; -import { utimes } from 'fs/promises'; import type { DocumentType } from '@/types/documents'; -export function isEnoent(error: unknown): boolean { - return typeof error === 'object' && error !== null && 'code' in error && (error as { code?: unknown }).code === 'ENOENT'; -} - export function safeDocumentName(rawName: string, fallback: string): string { const baseName = path.basename(rawName || fallback); return baseName.replaceAll('\u0000', '').slice(0, 240) || fallback; @@ -18,16 +13,3 @@ export function toDocumentTypeFromName(name: string): DocumentType { if (ext === '.docx') return 'docx'; return 'html'; } - -export async function trySetFileMtime(filePath: string, lastModifiedMs: number): Promise { - if (!Number.isFinite(lastModifiedMs)) return; - const mtime = new Date(lastModifiedMs); - if (Number.isNaN(mtime.getTime())) return; - - try { - await utimes(filePath, mtime, mtime); - } catch (error) { - console.warn('Failed to set document mtime:', filePath, error); - } -} - diff --git a/src/lib/server/rate-limiter.ts b/src/lib/server/rate-limiter.ts index 2cf097a..a041ce4 100644 --- a/src/lib/server/rate-limiter.ts +++ b/src/lib/server/rate-limiter.ts @@ -111,6 +111,17 @@ export class RateLimiter { return this.isPostgres() ? new Date() : Date.now(); } + // Use a transaction only when running with Postgres. + // better-sqlite3 transactions require sync callbacks and cannot be awaited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private async runMutation(fn: (conn: any) => Promise): Promise { + if (this.isPostgres()) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return safeDb().transaction(async (tx: any) => fn(tx)); + } + return fn(safeDb()); + } + /** * Check if a user can use TTS and increment their char count if allowed */ @@ -147,16 +158,10 @@ export class RateLimiter { try { const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue; - - // Use a DB transaction to avoid partial increments across buckets and to avoid - // non-transactional "rollback" logic that can corrupt counts under concurrency. - // Note: We intentionally allow a request to push a bucket over its limit; we only - // block when the bucket was already exhausted before this request starts updating. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return await safeDb().transaction(async (tx: any) => { + return await this.runMutation(async (conn) => { // Ensure records exist for each bucket for (const bucket of buckets) { - await tx.insert(userTtsChars) + await conn.insert(userTtsChars) .values({ userId: bucket.key, date: dateValue, @@ -172,7 +177,7 @@ export class RateLimiter { // that start after the bucket is already exhausted, while still allowing a // request to push the count over the limit. for (const bucket of buckets) { - const updateResult = await tx.update(userTtsChars) + const updateResult = await conn.update(userTtsChars) .set({ charCount: sql`${userTtsChars.charCount} + ${charCount}`, updatedAt, @@ -191,7 +196,7 @@ export class RateLimiter { // Fetch current counts const bucketResults: Array<{ currentCount: number; limit: number }> = []; for (const bucket of buckets) { - const result = await tx.select({ currentCount: userTtsChars.charCount }) + const result = await conn.select({ currentCount: userTtsChars.charCount }) .from(userTtsChars) .where(and(eq(userTtsChars.userId, bucket.key), eq(userTtsChars.date, dateValue))); diff --git a/src/lib/server/s3.ts b/src/lib/server/s3.ts new file mode 100644 index 0000000..efd9f8a --- /dev/null +++ b/src/lib/server/s3.ts @@ -0,0 +1,82 @@ +import { S3Client } from '@aws-sdk/client-s3'; + +type S3Config = { + bucket: string; + region: string; + endpoint?: string; + accessKeyId: string; + secretAccessKey: string; + forcePathStyle: boolean; + prefix: string; +}; + +let cachedClient: S3Client | null = null; +let cachedConfig: S3Config | null = null; + +function parseBool(value: string | undefined): boolean { + if (!value) return false; + const normalized = value.trim().toLowerCase(); + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; +} + +function normalizePrefix(prefix: string | undefined): string { + const base = (prefix || 'openreader').trim(); + if (!base) return 'openreader'; + return base.replace(/^\/+|\/+$/g, ''); +} + +function loadS3ConfigFromEnv(): S3Config | null { + const bucket = process.env.S3_BUCKET?.trim(); + const region = process.env.S3_REGION?.trim(); + const accessKeyId = process.env.S3_ACCESS_KEY_ID?.trim(); + const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY?.trim(); + const endpoint = process.env.S3_ENDPOINT?.trim(); + + if (!bucket || !region || !accessKeyId || !secretAccessKey) { + return null; + } + + return { + bucket, + region, + endpoint: endpoint || undefined, + accessKeyId, + secretAccessKey, + forcePathStyle: parseBool(process.env.S3_FORCE_PATH_STYLE), + prefix: normalizePrefix(process.env.S3_PREFIX), + }; +} + +export function isS3Configured(): boolean { + return loadS3ConfigFromEnv() !== null; +} + +export function getS3Config(): S3Config { + if (cachedConfig) return cachedConfig; + const config = loadS3ConfigFromEnv(); + if (!config) { + throw new Error( + 'S3 is not configured. Required env vars: S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.', + ); + } + cachedConfig = config; + return config; +} + +export function getS3Client(): S3Client { + if (cachedClient) return cachedClient; + const config = getS3Config(); + + cachedClient = new S3Client({ + region: config.region, + endpoint: config.endpoint, + forcePathStyle: config.forcePathStyle, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }); + + return cachedClient; +} + diff --git a/tests/export.spec.ts b/tests/export.spec.ts index 39cbd64..05fbe1e 100644 --- a/tests/export.spec.ts +++ b/tests/export.spec.ts @@ -251,7 +251,8 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async await startGeneration(page); // Progress card should appear with a Cancel button while chapters are being generated - const cancelButton = page.getByRole('button', { name: 'Cancel' }); + const generationCard = page.locator('div', { hasText: 'Generating Audiobook' }).first(); + const cancelButton = generationCard.getByRole('button', { name: 'Cancel' }); await expect(cancelButton).toBeVisible({ timeout: 60_000 }); await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 }); @@ -266,8 +267,10 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async // Now cancel the in-flight generation await cancelButton.click(); - // After cancellation, the inline progress card's Cancel button should be gone - await expect(page.getByRole('button', { name: 'Cancel' })).toHaveCount(0); + // Cancellation is asynchronous: wait for generation to settle before asserting + // that the inline progress card has disappeared. + await expect(page.getByRole('button', { name: 'Resume' })).toBeVisible({ timeout: 30_000 }); + await expect(generationCard).toHaveCount(0, { timeout: 30_000 }); // After cancellation, wait for the chapter count to stabilize. In-flight TTS // requests may still complete after we click cancel, so we poll until the diff --git a/tests/global-teardown.ts b/tests/global-teardown.ts index f8040c4..fb2bea2 100644 --- a/tests/global-teardown.ts +++ b/tests/global-teardown.ts @@ -1,29 +1,11 @@ -import fs from 'fs/promises'; -import path from 'path'; - -async function listWorkerNamespaces(documentsRoot: string): Promise { - let entries: Array<{ name: string; isDirectory: () => boolean }> = []; - try { - entries = await fs.readdir(documentsRoot, { withFileTypes: true }); - } catch { - return []; - } - - return entries - .filter((d) => d.isDirectory()) - .map((d) => d.name) - .filter((name) => /^(chromium|firefox|webkit)-worker\d+$/.test(name)); -} +import { deleteDocumentPrefix } from '../src/lib/server/documents-blobstore'; +import { getS3Config, isS3Configured } from '../src/lib/server/s3'; export default async function globalTeardown(): Promise { - const documentsRoot = path.join(process.cwd(), 'docstore', 'documents_v1'); - const namespaces = await listWorkerNamespaces(documentsRoot); - if (!namespaces.length) return; + if (!isS3Configured()) return; - await Promise.all( - namespaces.map(async (ns) => { - const dir = path.join(documentsRoot, ns); - await fs.rm(dir, { recursive: true, force: true }).catch(() => {}); - }), - ); + const config = getS3Config(); + const nsRootPrefix = `${config.prefix}/documents_v1/ns/`; + await deleteDocumentPrefix(nsRootPrefix); } + diff --git a/tests/unit/transfer-user-documents.spec.ts b/tests/unit/transfer-user-documents.spec.ts index 290c3f5..715bf5f 100644 --- a/tests/unit/transfer-user-documents.spec.ts +++ b/tests/unit/transfer-user-documents.spec.ts @@ -7,8 +7,8 @@ import { transferUserDocuments } from '../../src/lib/server/claim-data'; test.describe('transferUserDocuments', () => { test('moves document rows to new user without PK conflicts', async () => { - process.env.BETTER_AUTH_URL = 'http://localhost:3003'; - process.env.BETTER_AUTH_SECRET = 'test-secret'; + process.env.BASE_URL = 'http://localhost:3003'; + process.env.AUTH_SECRET = 'test-secret'; const sqlite = new Database(':memory:'); sqlite.exec(` @@ -72,4 +72,3 @@ test.describe('transferUserDocuments', () => { expect(ids).toEqual(['doc-a', 'doc-b']); }); }); -