refactor(audiobooks): migrate audiobook pipeline to object storage
This commit restructures the audiobook generation and serving layer to rely exclusively on S3-compatible blob storage, removing the dependency on local filesystem paths. - Bundle `ffmpeg` and `ffprobe` binaries via npm to ensure portability across environments. - Update API routes to stream audio directly from blob storage instead of local disk. - Remove legacy migration endpoints and filesystem-based indexing logic. - Add startup scripts to facilitate the transition from local to remote storage. BREAKING CHANGE: Audiobook functionality is now strictly dependent on S3 configuration. The previous filesystem-based storage method has been removed.
This commit is contained in:
Richard R
parent
91bcf232e1
commit
9b9206f50d
49 changed files with 2968 additions and 2573 deletions
33
.env.example
33
.env.example
|
|
@ -1,5 +1,7 @@
|
|||
# Determins the environment mode, `production` limits many features
|
||||
NEXT_PUBLIC_NODE_ENV=development # Not availble in Docker containers (due to being build-time only)
|
||||
# (Optional) Client feature flags (does not work in Docker containers due to being build-time only)
|
||||
NEXT_PUBLIC_NODE_ENV=development
|
||||
NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=
|
||||
NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=
|
||||
|
||||
# Local / OpenAI TTS API Configuration (default)
|
||||
# Suggest using https://github.com/remsky/Kokoro-FastAPI
|
||||
|
|
@ -23,9 +25,6 @@ API_KEY=api_key_optional
|
|||
# TTS_IP_DAILY_LIMIT_ANONYMOUS=100000
|
||||
# TTS_IP_DAILY_LIMIT_AUTHENTICATED=1000000
|
||||
|
||||
# Path to your local whisper.cpp CLI binary for STT timestamp generation
|
||||
WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli
|
||||
|
||||
# Auth configuration (recommended for contributors and public instances)
|
||||
# (Optional) Auth is only enabled when AUTH_SECRET and BASE_URL are set
|
||||
BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network)
|
||||
|
|
@ -37,28 +36,40 @@ GITHUB_CLIENT_SECRET=
|
|||
# (Optional) Disable Better Auth built-in rate limiting (useful for testing)
|
||||
DISABLE_AUTH_RATE_LIMIT=
|
||||
|
||||
# Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables.
|
||||
# (Optional) Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables.
|
||||
# Defaults to SQLite at docstore/sqlite3.db when not set.
|
||||
POSTGRES_URL=
|
||||
# (Optional) Skip automatic startup migrations when set to `false` (default: `true`)
|
||||
RUN_DB_MIGRATIONS=
|
||||
|
||||
# Embedded SeaweedFS weed mini config
|
||||
# (Optional) Enable embedded weed mini for local S3-compatible storage (default: `true`)
|
||||
USE_EMBEDDED_WEED_MINI=
|
||||
WEED_MINI_DIR=
|
||||
WEED_MINI_WAIT_SEC=
|
||||
|
||||
# S3 storage config (use with embedded weed mini or external S3-compatible storage)
|
||||
# For embedded weed mini, set explicit keys if you want stable credentials across restarts.
|
||||
# (Optional) For embedded weed mini, set explicit keys if you want stable credentials across restarts.
|
||||
S3_ACCESS_KEY_ID=
|
||||
S3_SECRET_ACCESS_KEY=
|
||||
S3_BUCKET=
|
||||
S3_REGION=
|
||||
# If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host.
|
||||
# (Optional) If empty in embedded mode, OpenReader uses BASE_URL host (when set) or detected LAN host.
|
||||
S3_ENDPOINT=
|
||||
S3_FORCE_PATH_STYLE=
|
||||
S3_PREFIX=
|
||||
|
||||
# Server library import roots (optional, uses /docstore/library by default)
|
||||
# Migrations configuration
|
||||
# (Optional) Skip automatic startup migrations when set to `false` (default: `true`)
|
||||
RUN_DRIZZLE_MIGRATIONS=
|
||||
# (Optional) Skip automatic filesystem->S3/DB migration pass when set to `false` (default: `true`)
|
||||
RUN_FS_MIGRATIONS=
|
||||
|
||||
# (Optional) Server library import roots (uses /docstore/library by default)
|
||||
IMPORT_LIBRARY_DIR=
|
||||
IMPORT_LIBRARY_DIRS=
|
||||
|
||||
# (Required without Docker) Path to your local whisper.cpp CLI binary for STT timestamp generation
|
||||
WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli
|
||||
|
||||
# (Optional) Override ffmpeg/ffprobe binary paths used for audiobook processing/probing
|
||||
FFMPEG_BIN=
|
||||
FFPROBE_BIN=
|
||||
|
|
|
|||
|
|
@ -44,9 +44,9 @@ RUN pnpm build
|
|||
FROM node:lts-alpine AS runner
|
||||
|
||||
# Add runtime OS dependencies:
|
||||
# - ffmpeg: required for audiobook export and word-by-word alignment (/api/whisper)
|
||||
# - libreoffice-writer: required for DOCX → PDF conversion
|
||||
RUN apk add --no-cache ca-certificates ffmpeg libreoffice-writer
|
||||
# ffmpeg/ffprobe are provided by ffmpeg-static/ffprobe-static from node_modules.
|
||||
RUN apk add --no-cache ca-certificates libreoffice-writer
|
||||
|
||||
# Install pnpm globally for running the app
|
||||
RUN npm install -g pnpm
|
||||
|
|
|
|||
|
|
@ -29,9 +29,10 @@ OpenReader WebUI is an open source, self-host-friendly text-to-speech document r
|
|||
| Goal | Link |
|
||||
| --- | --- |
|
||||
| Run with Docker | [Docker Quick Start](https://docs.openreader.richardr.dev/getting-started/docker-quick-start) |
|
||||
| Deploy on Vercel | [Vercel Deployment](https://docs.openreader.richardr.dev/getting-started/vercel-deployment) |
|
||||
| Develop locally | [Local Development](https://docs.openreader.richardr.dev/getting-started/local-development) |
|
||||
| Configure auth | [Auth](https://docs.openreader.richardr.dev/guides/configuration) |
|
||||
| Configure SQL database | [SQL Database](https://docs.openreader.richardr.dev/operations/database-and-migrations) |
|
||||
| Configure SQL database | [Database and Migrations](https://docs.openreader.richardr.dev/operations/database-and-migrations) |
|
||||
| Configure object storage | [Object / Blob Storage](https://docs.openreader.richardr.dev/guides/storage-and-blob-behavior) |
|
||||
| Configure TTS providers | [TTS Providers](https://docs.openreader.richardr.dev/guides/tts-providers) |
|
||||
| Run Kokoro locally | [Kokoro-FastAPI](https://docs.openreader.richardr.dev/integrations/kokoro-fastapi) |
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ This page covers application-level configuration for provider access and authent
|
|||
|
||||
## Related docs
|
||||
|
||||
- For the complete variable reference: [Environment Variables](./environment-variables)
|
||||
- For the complete variable reference: [Environment Variables](../reference/environment-variables)
|
||||
- For TTS character limits and quota behavior: [TTS Rate Limiting](./tts-rate-limiting)
|
||||
- For provider-specific guidance: [TTS Providers](./tts-providers)
|
||||
- For storage/S3/SeaweedFS behavior: [Object / Blob Storage](./storage-and-blob-behavior)
|
||||
- For database mode and migration commands: [SQL Database](../operations/database-and-migrations)
|
||||
- For database mode and migration commands: [Database and Migrations](./database-and-migrations)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
---
|
||||
title: SQL Database
|
||||
title: Database and Migrations
|
||||
---
|
||||
|
||||
This page covers database mode selection and migration behavior for OpenReader WebUI.
|
||||
|
|
@ -11,17 +11,23 @@ This page covers database mode selection and migration behavior for OpenReader W
|
|||
|
||||
## Startup migration behavior
|
||||
|
||||
By default, the shared entrypoint runs DB migrations automatically before app startup in:
|
||||
By default, the shared entrypoint runs migrations automatically before app startup in:
|
||||
|
||||
- Docker container startup
|
||||
- `pnpm dev`
|
||||
- `pnpm start`
|
||||
|
||||
Startup migration phases:
|
||||
|
||||
- DB schema migrations (`pnpm migrate`)
|
||||
- Storage/data migration (`pnpm migrate-fs`) for legacy filesystem content into S3 + DB rows
|
||||
|
||||
To skip automatic startup migrations:
|
||||
|
||||
- Set `RUN_DB_MIGRATIONS=false`
|
||||
- Set `RUN_DRIZZLE_MIGRATIONS=false`
|
||||
- Set `RUN_FS_MIGRATIONS=false`
|
||||
|
||||
Database variables are documented in [Environment Variables](../guides/environment-variables).
|
||||
Database variables are documented in [Environment Variables](../reference/environment-variables).
|
||||
|
||||
## Common project commands
|
||||
|
||||
|
|
@ -31,6 +37,12 @@ In most cases, you do not need manual migration commands because startup runs mi
|
|||
# Run pending migrations (uses Postgres config when POSTGRES_URL is set, otherwise SQLite)
|
||||
pnpm migrate
|
||||
|
||||
# Run storage migration (filesystem -> S3 + DB)
|
||||
pnpm migrate-fs
|
||||
|
||||
# Dry-run storage migration without uploading/deleting
|
||||
pnpm migrate-fs:dry-run
|
||||
|
||||
# Generate new migration files for both SQLite and Postgres outputs
|
||||
pnpm generate
|
||||
```
|
||||
|
|
@ -39,14 +51,14 @@ pnpm generate
|
|||
|
||||
```bash
|
||||
# Migrate SQLite
|
||||
npx drizzle-kit migrate --config drizzle.config.sqlite.ts
|
||||
pnpm exec drizzle-kit migrate --config drizzle.config.sqlite.ts
|
||||
|
||||
# Migrate Postgres
|
||||
npx drizzle-kit migrate --config drizzle.config.pg.ts
|
||||
pnpm exec drizzle-kit migrate --config drizzle.config.pg.ts
|
||||
|
||||
# Generate SQLite migrations
|
||||
npx drizzle-kit generate --config drizzle.config.sqlite.ts
|
||||
pnpm exec drizzle-kit generate --config drizzle.config.sqlite.ts
|
||||
|
||||
# Generate Postgres migrations
|
||||
npx drizzle-kit generate --config drizzle.config.pg.ts
|
||||
pnpm exec drizzle-kit generate --config drizzle.config.pg.ts
|
||||
```
|
||||
|
|
@ -9,7 +9,7 @@ This page documents storage backends, blob upload routing, and Docker mount beha
|
|||
- Default: embedded SQLite metadata + embedded SeaweedFS (`weed mini`) blobs
|
||||
- External option: Postgres + external S3-compatible object storage
|
||||
|
||||
Storage variables are documented in [Environment Variables](./environment-variables) under the storage sections.
|
||||
Storage variables are documented in [Environment Variables](../reference/environment-variables) under the storage sections.
|
||||
|
||||
## Ports
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ Storage variables are documented in [Environment Variables](./environment-variab
|
|||
|
||||
| Mount | Type | Recommended | Purpose | Example |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `/app/docstore` | Docker named volume | Yes (for persistence) | Persists SeaweedFS blob data, SQLite metadata DB, audiobook artifacts, migration/runtime files | `-v openreader_docstore:/app/docstore` |
|
||||
| `/app/docstore` | Docker named volume | Yes (for persistence) | Persists SeaweedFS blob data, SQLite metadata DB, migrations, and local runtime temp state | `-v openreader_docstore:/app/docstore` |
|
||||
| `/app/docstore/library` | Bind mount | Optional + `:ro` | Read-only source for server library import (files are copied/imported into client storage) | `-v /path/to/your/library:/app/docstore/library:ro` |
|
||||
|
||||
To import from mounted library: **Settings -> Documents -> Server Library Import**.
|
||||
|
|
@ -42,3 +42,8 @@ If `8333` is not published externally:
|
|||
- Document uploads still work through upload fallback proxy
|
||||
- Reads/snippets continue through app API routes
|
||||
- Direct presigned browser upload/download to embedded endpoint is unavailable
|
||||
|
||||
## Audiobook storage note
|
||||
|
||||
- In current versions, audiobook assets live in object storage (`audiobooks_v1` keyspace), not as durable files under `/app/docstore`.
|
||||
- Local filesystem usage for audiobook routes is temporary processing only.
|
||||
|
|
@ -42,7 +42,7 @@ If your provider exposes this interface, it can be used as an OpenAI-compatible
|
|||
3. Set `API_KEY` if required by your provider.
|
||||
4. Choose model and voice.
|
||||
|
||||
For environment variables, see [Environment Variables](./environment-variables).
|
||||
For environment variables, see [Environment Variables](../reference/environment-variables).
|
||||
For TTS quota behavior, see [TTS Rate Limiting](./tts-rate-limiting).
|
||||
For auth behavior, see [Auth](./configuration).
|
||||
For provider-specific integration guides, see [Kokoro-FastAPI](../integrations/kokoro-fastapi), [Orpheus-FastAPI](../integrations/orpheus-fastapi), [Deepinfra](../integrations/deepinfra), [OpenAI](../integrations/openai), and [Custom OpenAI](../integrations/custom-openai).
|
||||
|
|
@ -46,6 +46,6 @@ IP backstop daily limits:
|
|||
|
||||
## Related docs
|
||||
|
||||
- Full variable list: [Environment Variables](./environment-variables)
|
||||
- Full variable list: [Environment Variables](../reference/environment-variables)
|
||||
- Auth configuration: [Auth](./configuration)
|
||||
- Provider setup: [TTS Providers](./tts-providers)
|
||||
|
|
@ -25,4 +25,4 @@ Your provider should expose:
|
|||
|
||||
- `API_BASE` is required for this provider path because OpenReader cannot infer your custom host.
|
||||
- If voices do not load, verify the `/v1/audio/voices` response format.
|
||||
- For variable details, see [Environment Variables](../guides/environment-variables).
|
||||
- For variable details, see [Environment Variables](../reference/environment-variables).
|
||||
|
|
|
|||
|
|
@ -16,4 +16,4 @@ Use Deepinfra as a hosted OpenAI-compatible TTS provider.
|
|||
|
||||
- `Deepinfra` is a built-in provider in the dropdown, so `API_BASE` is usually not required.
|
||||
- Deepinfra supports multiple TTS models, including Kokoro-family options.
|
||||
- For variable details, see [Environment Variables](../guides/environment-variables).
|
||||
- For variable details, see [Environment Variables](../reference/environment-variables).
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@ Use OpenAI directly as an OpenAI-compatible TTS provider.
|
|||
|
||||
- `OpenAI` is a built-in provider in the dropdown, so `API_BASE` is usually not required.
|
||||
- OpenReader routes TTS calls through its server API.
|
||||
- For variable details, see [Environment Variables](../guides/environment-variables).
|
||||
- For variable details, see [Environment Variables](../reference/environment-variables).
|
||||
|
|
|
|||
|
|
@ -20,4 +20,4 @@ Use Orpheus-FastAPI as an OpenAI-compatible TTS backend for OpenReader.
|
|||
|
||||
- `API_BASE` is needed here because Orpheus is configured through the custom provider path.
|
||||
- OpenReader expects OpenAI-compatible audio endpoints.
|
||||
- For variable details, see [Environment Variables](../guides/environment-variables).
|
||||
- For variable details, see [Environment Variables](../reference/environment-variables).
|
||||
|
|
|
|||
|
|
@ -27,13 +27,13 @@ It supports multiple TTS providers including OpenAI, DeepInfra, and custom OpenA
|
|||
|
||||
## Start Here
|
||||
|
||||
- [Docker Quick Start](./getting-started/docker-quick-start)
|
||||
- [Local Development](./getting-started/local-development)
|
||||
- [Environment Variables](./guides/environment-variables)
|
||||
- [Auth](./guides/configuration)
|
||||
- [SQL Database](./operations/database-and-migrations)
|
||||
- [Object / Blob Storage](./guides/storage-and-blob-behavior)
|
||||
- [TTS Providers](./guides/tts-providers)
|
||||
- [Docker Quick Start](./start-here/docker-quick-start)
|
||||
- [Local Development](./start-here/local-development)
|
||||
- [Environment Variables](./reference/environment-variables)
|
||||
- [Auth](./configure/configuration)
|
||||
- [Database and Migrations](./configure/database-and-migrations)
|
||||
- [Object / Blob Storage](./configure/storage-and-blob-behavior)
|
||||
- [TTS Providers](./configure/tts-providers)
|
||||
|
||||
## Source Repository
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ This is the single reference page for OpenReader WebUI environment variables.
|
|||
| Variable | Area | Default | When to set |
|
||||
| --- | --- | --- | --- |
|
||||
| `NEXT_PUBLIC_NODE_ENV` | Runtime mode | `development` | Set `production` for production builds |
|
||||
| `NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT` | Client feature flags | `true` unless set to `false` | Disable audiobook export UI in any environment |
|
||||
| `NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT` | Client feature flags | `true` in dev, `false` in production | Force-enable word highlight UI in production |
|
||||
| `API_BASE` | TTS provider | none | Point to your OpenAI-compatible TTS base URL |
|
||||
| `API_KEY` | TTS provider | `none` fallback in TTS route | Set when provider requires auth |
|
||||
| `TTS_CACHE_MAX_SIZE_BYTES` | TTS caching | `268435456` (256 MB) | Tune in-memory TTS cache size |
|
||||
|
|
@ -23,7 +25,6 @@ This is the single reference page for OpenReader WebUI environment variables.
|
|||
| `TTS_DAILY_LIMIT_AUTHENTICATED` | Rate limiting | `500000` | Override authenticated per-user daily character limit |
|
||||
| `TTS_IP_DAILY_LIMIT_ANONYMOUS` | Rate limiting | `100000` | Override anonymous IP backstop daily limit |
|
||||
| `TTS_IP_DAILY_LIMIT_AUTHENTICATED` | Rate limiting | `1000000` | Override authenticated IP backstop daily limit |
|
||||
| `WHISPER_CPP_BIN` | Word timing | unset | Set to enable `whisper.cpp` timestamps |
|
||||
| `BASE_URL` | Auth | unset | Required (with `AUTH_SECRET`) to enable auth |
|
||||
| `AUTH_SECRET` | Auth | unset | Required (with `BASE_URL`) to enable auth |
|
||||
| `AUTH_TRUSTED_ORIGINS` | Auth | empty | Add extra allowed origins |
|
||||
|
|
@ -31,7 +32,6 @@ This is the single reference page for OpenReader WebUI environment variables.
|
|||
| `GITHUB_CLIENT_SECRET` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_ID` to enable GitHub sign-in |
|
||||
| `DISABLE_AUTH_RATE_LIMIT` | Rate limiting | `false` | Set `true` to disable auth-layer rate limiting |
|
||||
| `POSTGRES_URL` | Database | unset (SQLite mode) | Set to switch metadata/auth DB to Postgres |
|
||||
| `RUN_DB_MIGRATIONS` | Database | `true` | Set `false` to skip startup migrations |
|
||||
| `USE_EMBEDDED_WEED_MINI` | Storage | `true` when unset | Set `false` to use external S3-compatible storage only |
|
||||
| `WEED_MINI_DIR` | Storage | `docstore/seaweedfs` | Override embedded SeaweedFS data directory |
|
||||
| `WEED_MINI_WAIT_SEC` | Storage | `20` | Tune SeaweedFS startup wait timeout |
|
||||
|
|
@ -42,8 +42,13 @@ This is the single reference page for OpenReader WebUI environment variables.
|
|||
| `S3_ENDPOINT` | Storage | derived in embedded mode | Set for S3-compatible providers (MinIO/SeaweedFS/R2/etc.) |
|
||||
| `S3_FORCE_PATH_STYLE` | Storage | `true` in embedded mode | Set per provider requirement |
|
||||
| `S3_PREFIX` | Storage | `openreader` | Customize object key prefix |
|
||||
| `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations |
|
||||
| `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass |
|
||||
| `IMPORT_LIBRARY_DIR` | Library import | `docstore/library` fallback | Set a single server library root |
|
||||
| `IMPORT_LIBRARY_DIRS` | Library import | unset | Set multiple roots (comma/colon/semicolon separated) |
|
||||
| `WHISPER_CPP_BIN` | Word timing | unset | Set to enable `whisper.cpp` timestamps |
|
||||
| `FFMPEG_BIN` | Audio runtime | auto-detected (`ffmpeg-static`) | Override ffmpeg binary path |
|
||||
| `FFPROBE_BIN` | Audio runtime | auto-detected (`ffprobe-static`) | Override ffprobe binary path |
|
||||
|
||||
## Detailed Reference
|
||||
|
||||
|
|
@ -54,6 +59,21 @@ Controls development vs production behavior in client/server code paths.
|
|||
- Typical values: `development`, `production`
|
||||
- In production builds, set `production`
|
||||
|
||||
### NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT
|
||||
|
||||
Controls whether audiobook export UI/actions are shown in the client.
|
||||
|
||||
- Default behavior: enabled unless explicitly set to `false`
|
||||
- Applies in both development and production
|
||||
|
||||
### NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT
|
||||
|
||||
Controls word-by-word highlighting UI in production builds.
|
||||
|
||||
- Development default: enabled
|
||||
- Production default: disabled unless set to `true`
|
||||
- Requires working timestamp generation (for example `WHISPER_CPP_BIN`)
|
||||
|
||||
### API_BASE
|
||||
|
||||
Base URL for OpenAI-compatible TTS API requests.
|
||||
|
|
@ -110,7 +130,7 @@ Controls TTS character rate limiting in the TTS API.
|
|||
|
||||
- Default: `false` (TTS char limits disabled)
|
||||
- Set to `true` to enforce `TTS_DAILY_LIMIT_*` and `TTS_IP_DAILY_LIMIT_*`
|
||||
- For behavior details and examples, see [TTS Rate Limiting](./tts-rate-limiting)
|
||||
- For behavior details and examples, see [TTS Rate Limiting](../configure/tts-rate-limiting)
|
||||
|
||||
### TTS_DAILY_LIMIT_ANONYMOUS
|
||||
|
||||
|
|
@ -136,13 +156,6 @@ Authenticated IP backstop daily character limit.
|
|||
|
||||
- Default: `1000000`
|
||||
|
||||
### WHISPER_CPP_BIN
|
||||
|
||||
Absolute path to compiled `whisper.cpp` binary for word-level timestamps.
|
||||
|
||||
- Example: `/whisper.cpp/build/bin/whisper-cli`
|
||||
- Required only for optional word-by-word highlighting
|
||||
|
||||
### BASE_URL
|
||||
|
||||
External base URL for this OpenReader instance.
|
||||
|
|
@ -191,13 +204,6 @@ Switches metadata/auth storage from SQLite to Postgres.
|
|||
- Unset: SQLite at `docstore/sqlite3.db`
|
||||
- Set: Postgres mode
|
||||
|
||||
### RUN_DB_MIGRATIONS
|
||||
|
||||
Controls startup migration execution in shared entrypoint.
|
||||
|
||||
- Default: `true`
|
||||
- Set `false` to skip automatic startup migrations
|
||||
|
||||
### USE_EMBEDDED_WEED_MINI
|
||||
|
||||
Controls embedded SeaweedFS startup.
|
||||
|
|
@ -265,6 +271,21 @@ Prefix prepended to stored object keys.
|
|||
|
||||
- Default: `openreader`
|
||||
|
||||
### RUN_DRIZZLE_MIGRATIONS
|
||||
|
||||
Controls startup migration execution in shared entrypoint.
|
||||
|
||||
- Default: `true`
|
||||
- Set `false` to skip automatic startup Drizzle schema migrations
|
||||
|
||||
### RUN_FS_MIGRATIONS
|
||||
|
||||
Controls startup filesystem-to-object-store migration execution in shared entrypoint.
|
||||
|
||||
- Default: `true`
|
||||
- Runs `scripts/migrate-fs-v2.mjs` at startup after DB migrations
|
||||
- Set `false` to skip automatic storage migration pass
|
||||
|
||||
### IMPORT_LIBRARY_DIR
|
||||
|
||||
Single directory root for server library import.
|
||||
|
|
@ -278,3 +299,24 @@ Multiple library roots for server library import.
|
|||
|
||||
- Separator: comma, colon, or semicolon
|
||||
- Takes precedence over `IMPORT_LIBRARY_DIR`
|
||||
|
||||
### WHISPER_CPP_BIN
|
||||
|
||||
Absolute path to compiled `whisper.cpp` binary for word-level timestamps.
|
||||
|
||||
- Example: `/whisper.cpp/build/bin/whisper-cli`
|
||||
- Required only for optional word-by-word highlighting
|
||||
|
||||
### FFMPEG_BIN
|
||||
|
||||
Absolute path or executable name for the ffmpeg binary used by audiobook/processing routes.
|
||||
|
||||
- Resolution order: `FFMPEG_BIN` -> `ffmpeg-static`
|
||||
- Example: `/var/task/node_modules/ffmpeg-static/ffmpeg`
|
||||
|
||||
### FFPROBE_BIN
|
||||
|
||||
Absolute path or executable name for the ffprobe binary used for audio probing.
|
||||
|
||||
- Resolution order: `FFPROBE_BIN` -> `ffprobe-static`
|
||||
- Example: `/var/task/node_modules/ffprobe-static/bin/linux/x64/ffprobe`
|
||||
|
|
@ -50,10 +50,10 @@ Quick notes:
|
|||
- To enable auth, set both `BASE_URL` and `AUTH_SECRET`.
|
||||
- DB migrations run automatically during container startup via the shared entrypoint.
|
||||
|
||||
For all environment variables, see [Environment Variables](../guides/environment-variables).
|
||||
For app/auth behavior, see [Auth](../guides/configuration).
|
||||
For database startup and migration behavior, see [SQL Database](../operations/database-and-migrations).
|
||||
For blob behavior and mounts, see [Object / Blob Storage](../guides/storage-and-blob-behavior).
|
||||
For all environment variables, see [Environment Variables](../reference/environment-variables).
|
||||
For app/auth behavior, see [Auth](../configure/configuration).
|
||||
For database startup and migration behavior, see [Database and Migrations](../configure/database-and-migrations).
|
||||
For blob behavior and mounts, see [Object / Blob Storage](../configure/storage-and-blob-behavior).
|
||||
|
||||
## 2. Configure settings in the app UI
|
||||
|
||||
|
|
@ -20,12 +20,6 @@ brew install seaweedfs
|
|||
|
||||
Optional, depending on features:
|
||||
|
||||
- [FFmpeg](https://ffmpeg.org) (required for `m4b` audiobook generation)
|
||||
|
||||
```bash
|
||||
brew install ffmpeg
|
||||
```
|
||||
|
||||
- [libreoffice](https://www.libreoffice.org) (required for DOCX conversion)
|
||||
|
||||
```bash
|
||||
|
|
@ -83,10 +77,10 @@ Optional:
|
|||
- Stable S3 credentials via `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`
|
||||
- External S3 storage by setting `USE_EMBEDDED_WEED_MINI=false` and related S3 vars
|
||||
|
||||
For all environment variables, see [Environment Variables](../guides/environment-variables).
|
||||
For app/auth behavior, see [Auth](../guides/configuration).
|
||||
For storage configuration, see [Object / Blob Storage](../guides/storage-and-blob-behavior).
|
||||
For database mode and migrations, see [SQL Database](../operations/database-and-migrations).
|
||||
For all environment variables, see [Environment Variables](../reference/environment-variables).
|
||||
For app/auth behavior, see [Auth](../configure/configuration).
|
||||
For storage configuration, see [Object / Blob Storage](../configure/storage-and-blob-behavior).
|
||||
For database mode and migrations, see [Database and Migrations](../configure/database-and-migrations).
|
||||
|
||||
4. Run DB migrations.
|
||||
|
||||
|
|
@ -98,7 +92,7 @@ pnpm migrate
|
|||
```
|
||||
|
||||
:::note
|
||||
If `POSTGRES_URL` is set, migrations target Postgres; otherwise local SQLite is used. To disable automatic startup migrations, set `RUN_DB_MIGRATIONS=false`.
|
||||
If `POSTGRES_URL` is set, migrations target Postgres; otherwise local SQLite is used. To disable automatic startup migrations, set `RUN_DRIZZLE_MIGRATIONS=false` and/or `RUN_FS_MIGRATIONS=false`. You can run storage migration manually with `pnpm migrate-fs`.
|
||||
:::
|
||||
|
||||
5. Start the app.
|
||||
78
docs-site/docs/start-here/vercel-deployment.md
Normal file
78
docs-site/docs/start-here/vercel-deployment.md
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
title: Vercel Deployment
|
||||
---
|
||||
|
||||
This guide covers deploying OpenReader WebUI to Vercel with external Postgres and S3-compatible object storage.
|
||||
|
||||
## What works on Vercel
|
||||
|
||||
- Documents (PDF/EPUB/TXT/MD) work with `POSTGRES_URL` + external S3 storage.
|
||||
- Audiobook routes work on Node.js serverless functions using `ffmpeg-static`/`ffprobe-static`.
|
||||
- `docx` conversion requires `soffice` (LibreOffice), which is not available in a standard Vercel runtime.
|
||||
|
||||
## 1. Required environment variables
|
||||
|
||||
Set these in your Vercel project:
|
||||
|
||||
```bash
|
||||
POSTGRES_URL=postgres://...
|
||||
USE_EMBEDDED_WEED_MINI=false
|
||||
S3_ACCESS_KEY_ID=...
|
||||
S3_SECRET_ACCESS_KEY=...
|
||||
S3_BUCKET=...
|
||||
S3_REGION=us-east-1
|
||||
S3_ENDPOINT=https://...
|
||||
S3_FORCE_PATH_STYLE=true
|
||||
S3_PREFIX=openreader
|
||||
```
|
||||
|
||||
Optional but common:
|
||||
|
||||
```bash
|
||||
BASE_URL=https://your-app.vercel.app
|
||||
AUTH_SECRET=...
|
||||
NEXT_PUBLIC_NODE_ENV=production
|
||||
NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true
|
||||
NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=true
|
||||
```
|
||||
|
||||
For all variables and defaults, see [Environment Variables](../reference/environment-variables).
|
||||
|
||||
## 2. FFmpeg/ffprobe packaging in Vercel functions
|
||||
|
||||
`ffmpeg-static` and `ffprobe-static` binaries must be included in function traces. This repo already does that in `next.config.ts` via `outputFileTracingIncludes` for:
|
||||
|
||||
- `/api/audiobook(.*)`
|
||||
- `/api/whisper`
|
||||
|
||||
If you change route paths or split handlers, update `outputFileTracingIncludes` accordingly.
|
||||
|
||||
## 3. Function memory sizing
|
||||
|
||||
FFmpeg workloads benefit from more memory/CPU. This repo includes:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://openapi.vercel.sh/vercel.json",
|
||||
"functions": {
|
||||
"app/api/audiobook/route.ts": { "memory": 3009 },
|
||||
"app/api/whisper/route.ts": { "memory": 3009 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Adjust memory per route if your files are larger or your plan differs.
|
||||
|
||||
## 4. Runtime expectations and caveats
|
||||
|
||||
- Audiobook APIs require S3 configuration; otherwise they return `503`.
|
||||
- `better-sqlite3` remains in `serverExternalPackages` for mixed/self-host setups, but production Vercel should use `POSTGRES_URL`.
|
||||
- Filesystem-to-object-store migrations run via server scripts/entrypoint (`scripts/migrate-fs-v2.mjs`), not API routes.
|
||||
- Vercel deployments do not run `scripts/openreader-entrypoint.mjs`, so run `pnpm migrate-fs` in a controlled environment when migrating legacy filesystem data.
|
||||
|
||||
## 5. Smoke test after deploy
|
||||
|
||||
1. Upload and read a PDF/EPUB document.
|
||||
2. Confirm sync/blob fetch works across refreshes/devices.
|
||||
3. Generate at least one audiobook chapter and play/download it.
|
||||
4. If using word highlighting, verify timestamps are produced and rendered.
|
||||
|
|
@ -5,6 +5,7 @@ import type * as Preset from '@docusaurus/preset-classic';
|
|||
const config: Config = {
|
||||
title: 'OpenReader WebUI Docs',
|
||||
tagline: 'Docs for OpenReader',
|
||||
favicon: 'favicon.ico',
|
||||
|
||||
future: {
|
||||
v4: true,
|
||||
|
|
@ -97,7 +98,7 @@ const config: Config = {
|
|||
{
|
||||
title: 'Community',
|
||||
items: [
|
||||
{ label: 'Support', to: '/community/support' },
|
||||
{ label: 'Support', to: '/project/support' },
|
||||
{ label: 'GitHub Discussions', href: 'https://github.com/richardr1126/OpenReader-WebUI/discussions' },
|
||||
{ label: 'Issues', href: 'https://github.com/richardr1126/OpenReader-WebUI/issues' },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ const sidebars: SidebarsConfig = {
|
|||
{
|
||||
type: 'category',
|
||||
label: 'Start Here',
|
||||
items: ['getting-started/docker-quick-start', 'getting-started/local-development'],
|
||||
items: ['start-here/docker-quick-start', 'start-here/vercel-deployment', 'start-here/local-development'],
|
||||
},
|
||||
{
|
||||
type: 'category',
|
||||
label: 'Reference',
|
||||
items: [
|
||||
'guides/environment-variables',
|
||||
'reference/environment-variables',
|
||||
'reference/stack',
|
||||
],
|
||||
},
|
||||
|
|
@ -20,15 +20,15 @@ const sidebars: SidebarsConfig = {
|
|||
type: 'category',
|
||||
label: 'Configure',
|
||||
items: [
|
||||
'guides/tts-providers',
|
||||
'configure/tts-providers',
|
||||
{
|
||||
type: 'doc',
|
||||
id: 'guides/configuration',
|
||||
id: 'configure/configuration',
|
||||
label: 'Auth (Reccomended)',
|
||||
},
|
||||
'guides/tts-rate-limiting',
|
||||
'operations/database-and-migrations',
|
||||
'guides/storage-and-blob-behavior',
|
||||
'configure/tts-rate-limiting',
|
||||
'configure/database-and-migrations',
|
||||
'configure/storage-and-blob-behavior',
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
@ -45,7 +45,7 @@ const sidebars: SidebarsConfig = {
|
|||
{
|
||||
type: 'category',
|
||||
label: 'Project',
|
||||
items: ['community/support', 'community/acknowledgements', 'community/license'],
|
||||
items: ['project/support', 'project/acknowledgements', 'project/license'],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
|
|
|||
BIN
docs-site/static/favicon.ico
Normal file
BIN
docs-site/static/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
|
|
@ -7,6 +7,18 @@ const nextConfig: NextConfig = {
|
|||
},
|
||||
},
|
||||
serverExternalPackages: ["better-sqlite3"],
|
||||
outputFileTracingIncludes: {
|
||||
'/api/audiobook(.*)': [
|
||||
'node_modules/ffmpeg-static/ffmpeg',
|
||||
'node_modules/ffprobe-static/bin/linux/*/ffprobe',
|
||||
'node_modules/.pnpm/ffmpeg-static@*/node_modules/ffmpeg-static/ffmpeg',
|
||||
'node_modules/.pnpm/ffprobe-static@*/node_modules/ffprobe-static/bin/linux/*/ffprobe',
|
||||
],
|
||||
'/api/whisper': [
|
||||
'node_modules/ffmpeg-static/ffmpeg',
|
||||
'node_modules/.pnpm/ffmpeg-static@*/node_modules/ffmpeg-static/ffmpeg',
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
|
|
|
|||
48
package.json
48
package.json
|
|
@ -11,6 +11,8 @@
|
|||
"lint": "next lint",
|
||||
"test": "playwright test",
|
||||
"migrate": "node drizzle/scripts/migrate.mjs",
|
||||
"migrate-fs": "node scripts/migrate-fs-v2.mjs",
|
||||
"migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true",
|
||||
"generate": "node drizzle/scripts/generate.mjs",
|
||||
"docs:init": "pnpm --dir docs-site install",
|
||||
"docs:dev": "pnpm --dir docs-site start",
|
||||
|
|
@ -20,34 +22,34 @@
|
|||
"docs:clear": "pnpm --dir docs-site clear"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.985.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.985.0",
|
||||
"@aws-sdk/client-s3": "^3.987.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.987.0",
|
||||
"@headlessui/react": "^2.2.9",
|
||||
"@types/howler": "^2.2.12",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"@vercel/analytics": "^1.6.1",
|
||||
"better-auth": "^1.4.17",
|
||||
"better-auth": "^1.4.18",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"cmpstr": "^3.2.0",
|
||||
"cmpstr": "^3.2.1",
|
||||
"compromise": "^14.14.5",
|
||||
"core-js": "^3.48.0",
|
||||
"dexie": "^4.2.1",
|
||||
"dexie": "^4.3.0",
|
||||
"dexie-react-hooks": "^4.2.0",
|
||||
"dotenv": "^17.2.3",
|
||||
"drizzle-kit": "^0.31.8",
|
||||
"dotenv": "^17.2.4",
|
||||
"drizzle-kit": "^0.31.9",
|
||||
"drizzle-orm": "^0.45.1",
|
||||
"epubjs": "^0.3.93",
|
||||
"ffmpeg-static": "^5.3.0",
|
||||
"ffprobe-static": "^3.1.0",
|
||||
"howler": "^2.2.4",
|
||||
"lru-cache": "^11.2.4",
|
||||
"next": "^15.5.9",
|
||||
"openai": "^6.16.0",
|
||||
"lru-cache": "^11.2.6",
|
||||
"next": "^15.5.12",
|
||||
"openai": "^6.21.0",
|
||||
"pdfjs-dist": "4.8.69",
|
||||
"pg": "^8.17.2",
|
||||
"react": "^19.2.3",
|
||||
"pg": "^8.18.0",
|
||||
"react": "^19.2.4",
|
||||
"react-dnd": "^16.0.1",
|
||||
"react-dnd-html5-backend": "^16.0.1",
|
||||
"react-dom": "^19.2.3",
|
||||
"react-dropzone": "^14.3.8",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-dropzone": "^14.4.1",
|
||||
"react-hot-toast": "^2.6.0",
|
||||
"react-markdown": "^10.1.0",
|
||||
"react-pdf": "^9.2.1",
|
||||
|
|
@ -57,22 +59,26 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"@eslint/eslintrc": "^3.3.3",
|
||||
"@playwright/test": "^1.58.0",
|
||||
"@playwright/test": "^1.58.2",
|
||||
"@tailwindcss/typography": "^0.5.19",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^20.19.30",
|
||||
"@types/ffprobe-static": "^2.0.3",
|
||||
"@types/node": "^20.19.33",
|
||||
"@types/pg": "^8.16.0",
|
||||
"@types/react": "^19.2.9",
|
||||
"@types/react": "^19.2.13",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@types/howler": "^2.2.12",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.39.2",
|
||||
"eslint-config-next": "^15.5.9",
|
||||
"eslint-config-next": "^15.5.12",
|
||||
"postcss": "^8.5.6",
|
||||
"tailwindcss": "^3.4.19",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"better-sqlite3"
|
||||
"better-sqlite3",
|
||||
"ffmpeg-static"
|
||||
],
|
||||
"overrides": {
|
||||
"lodash": "^4.17.23",
|
||||
|
|
|
|||
1004
pnpm-lock.yaml
1004
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
919
scripts/migrate-fs-v2.mjs
Normal file
919
scripts/migrate-fs-v2.mjs
Normal file
|
|
@ -0,0 +1,919 @@
|
|||
#!/usr/bin/env node
|
||||
import { createHash } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createRequire } from 'node:module';
|
||||
import * as dotenv from 'dotenv';
|
||||
import {
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { Pool } = require('pg');
|
||||
const BetterSqlite3 = require('better-sqlite3');
|
||||
const ffprobeStatic = require('ffprobe-static');
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const DOCUMENTS_V1_DIR = path.join(DOCSTORE_DIR, 'documents_v1');
|
||||
const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1');
|
||||
const UNCLAIMED_USER_ID = 'unclaimed';
|
||||
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const SAFE_AUDIOBOOK_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
|
||||
function loadEnvFiles() {
|
||||
const envPath = path.join(process.cwd(), '.env');
|
||||
const envLocalPath = path.join(process.cwd(), '.env.local');
|
||||
if (fs.existsSync(envPath)) dotenv.config({ path: envPath });
|
||||
if (fs.existsSync(envLocalPath)) dotenv.config({ path: envLocalPath, override: true });
|
||||
}
|
||||
|
||||
function parseBool(value, fallback = false) {
|
||||
if (value == null || String(value).trim() === '') return fallback;
|
||||
const normalized = String(value).trim().toLowerCase();
|
||||
if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
|
||||
if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
dryRun: false,
|
||||
deleteLocal: false,
|
||||
namespace: null,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const raw = argv[i];
|
||||
if (raw === '--dry-run' && argv[i + 1]) {
|
||||
args.dryRun = parseBool(argv[i + 1], true);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (raw.startsWith('--dry-run=')) {
|
||||
args.dryRun = parseBool(raw.slice('--dry-run='.length), true);
|
||||
continue;
|
||||
}
|
||||
if (raw === '--delete-local' && argv[i + 1]) {
|
||||
args.deleteLocal = parseBool(argv[i + 1], true);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (raw.startsWith('--delete-local=')) {
|
||||
args.deleteLocal = parseBool(raw.slice('--delete-local='.length), true);
|
||||
continue;
|
||||
}
|
||||
if (raw === '--namespace' && argv[i + 1]) {
|
||||
args.namespace = sanitizeNamespace(argv[i + 1]);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (raw.startsWith('--namespace=')) {
|
||||
args.namespace = sanitizeNamespace(raw.slice('--namespace='.length));
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function sanitizeNamespace(namespace) {
|
||||
if (!namespace) return null;
|
||||
const safe = String(namespace).trim();
|
||||
if (!safe || !SAFE_NAMESPACE_REGEX.test(safe)) return null;
|
||||
return safe;
|
||||
}
|
||||
|
||||
function applyNamespacePath(baseDir, namespace) {
|
||||
if (!namespace) return baseDir;
|
||||
const resolved = path.resolve(baseDir, namespace);
|
||||
if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function getUnclaimedUserIdForNamespace(namespace) {
|
||||
if (!namespace) return UNCLAIMED_USER_ID;
|
||||
return `${UNCLAIMED_USER_ID}::${namespace}`;
|
||||
}
|
||||
|
||||
function normalizePrefix(prefix) {
|
||||
const base = String(prefix || 'openreader').trim();
|
||||
if (!base) return 'openreader';
|
||||
return base.replace(/^\/+|\/+$/g, '');
|
||||
}
|
||||
|
||||
function parseS3ConfigFromEnv() {
|
||||
const bucket = process.env.S3_BUCKET?.trim();
|
||||
const region = process.env.S3_REGION?.trim();
|
||||
const accessKeyId = process.env.S3_ACCESS_KEY_ID?.trim();
|
||||
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY?.trim();
|
||||
const endpoint = process.env.S3_ENDPOINT?.trim();
|
||||
|
||||
if (!bucket || !region || !accessKeyId || !secretAccessKey) {
|
||||
throw new Error('S3 is not configured. Required env vars: S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.');
|
||||
}
|
||||
|
||||
return {
|
||||
bucket,
|
||||
region,
|
||||
endpoint: endpoint || undefined,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
forcePathStyle: parseBool(process.env.S3_FORCE_PATH_STYLE, false),
|
||||
prefix: normalizePrefix(process.env.S3_PREFIX),
|
||||
};
|
||||
}
|
||||
|
||||
function createS3Client(config) {
|
||||
return new S3Client({
|
||||
region: config.region,
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function isPreconditionFailed(error) {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
return error?.$metadata?.httpStatusCode === 412 || error?.name === 'PreconditionFailed';
|
||||
}
|
||||
|
||||
function documentKey(s3Config, id, namespace) {
|
||||
if (!DOCUMENT_ID_REGEX.test(id)) {
|
||||
throw new Error(`Invalid document id: ${id}`);
|
||||
}
|
||||
const nsSegment = namespace ? `ns/${namespace}/` : '';
|
||||
return `${s3Config.prefix}/documents_v1/${nsSegment}${id}`;
|
||||
}
|
||||
|
||||
function audiobookKey(s3Config, bookId, userId, fileName, namespace) {
|
||||
if (!SAFE_AUDIOBOOK_ID_REGEX.test(bookId)) throw new Error(`Invalid audiobook id: ${bookId}`);
|
||||
if (!userId) throw new Error('Missing user id for audiobook key');
|
||||
if (!fileName || fileName.includes('/') || fileName.includes('\\')) throw new Error(`Invalid audiobook file name: ${fileName}`);
|
||||
const nsSegment = namespace ? `ns/${namespace}/` : '';
|
||||
return `${s3Config.prefix}/audiobooks_v1/${nsSegment}users/${encodeURIComponent(String(userId))}/${bookId}-audiobook/${fileName}`;
|
||||
}
|
||||
|
||||
async function putObjectIfMissing(s3Client, s3Config, key, body, contentType) {
|
||||
await s3Client.send(new PutObjectCommand({
|
||||
Bucket: s3Config.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
IfNoneMatch: '*',
|
||||
}));
|
||||
}
|
||||
|
||||
function isLegacyDocumentMetadata(value) {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const v = value;
|
||||
return typeof v.id === 'string'
|
||||
&& typeof v.name === 'string'
|
||||
&& typeof v.size === 'number'
|
||||
&& typeof v.lastModified === 'number'
|
||||
&& typeof v.type === 'string';
|
||||
}
|
||||
|
||||
function isValidDocumentId(id) {
|
||||
return DOCUMENT_ID_REGEX.test(id);
|
||||
}
|
||||
|
||||
function extractIdFromFileName(fileName) {
|
||||
const match = /^([a-f0-9]{64})__/i.exec(fileName);
|
||||
if (!match) return null;
|
||||
const id = match[1].toLowerCase();
|
||||
return isValidDocumentId(id) ? id : null;
|
||||
}
|
||||
|
||||
function decodeNameFromFileName(fileName, id) {
|
||||
const prefix = `${id}__`;
|
||||
if (!fileName.startsWith(prefix)) return fileName;
|
||||
const encoded = fileName.slice(prefix.length);
|
||||
try {
|
||||
return decodeURIComponent(encoded);
|
||||
} catch {
|
||||
return fileName;
|
||||
}
|
||||
}
|
||||
|
||||
function sniffBinaryDocumentType(bytes) {
|
||||
if (bytes.length >= 5 && bytes.subarray(0, 5).toString('ascii') === '%PDF-') {
|
||||
return 'pdf';
|
||||
}
|
||||
|
||||
const isZip = bytes.length >= 4
|
||||
&& bytes[0] === 0x50
|
||||
&& bytes[1] === 0x4b
|
||||
&& (bytes[2] === 0x03 || bytes[2] === 0x05 || bytes[2] === 0x07)
|
||||
&& (bytes[3] === 0x04 || bytes[3] === 0x06 || bytes[3] === 0x08);
|
||||
if (!isZip) return null;
|
||||
|
||||
const probe = bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString('latin1');
|
||||
if (probe.includes('application/epub+zip') || probe.includes('META-INF/container.xml')) return 'epub';
|
||||
if (probe.includes('[Content_Types].xml') && probe.includes('word/')) return 'docx';
|
||||
return null;
|
||||
}
|
||||
|
||||
function toDocumentTypeFromName(name) {
|
||||
const ext = path.extname(name).toLowerCase();
|
||||
if (ext === '.pdf') return 'pdf';
|
||||
if (ext === '.epub') return 'epub';
|
||||
if (ext === '.docx') return 'docx';
|
||||
return 'html';
|
||||
}
|
||||
|
||||
function safeDocumentName(rawName, fallback) {
|
||||
const baseName = path.basename(rawName || fallback);
|
||||
return baseName.replaceAll('\u0000', '').slice(0, 240) || fallback;
|
||||
}
|
||||
|
||||
function normalizeNameForType(name, id, type) {
|
||||
if (type === 'html') return name;
|
||||
const expectedExt = type === 'pdf' ? '.pdf' : type === 'epub' ? '.epub' : '.docx';
|
||||
if (name.toLowerCase().endsWith(expectedExt)) return name;
|
||||
const base = name.replace(/\.bin$/i, '');
|
||||
return `${base || id}${expectedExt}`;
|
||||
}
|
||||
|
||||
function contentTypeForName(name) {
|
||||
const ext = path.extname(name).toLowerCase();
|
||||
if (ext === '.pdf') return 'application/pdf';
|
||||
if (ext === '.epub') return 'application/epub+zip';
|
||||
if (ext === '.docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
||||
if (ext === '.md' || ext === '.mdown' || ext === '.markdown') return 'text/markdown; charset=utf-8';
|
||||
if (ext === '.html' || ext === '.htm') return 'text/html; charset=utf-8';
|
||||
return 'text/plain; charset=utf-8';
|
||||
}
|
||||
|
||||
function contentTypeForDocument(type, name) {
|
||||
if (type === 'pdf') return 'application/pdf';
|
||||
if (type === 'epub') return 'application/epub+zip';
|
||||
if (type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
||||
return contentTypeForName(name);
|
||||
}
|
||||
|
||||
function sanitizeTagValue(value) {
|
||||
return String(value || '').replaceAll('\u0000', '').replaceAll(/\r?\n/g, ' ').trim();
|
||||
}
|
||||
|
||||
function sanitizeFileStem(value) {
|
||||
return sanitizeTagValue(value)
|
||||
.replaceAll(/[\\/]/g, ' ')
|
||||
.replaceAll(/[<>:"|?*\u0000]/g, '')
|
||||
.replaceAll(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 180);
|
||||
}
|
||||
|
||||
function encodeChapterFileName(index, title, format) {
|
||||
const oneBased = String(index + 1).padStart(4, '0');
|
||||
const safeTitle = sanitizeFileStem(title) || `Chapter ${index + 1}`;
|
||||
return `${oneBased}__${encodeURIComponent(safeTitle)}.${format}`;
|
||||
}
|
||||
|
||||
function decodeChapterFileName(fileName) {
|
||||
const match = /^(\d{1,6})__(.+)\.(mp3|m4b)$/i.exec(fileName);
|
||||
if (!match) return null;
|
||||
const oneBased = Number(match[1]);
|
||||
if (!Number.isInteger(oneBased) || oneBased <= 0) return null;
|
||||
const format = match[3].toLowerCase();
|
||||
try {
|
||||
const title = decodeURIComponent(match[2]);
|
||||
return { index: oneBased - 1, title: title || `Chapter ${oneBased}`, format };
|
||||
} catch {
|
||||
return { index: oneBased - 1, title: match[2], format };
|
||||
}
|
||||
}
|
||||
|
||||
function decodeChapterTitleTag(tag) {
|
||||
const raw = sanitizeTagValue(tag);
|
||||
if (!raw) return null;
|
||||
const match = /^(\d{1,6})\s*[-.:]\s*(.+)$/.exec(raw);
|
||||
if (!match) return null;
|
||||
const oneBased = Number(match[1]);
|
||||
if (!Number.isFinite(oneBased) || !Number.isInteger(oneBased) || oneBased <= 0) return null;
|
||||
return { index: oneBased - 1, title: match[2].trim() || `Chapter ${oneBased}` };
|
||||
}
|
||||
|
||||
function chooseBinary(preferred, bundled, envVarName, packageName) {
|
||||
const envValue = preferred ? String(preferred).trim() : '';
|
||||
if (envValue) {
|
||||
if ((envValue.includes('/') || envValue.includes('\\')) && !fs.existsSync(envValue)) {
|
||||
throw new Error(`${envVarName} points to a missing binary: ${envValue}`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
const bundledValue = bundled ? String(bundled).trim() : '';
|
||||
if (!bundledValue) {
|
||||
throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`);
|
||||
}
|
||||
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !fs.existsSync(bundledValue)) {
|
||||
throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`);
|
||||
}
|
||||
return bundledValue;
|
||||
}
|
||||
|
||||
function getFFprobePath() {
|
||||
return chooseBinary(process.env.FFPROBE_BIN || null, ffprobeStatic?.path || null, 'FFPROBE_BIN', 'ffprobe-static');
|
||||
}
|
||||
|
||||
async function ffprobeTitleTag(filePath) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(getFFprobePath(), [
|
||||
'-v', 'quiet',
|
||||
'-print_format', 'json',
|
||||
'-show_entries', 'format_tags=title',
|
||||
filePath,
|
||||
]);
|
||||
|
||||
let output = '';
|
||||
child.stdout.on('data', (data) => { output += data.toString(); });
|
||||
child.on('error', () => resolve(null));
|
||||
child.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(output);
|
||||
const title = parsed?.format?.tags?.title;
|
||||
resolve(typeof title === 'string' ? title : null);
|
||||
} catch {
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function isPersistedAudiobookFileName(fileName) {
|
||||
if (fileName === 'audiobook.meta.json') return true;
|
||||
if (fileName === 'complete.mp3' || fileName === 'complete.m4b') return true;
|
||||
if (/^complete\.(mp3|m4b)\.manifest\.json$/i.test(fileName)) return true;
|
||||
return decodeChapterFileName(fileName) !== null;
|
||||
}
|
||||
|
||||
function isTransientAudiobookFileName(fileName) {
|
||||
if (/^-?\d+-input\.mp3$/i.test(fileName)) return true;
|
||||
if (/\.tmp\./i.test(fileName)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function contentTypeForAudiobookFileName(fileName) {
|
||||
if (fileName.endsWith('.mp3')) return 'audio/mpeg';
|
||||
if (fileName.endsWith('.m4b')) return 'audio/mp4';
|
||||
if (fileName.endsWith('.json')) return 'application/json; charset=utf-8';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
async function collectDocumentCandidates(docsDir, docstoreDir) {
|
||||
const byId = new Map();
|
||||
let filesScanned = 0;
|
||||
let skippedInvalid = 0;
|
||||
|
||||
if (fs.existsSync(docsDir)) {
|
||||
const entries = await fsp.readdir(docsDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
filesScanned += 1;
|
||||
const fullPath = path.join(docsDir, entry.name);
|
||||
const bytes = await fsp.readFile(fullPath);
|
||||
const st = await fsp.stat(fullPath);
|
||||
|
||||
const extractedId = extractIdFromFileName(entry.name);
|
||||
const id = extractedId ?? createHash('sha256').update(bytes).digest('hex');
|
||||
if (!isValidDocumentId(id)) {
|
||||
skippedInvalid += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const inferredName = decodeNameFromFileName(entry.name, id);
|
||||
const inferredType = toDocumentTypeFromName(inferredName);
|
||||
const type = inferredType === 'html' ? (sniffBinaryDocumentType(bytes) ?? inferredType) : inferredType;
|
||||
const normalizedName = normalizeNameForType(inferredName, id, type);
|
||||
const contentType = contentTypeForDocument(type, normalizedName);
|
||||
const lastModified = Number.isFinite(st.mtimeMs) ? Math.floor(st.mtimeMs) : Date.now();
|
||||
|
||||
if (!byId.has(id)) {
|
||||
byId.set(id, {
|
||||
id,
|
||||
name: normalizedName,
|
||||
type,
|
||||
size: bytes.length,
|
||||
lastModified,
|
||||
contentType,
|
||||
bytes,
|
||||
localPaths: new Set([fullPath]),
|
||||
});
|
||||
} else {
|
||||
byId.get(id).localPaths.add(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fs.existsSync(docstoreDir)) {
|
||||
const entries = await fsp.readdir(docstoreDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
|
||||
const metadataPath = path.join(docstoreDir, entry.name);
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(await fsp.readFile(metadataPath, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!isLegacyDocumentMetadata(parsed)) continue;
|
||||
|
||||
const contentPath = path.join(docstoreDir, `${parsed.id}.${parsed.type}`);
|
||||
if (!fs.existsSync(contentPath)) continue;
|
||||
|
||||
filesScanned += 1;
|
||||
const bytes = await fsp.readFile(contentPath);
|
||||
const st = await fsp.stat(contentPath);
|
||||
const id = createHash('sha256').update(bytes).digest('hex');
|
||||
if (!isValidDocumentId(id)) {
|
||||
skippedInvalid += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const fallbackName = `${id}.${parsed.type}`;
|
||||
const normalizedInputName = safeDocumentName(parsed.name, fallbackName);
|
||||
const inferredType = toDocumentTypeFromName(normalizedInputName);
|
||||
const type = inferredType === 'html' ? (sniffBinaryDocumentType(bytes) ?? inferredType) : inferredType;
|
||||
const normalizedName = normalizeNameForType(normalizedInputName, id, type);
|
||||
const contentType = contentTypeForDocument(type, normalizedName);
|
||||
const lastModified = Number.isFinite(st.mtimeMs) ? Math.floor(st.mtimeMs) : Date.now();
|
||||
|
||||
if (!byId.has(id)) {
|
||||
byId.set(id, {
|
||||
id,
|
||||
name: normalizedName,
|
||||
type,
|
||||
size: bytes.length,
|
||||
lastModified,
|
||||
contentType,
|
||||
bytes,
|
||||
localPaths: new Set([contentPath, metadataPath]),
|
||||
});
|
||||
} else {
|
||||
byId.get(id).localPaths.add(contentPath);
|
||||
byId.get(id).localPaths.add(metadataPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
candidates: Array.from(byId.values()).map((item) => ({
|
||||
...item,
|
||||
localPaths: Array.from(item.localPaths),
|
||||
})),
|
||||
filesScanned,
|
||||
skippedInvalid,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectAudiobookCandidates(audiobooksDir, docstoreDir) {
|
||||
const stats = {
|
||||
booksScanned: 0,
|
||||
filesScanned: 0,
|
||||
skippedTransient: 0,
|
||||
};
|
||||
|
||||
const sourceDirs = new Map();
|
||||
|
||||
if (fs.existsSync(audiobooksDir)) {
|
||||
const entries = await fsp.readdir(audiobooksDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !entry.name.endsWith('-audiobook')) continue;
|
||||
const bookId = entry.name.slice(0, -'-audiobook'.length);
|
||||
if (!SAFE_AUDIOBOOK_ID_REGEX.test(bookId)) continue;
|
||||
sourceDirs.set(`${bookId}::${path.join(audiobooksDir, entry.name)}`, {
|
||||
bookId,
|
||||
dirPath: path.join(audiobooksDir, entry.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (fs.existsSync(docstoreDir)) {
|
||||
const entries = await fsp.readdir(docstoreDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !entry.name.endsWith('-audiobook')) continue;
|
||||
const bookId = entry.name.slice(0, -'-audiobook'.length);
|
||||
if (!SAFE_AUDIOBOOK_ID_REGEX.test(bookId)) continue;
|
||||
sourceDirs.set(`${bookId}::${path.join(docstoreDir, entry.name)}`, {
|
||||
bookId,
|
||||
dirPath: path.join(docstoreDir, entry.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const grouped = new Map();
|
||||
for (const source of sourceDirs.values()) {
|
||||
if (!grouped.has(source.bookId)) grouped.set(source.bookId, []);
|
||||
grouped.get(source.bookId).push(source.dirPath);
|
||||
}
|
||||
|
||||
const books = [];
|
||||
for (const [bookId, dirPaths] of grouped.entries()) {
|
||||
stats.booksScanned += 1;
|
||||
|
||||
const uploads = [];
|
||||
const dedupedChapterByIndex = new Map();
|
||||
let title = 'Unknown Title';
|
||||
|
||||
for (const dirPath of dirPaths) {
|
||||
const entries = await fsp.readdir(dirPath, { withFileTypes: true }).catch(() => []);
|
||||
const chapterMetaByIndex = new Map();
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (entry.name === 'audiobook.meta.json') continue;
|
||||
if (!entry.name.endsWith('.meta.json')) continue;
|
||||
const metaPath = path.join(dirPath, entry.name);
|
||||
try {
|
||||
const raw = JSON.parse(await fsp.readFile(metaPath, 'utf8'));
|
||||
const idx = Number(raw?.index ?? entry.name.replace(/\.meta\.json$/i, ''));
|
||||
const chapterTitle = typeof raw?.title === 'string' ? raw.title : null;
|
||||
if (Number.isInteger(idx) && idx >= 0 && chapterTitle) {
|
||||
chapterMetaByIndex.set(idx, chapterTitle);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const unresolvedSha = [];
|
||||
const usedIndices = new Set();
|
||||
const chapterUploads = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
const fileName = entry.name;
|
||||
stats.filesScanned += 1;
|
||||
|
||||
if (isTransientAudiobookFileName(fileName)) {
|
||||
stats.skippedTransient += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = path.join(dirPath, fileName);
|
||||
const st = await fsp.stat(fullPath).catch(() => null);
|
||||
const mtimeMs = Number(st?.mtimeMs ?? 0);
|
||||
|
||||
const canonical = decodeChapterFileName(fileName);
|
||||
if (canonical) {
|
||||
const targetFileName = encodeChapterFileName(canonical.index, canonical.title, canonical.format);
|
||||
chapterUploads.push({
|
||||
index: canonical.index,
|
||||
title: canonical.title,
|
||||
format: canonical.format,
|
||||
targetFileName,
|
||||
sourcePath: fullPath,
|
||||
mtimeMs,
|
||||
});
|
||||
usedIndices.add(canonical.index);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isPersistedAudiobookFileName(fileName)) {
|
||||
uploads.push({
|
||||
sourcePath: fullPath,
|
||||
targetFileName: fileName,
|
||||
contentType: contentTypeForAudiobookFileName(fileName),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const legacy = /^(\d+)-chapter\.(mp3|m4b)$/i.exec(fileName);
|
||||
if (legacy) {
|
||||
const index = Number(legacy[1]);
|
||||
const format = legacy[2].toLowerCase();
|
||||
const chapterTitle = chapterMetaByIndex.get(index) ?? `Chapter ${index + 1}`;
|
||||
const targetFileName = encodeChapterFileName(index, chapterTitle, format);
|
||||
chapterUploads.push({
|
||||
index,
|
||||
title: chapterTitle,
|
||||
format,
|
||||
targetFileName,
|
||||
sourcePath: fullPath,
|
||||
mtimeMs,
|
||||
});
|
||||
usedIndices.add(index);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sha = /^[a-f0-9]{64}\.(mp3|m4b)$/i.test(fileName);
|
||||
if (sha) {
|
||||
unresolvedSha.push({
|
||||
sourcePath: fullPath,
|
||||
format: fileName.toLowerCase().endsWith('.mp3') ? 'mp3' : 'm4b',
|
||||
mtimeMs,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
unresolvedSha.sort((a, b) => a.sourcePath.localeCompare(b.sourcePath));
|
||||
const nextIndex = () => {
|
||||
let idx = 0;
|
||||
while (usedIndices.has(idx)) idx += 1;
|
||||
usedIndices.add(idx);
|
||||
return idx;
|
||||
};
|
||||
|
||||
for (const item of unresolvedSha) {
|
||||
let decoded = null;
|
||||
const titleTag = await ffprobeTitleTag(item.sourcePath);
|
||||
if (titleTag) decoded = decodeChapterTitleTag(titleTag);
|
||||
const index = decoded?.index ?? nextIndex();
|
||||
const chapterTitle = decoded?.title ?? `Chapter ${index + 1}`;
|
||||
const targetFileName = encodeChapterFileName(index, chapterTitle, item.format);
|
||||
chapterUploads.push({
|
||||
index,
|
||||
title: chapterTitle,
|
||||
format: item.format,
|
||||
targetFileName,
|
||||
sourcePath: item.sourcePath,
|
||||
mtimeMs: item.mtimeMs,
|
||||
});
|
||||
}
|
||||
|
||||
for (const chapter of chapterUploads) {
|
||||
uploads.push({
|
||||
sourcePath: chapter.sourcePath,
|
||||
targetFileName: chapter.targetFileName,
|
||||
contentType: contentTypeForAudiobookFileName(chapter.targetFileName),
|
||||
});
|
||||
|
||||
const current = dedupedChapterByIndex.get(chapter.index);
|
||||
if (!current || chapter.targetFileName > current.fileName || chapter.mtimeMs > current.mtimeMs) {
|
||||
dedupedChapterByIndex.set(chapter.index, {
|
||||
index: chapter.index,
|
||||
title: chapter.title,
|
||||
format: chapter.format,
|
||||
fileName: chapter.targetFileName,
|
||||
mtimeMs: chapter.mtimeMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const chapters = Array.from(dedupedChapterByIndex.values())
|
||||
.sort((a, b) => a.index - b.index)
|
||||
.map((chapter) => ({
|
||||
index: chapter.index,
|
||||
title: chapter.title,
|
||||
format: chapter.format,
|
||||
fileName: chapter.fileName,
|
||||
}));
|
||||
|
||||
if (chapters.length > 0) title = chapters[0].title || title;
|
||||
|
||||
books.push({
|
||||
id: bookId,
|
||||
title,
|
||||
chapters,
|
||||
uploads,
|
||||
sourceDirs: dirPaths,
|
||||
});
|
||||
}
|
||||
|
||||
return { books, stats };
|
||||
}
|
||||
|
||||
function openDatabase() {
|
||||
if (process.env.POSTGRES_URL) {
|
||||
const pool = new Pool({
|
||||
connectionString: process.env.POSTGRES_URL,
|
||||
ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false,
|
||||
});
|
||||
return {
|
||||
mode: 'pg',
|
||||
async query(sql, values = []) {
|
||||
return pool.query(sql, values);
|
||||
},
|
||||
async close() {
|
||||
await pool.end();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const dbPath = path.join(process.cwd(), 'docstore', 'sqlite3.db');
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
||||
const sqlite = new BetterSqlite3(dbPath);
|
||||
return {
|
||||
mode: 'sqlite',
|
||||
async query(sql, values = []) {
|
||||
if (/^\s*select/i.test(sql)) {
|
||||
const rows = sqlite.prepare(sql).all(...values);
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
const info = sqlite.prepare(sql).run(...values);
|
||||
return { rows: [], rowCount: Number(info.changes ?? 0) };
|
||||
},
|
||||
async close() {
|
||||
sqlite.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function migrateDatabaseRows(database, userId, docCandidates, audiobookBooks, dryRun) {
|
||||
const mismatched = await database.query('SELECT COUNT(*) AS count FROM documents WHERE file_path <> id');
|
||||
const rowsUpdated = Number(mismatched.rows[0]?.count ?? mismatched.rows[0]?.COUNT ?? 0);
|
||||
if (!dryRun && rowsUpdated > 0) {
|
||||
await database.query('UPDATE documents SET file_path = id WHERE file_path <> id');
|
||||
}
|
||||
|
||||
const existingDocs = database.mode === 'sqlite'
|
||||
? await database.query('SELECT id FROM documents WHERE user_id = ?', [userId])
|
||||
: await database.query('SELECT id FROM documents WHERE user_id = $1', [userId]);
|
||||
const existingDocIds = new Set(existingDocs.rows.map((row) => row.id));
|
||||
const toInsertDocs = docCandidates.filter((candidate) => !existingDocIds.has(candidate.id));
|
||||
|
||||
if (!dryRun) {
|
||||
for (const candidate of toInsertDocs) {
|
||||
if (database.mode === 'sqlite') {
|
||||
await database.query(
|
||||
'INSERT OR IGNORE INTO documents (id, user_id, name, type, size, last_modified, file_path) VALUES (?, ?, ?, ?, ?, ?, ?)',
|
||||
[candidate.id, userId, candidate.name, candidate.type, candidate.size, candidate.lastModified, candidate.id],
|
||||
);
|
||||
} else {
|
||||
await database.query(
|
||||
'INSERT INTO documents (id, user_id, name, type, size, last_modified, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING',
|
||||
[candidate.id, userId, candidate.name, candidate.type, candidate.size, candidate.lastModified, candidate.id],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const book of audiobookBooks) {
|
||||
if (database.mode === 'sqlite') {
|
||||
await database.query(
|
||||
'INSERT OR IGNORE INTO audiobooks (id, user_id, title, duration) VALUES (?, ?, ?, ?)',
|
||||
[book.id, userId, book.title || 'Unknown Title', 0],
|
||||
);
|
||||
} else {
|
||||
await database.query(
|
||||
'INSERT INTO audiobooks (id, user_id, title, duration) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING',
|
||||
[book.id, userId, book.title || 'Unknown Title', 0],
|
||||
);
|
||||
}
|
||||
|
||||
for (const chapter of book.chapters) {
|
||||
const chapterId = `${book.id}-${chapter.index}`;
|
||||
if (database.mode === 'sqlite') {
|
||||
await database.query(
|
||||
'INSERT OR IGNORE INTO audiobook_chapters (id, book_id, user_id, chapter_index, title, duration, file_path, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
|
||||
[chapterId, book.id, userId, chapter.index, chapter.title, 0, chapter.fileName, chapter.format],
|
||||
);
|
||||
} else {
|
||||
await database.query(
|
||||
'INSERT INTO audiobook_chapters (id, book_id, user_id, chapter_index, title, duration, file_path, format) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING',
|
||||
[chapterId, book.id, userId, chapter.index, chapter.title, 0, chapter.fileName, chapter.format],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dbRowsUpdated: rowsUpdated,
|
||||
dbRowsSeeded: toInsertDocs.length,
|
||||
audiobookDbBooksSeeded: audiobookBooks.length,
|
||||
audiobookDbChaptersSeeded: audiobookBooks.reduce((sum, book) => sum + book.chapters.length, 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
loadEnvFiles();
|
||||
const { dryRun, deleteLocal, namespace } = parseArgs(process.argv.slice(2));
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(namespace);
|
||||
const docsDir = applyNamespacePath(DOCUMENTS_V1_DIR, namespace);
|
||||
const audiobooksDir = applyNamespacePath(AUDIOBOOKS_V1_DIR, namespace);
|
||||
const docstoreDir = applyNamespacePath(DOCSTORE_DIR, namespace);
|
||||
|
||||
const s3Config = parseS3ConfigFromEnv();
|
||||
const s3Client = createS3Client(s3Config);
|
||||
|
||||
const { candidates: documentCandidates, filesScanned, skippedInvalid } = await collectDocumentCandidates(docsDir, docstoreDir);
|
||||
const { books: audiobookBooks, stats: audiobookStats } = await collectAudiobookCandidates(audiobooksDir, docstoreDir);
|
||||
|
||||
let uploaded = 0;
|
||||
let alreadyPresent = 0;
|
||||
let deletedLocal = 0;
|
||||
|
||||
for (const candidate of documentCandidates) {
|
||||
if (!dryRun) {
|
||||
try {
|
||||
await putObjectIfMissing(
|
||||
s3Client,
|
||||
s3Config,
|
||||
documentKey(s3Config, candidate.id, namespace),
|
||||
candidate.bytes,
|
||||
candidate.contentType,
|
||||
);
|
||||
uploaded += 1;
|
||||
} catch (error) {
|
||||
if (isPreconditionFailed(error)) {
|
||||
alreadyPresent += 1;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
for (const localPath of candidate.localPaths) {
|
||||
const removed = await fsp.unlink(localPath).then(() => true).catch(() => false);
|
||||
if (removed) deletedLocal += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let audiobookUploaded = 0;
|
||||
let audiobookAlreadyPresent = 0;
|
||||
let audiobookDeletedLocal = 0;
|
||||
|
||||
for (const book of audiobookBooks) {
|
||||
for (const upload of book.uploads) {
|
||||
const bytes = await fsp.readFile(upload.sourcePath);
|
||||
if (!dryRun) {
|
||||
try {
|
||||
await putObjectIfMissing(
|
||||
s3Client,
|
||||
s3Config,
|
||||
audiobookKey(s3Config, book.id, unclaimedUserId, upload.targetFileName, namespace),
|
||||
bytes,
|
||||
upload.contentType,
|
||||
);
|
||||
audiobookUploaded += 1;
|
||||
} catch (error) {
|
||||
if (isPreconditionFailed(error)) {
|
||||
audiobookAlreadyPresent += 1;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
const removed = await fsp.unlink(upload.sourcePath).then(() => true).catch(() => false);
|
||||
if (removed) audiobookDeletedLocal += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
for (const dirPath of book.sourceDirs) {
|
||||
await fsp.rm(dirPath, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const database = openDatabase();
|
||||
try {
|
||||
const dbStats = await migrateDatabaseRows(
|
||||
database,
|
||||
unclaimedUserId,
|
||||
documentCandidates,
|
||||
audiobookBooks,
|
||||
dryRun,
|
||||
);
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
dryRun,
|
||||
deleteLocal,
|
||||
docsDir,
|
||||
audiobooksDir,
|
||||
namespace,
|
||||
filesScanned,
|
||||
uploaded,
|
||||
alreadyPresent,
|
||||
skippedInvalid,
|
||||
deletedLocal,
|
||||
dbRowsUpdated: dbStats.dbRowsUpdated,
|
||||
dbRowsSeeded: dbStats.dbRowsSeeded,
|
||||
audiobookBooksScanned: audiobookStats.booksScanned,
|
||||
audiobookFilesScanned: audiobookStats.filesScanned,
|
||||
audiobookUploaded,
|
||||
audiobookAlreadyPresent,
|
||||
audiobookDeletedLocal,
|
||||
audiobookSkippedTransient: audiobookStats.skippedTransient,
|
||||
audiobookDbBooksSeeded: dbStats.audiobookDbBooksSeeded,
|
||||
audiobookDbChaptersSeeded: dbStats.audiobookDbChaptersSeeded,
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} finally {
|
||||
await database.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('Error running v2 migration script:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -28,6 +28,11 @@ function isTrue(value, defaultValue) {
|
|||
return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
|
||||
}
|
||||
|
||||
function resolveBooleanEnv(env, key, defaultValue) {
|
||||
const value = env[key];
|
||||
return isTrue(value, defaultValue);
|
||||
}
|
||||
|
||||
function withDefault(value, fallback) {
|
||||
return value && value.trim() ? value.trim() : fallback;
|
||||
}
|
||||
|
|
@ -193,6 +198,35 @@ function runDbMigrations(env) {
|
|||
}
|
||||
}
|
||||
|
||||
function runStorageMigrations(env) {
|
||||
const migrateScript = path.join(process.cwd(), 'scripts', 'migrate-fs-v2.mjs');
|
||||
if (!fs.existsSync(migrateScript)) {
|
||||
throw new Error(`Could not find storage migration script at ${migrateScript}`);
|
||||
}
|
||||
|
||||
console.log('Running storage migrations (v2)...');
|
||||
const migration = spawnSync(process.execPath, [migrateScript, '--dry-run', 'false', '--delete-local', 'false'], {
|
||||
env,
|
||||
stdio: 'inherit',
|
||||
});
|
||||
|
||||
if (migration.error) {
|
||||
throw migration.error;
|
||||
}
|
||||
if (typeof migration.status === 'number' && migration.status !== 0) {
|
||||
throw new Error(`Storage migrations failed with exit code ${migration.status}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function hasS3Config(env) {
|
||||
return Boolean(
|
||||
env.S3_BUCKET?.trim()
|
||||
&& env.S3_REGION?.trim()
|
||||
&& env.S3_ACCESS_KEY_ID?.trim()
|
||||
&& env.S3_SECRET_ACCESS_KEY?.trim()
|
||||
);
|
||||
}
|
||||
|
||||
function sendSignal(child, signal, useProcessGroup) {
|
||||
if (!child) return false;
|
||||
if (useProcessGroup && process.platform !== 'win32' && typeof child.pid === 'number' && child.pid > 0) {
|
||||
|
|
@ -291,7 +325,7 @@ async function main() {
|
|||
});
|
||||
|
||||
try {
|
||||
const shouldRunDbMigrations = isTrue(runtimeEnv.RUN_DB_MIGRATIONS, true);
|
||||
const shouldRunDbMigrations = resolveBooleanEnv(runtimeEnv, 'RUN_DRIZZLE_MIGRATIONS', true);
|
||||
if (shouldRunDbMigrations) {
|
||||
runDbMigrations(runtimeEnv);
|
||||
}
|
||||
|
|
@ -351,6 +385,15 @@ async function main() {
|
|||
console.log(`Embedded SeaweedFS is ready at ${endpoint}`);
|
||||
}
|
||||
|
||||
const shouldRunStorageMigrations = resolveBooleanEnv(runtimeEnv, 'RUN_FS_MIGRATIONS', true);
|
||||
if (shouldRunStorageMigrations) {
|
||||
if (hasS3Config(runtimeEnv)) {
|
||||
runStorageMigrations(runtimeEnv);
|
||||
} else {
|
||||
console.warn('Skipping storage migrations: S3 configuration is incomplete.');
|
||||
}
|
||||
}
|
||||
|
||||
const { child, exitPromise } = spawnMainCommand(command, runtimeEnv);
|
||||
appProc = child;
|
||||
const exitCode = await exitPromise;
|
||||
|
|
|
|||
|
|
@ -1,211 +1,196 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { createReadStream, existsSync } from 'fs';
|
||||
import { readdir, unlink } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
|
||||
import { findStoredChapterByIndex } from '@/lib/server/audiobook';
|
||||
import { and, eq, inArray } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { audiobooks, audiobookChapters } from '@/db/schema';
|
||||
import { and, eq, inArray } from 'drizzle-orm';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import {
|
||||
deleteAudiobookObject,
|
||||
getAudiobookObjectBuffer,
|
||||
isMissingBlobError,
|
||||
listAudiobookObjects,
|
||||
} from '@/lib/server/audiobooks-blobstore';
|
||||
import { decodeChapterFileName } from '@/lib/server/audiobook';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { pruneAudiobookChapterIfMissingFile, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
function streamBuffer(buffer: Buffer): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(buffer));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function findChapterFileNameByIndex(fileNames: string[], index: number): { fileName: string; title: string; format: 'mp3' | 'm4b' } | null {
|
||||
const matches = fileNames
|
||||
.map((fileName) => {
|
||||
const decoded = decodeChapterFileName(fileName);
|
||||
if (!decoded) return null;
|
||||
if (decoded.index !== index) return null;
|
||||
return { fileName, title: decoded.title, format: decoded.format };
|
||||
})
|
||||
.filter((value): value is { fileName: string; title: string; format: 'mp3' | 'm4b' } => Boolean(value))
|
||||
.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
||||
|
||||
return matches.at(-1) ?? null;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const bookId = request.nextUrl.searchParams.get('bookId');
|
||||
const chapterIndexStr = request.nextUrl.searchParams.get('chapterIndex');
|
||||
|
||||
if (!bookId || !chapterIndexStr) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing bookId or chapterIndex parameter' },
|
||||
{ status: 400 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Missing bookId or chapterIndex parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
const chapterIndex = parseInt(chapterIndexStr);
|
||||
if (isNaN(chapterIndex)) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid chapterIndex parameter' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
await ensureAudiobooksV1Ready();
|
||||
if (!(await isAudiobooksV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
const chapterIndex = Number.parseInt(chapterIndexStr, 10);
|
||||
if (!Number.isInteger(chapterIndex) || chapterIndex < 0) {
|
||||
return NextResponse.json({ error: 'Invalid chapterIndex parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const { userId, authEnabled } = ctxOrRes;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const storageUserId = userId ?? unclaimedUserId;
|
||||
const allowedUserIds = authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
// Verify ownership with composite PK - allow access to user's own OR unclaimed audiobooks
|
||||
const [existingBook] = await db
|
||||
.select({ userId: audiobooks.userId })
|
||||
.from(audiobooks)
|
||||
.where(and(eq(audiobooks.id, bookId), inArray(audiobooks.userId, allowedUserIds)));
|
||||
|
||||
if (!existingBook) {
|
||||
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: existingBook.userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
const objects = await listAudiobookObjects(bookId, existingBook.userId, testNamespace);
|
||||
const chapter = findChapterFileNameByIndex(
|
||||
objects.map((object) => object.fileName),
|
||||
chapterIndex,
|
||||
);
|
||||
const dirExists = existsSync(intermediateDir);
|
||||
if (!dirExists) {
|
||||
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);
|
||||
|
||||
if (!chapter) {
|
||||
await db
|
||||
.delete(audiobookChapters)
|
||||
.where(
|
||||
and(
|
||||
eq(audiobookChapters.bookId, bookId),
|
||||
eq(audiobookChapters.userId, existingBook.userId),
|
||||
eq(audiobookChapters.chapterIndex, chapterIndex),
|
||||
),
|
||||
);
|
||||
return NextResponse.json({ error: 'Chapter not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const chapter = await findStoredChapterByIndex(intermediateDir, chapterIndex, request.signal);
|
||||
const chapterFileExists = !!chapter?.filePath && existsSync(chapter.filePath);
|
||||
if (!chapterFileExists) {
|
||||
await pruneAudiobookChapterIfMissingFile(bookId, existingBook.userId, chapterIndex, false);
|
||||
return NextResponse.json({ error: 'Chapter not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
// Stream the chapter file
|
||||
const stream = createReadStream(chapter.filePath);
|
||||
|
||||
const readableWebStream = new ReadableStream({
|
||||
start(controller) {
|
||||
stream.on('data', (chunk) => {
|
||||
controller.enqueue(chunk);
|
||||
});
|
||||
stream.on('end', () => {
|
||||
controller.close();
|
||||
});
|
||||
stream.on('error', (err) => {
|
||||
controller.error(err);
|
||||
});
|
||||
},
|
||||
cancel() {
|
||||
stream.destroy();
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = await getAudiobookObjectBuffer(bookId, existingBook.userId, chapter.fileName, testNamespace);
|
||||
} catch (error) {
|
||||
if (isMissingBlobError(error)) {
|
||||
await db
|
||||
.delete(audiobookChapters)
|
||||
.where(
|
||||
and(
|
||||
eq(audiobookChapters.bookId, bookId),
|
||||
eq(audiobookChapters.userId, existingBook.userId),
|
||||
eq(audiobookChapters.chapterIndex, chapterIndex),
|
||||
),
|
||||
);
|
||||
return NextResponse.json({ error: 'Chapter not found' }, { status: 404 });
|
||||
}
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
const mimeType = chapter.format === 'mp3' ? 'audio/mpeg' : 'audio/mp4';
|
||||
const sanitizedTitle = chapter.title.replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
||||
|
||||
return new NextResponse(readableWebStream, {
|
||||
return new NextResponse(streamBuffer(buffer), {
|
||||
headers: {
|
||||
'Content-Type': mimeType,
|
||||
'Content-Disposition': `attachment; filename="${sanitizedTitle}.${chapter.format}"`,
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error downloading chapter:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to download chapter' },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Failed to download chapter' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function DELETE(request: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const bookId = request.nextUrl.searchParams.get('bookId');
|
||||
const chapterIndexStr = request.nextUrl.searchParams.get('chapterIndex');
|
||||
|
||||
if (!bookId || !chapterIndexStr) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing bookId or chapterIndex parameter' },
|
||||
{ status: 400 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Missing bookId or chapterIndex parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
const chapterIndex = parseInt(chapterIndexStr, 10);
|
||||
if (isNaN(chapterIndex)) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid chapterIndex parameter' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
await ensureAudiobooksV1Ready();
|
||||
if (!(await isAudiobooksV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
const chapterIndex = Number.parseInt(chapterIndexStr, 10);
|
||||
if (!Number.isInteger(chapterIndex) || chapterIndex < 0) {
|
||||
return NextResponse.json({ error: 'Invalid chapterIndex parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
const { userId, authEnabled } = ctxOrRes;
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const storageUserId = userId ?? getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const storageUserId = ctxOrRes.userId ?? getUnclaimedUserIdForNamespace(testNamespace);
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
// Verify ownership and delete from DB with composite PK
|
||||
const [existingBook] = await db
|
||||
.select({ userId: audiobooks.userId })
|
||||
.from(audiobooks)
|
||||
.where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId)));
|
||||
|
||||
if (!existingBook) {
|
||||
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
// Delete from DB
|
||||
await db.delete(audiobookChapters).where(
|
||||
and(
|
||||
eq(audiobookChapters.bookId, bookId),
|
||||
eq(audiobookChapters.userId, storageUserId),
|
||||
eq(audiobookChapters.chapterIndex, chapterIndex),
|
||||
),
|
||||
);
|
||||
await db
|
||||
.delete(audiobookChapters)
|
||||
.where(
|
||||
and(
|
||||
eq(audiobookChapters.bookId, bookId),
|
||||
eq(audiobookChapters.userId, storageUserId),
|
||||
eq(audiobookChapters.chapterIndex, chapterIndex),
|
||||
),
|
||||
);
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: storageUserId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
const objectNames = (await listAudiobookObjects(bookId, storageUserId, testNamespace)).map((object) => object.fileName);
|
||||
const chapterPrefix = `${String(chapterIndex + 1).padStart(4, '0')}__`;
|
||||
const files = await readdir(intermediateDir).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (!file.startsWith(chapterPrefix)) continue;
|
||||
if (!file.endsWith('.mp3') && !file.endsWith('.m4b')) continue;
|
||||
await unlink(join(intermediateDir, file)).catch(() => { });
|
||||
|
||||
for (const fileName of objectNames) {
|
||||
if (!fileName.startsWith(chapterPrefix)) continue;
|
||||
if (!fileName.endsWith('.mp3') && !fileName.endsWith('.m4b')) continue;
|
||||
await deleteAudiobookObject(bookId, storageUserId, fileName, testNamespace).catch(() => {});
|
||||
}
|
||||
|
||||
// Invalidate any combined "complete" files
|
||||
const completeM4b = join(intermediateDir, `complete.m4b`);
|
||||
const completeMp3 = join(intermediateDir, `complete.mp3`);
|
||||
if (existsSync(completeM4b)) await unlink(completeM4b).catch(() => { });
|
||||
if (existsSync(completeMp3)) await unlink(completeMp3).catch(() => { });
|
||||
await unlink(join(intermediateDir, 'complete.mp3.manifest.json')).catch(() => { });
|
||||
await unlink(join(intermediateDir, 'complete.m4b.manifest.json')).catch(() => { });
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.mp3', testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.m4b', testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.mp3.manifest.json', testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.m4b.manifest.json', testNamespace).catch(() => {});
|
||||
|
||||
return NextResponse.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error deleting chapter:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to delete chapter' },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Failed to delete chapter' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,34 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { spawn } from 'child_process';
|
||||
import { readFile, writeFile, mkdir, unlink, rm, rename, readdir } from 'fs/promises';
|
||||
import { existsSync, createReadStream } from 'fs';
|
||||
import { basename, join } from 'path';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { ensureAudiobooksV1Ready, isAudiobooksV1Ready, getAudiobooksRootDir } from '@/lib/server/docstore';
|
||||
import { encodeChapterFileName, encodeChapterTitleTag, listStoredChapters, ffprobeAudio, escapeFFMetadata } from '@/lib/server/audiobook';
|
||||
import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import { and, eq, inArray } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { audiobooks, audiobookChapters } from '@/db/schema';
|
||||
import { eq, and, inArray } from 'drizzle-orm';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import {
|
||||
audiobookPrefix,
|
||||
deleteAudiobookObject,
|
||||
deleteAudiobookPrefix,
|
||||
getAudiobookObjectBuffer,
|
||||
isMissingBlobError,
|
||||
listAudiobookObjects,
|
||||
putAudiobookObject,
|
||||
} from '@/lib/server/audiobooks-blobstore';
|
||||
import {
|
||||
decodeChapterFileName,
|
||||
encodeChapterFileName,
|
||||
encodeChapterTitleTag,
|
||||
escapeFFMetadata,
|
||||
ffprobeAudio,
|
||||
} from '@/lib/server/audiobook';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
|
||||
import { getFFmpegPath, getFFprobePath } from '@/lib/server/ffmpeg-bin';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -27,13 +41,134 @@ interface ConversionRequest {
|
|||
settings?: AudiobookGenerationSettings;
|
||||
}
|
||||
|
||||
type ChapterObject = {
|
||||
index: number;
|
||||
title: string;
|
||||
format: TTSAudiobookFormat;
|
||||
fileName: string;
|
||||
};
|
||||
|
||||
const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
function isSafeId(value: string): boolean {
|
||||
return SAFE_ID_REGEX.test(value);
|
||||
}
|
||||
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
function chapterFileMimeType(format: TTSAudiobookFormat): string {
|
||||
return format === 'mp3' ? 'audio/mpeg' : 'audio/mp4';
|
||||
}
|
||||
|
||||
function buildAtempoFilter(speed: number): string {
|
||||
const clamped = Math.max(0.5, Math.min(speed, 3));
|
||||
if (clamped <= 2) return `atempo=${clamped.toFixed(3)}`;
|
||||
const second = clamped / 2;
|
||||
return `atempo=2.0,atempo=${second.toFixed(3)}`;
|
||||
}
|
||||
|
||||
function listChapterObjects(objectNames: string[]): ChapterObject[] {
|
||||
const chapters = objectNames
|
||||
.filter((name) => !name.startsWith('complete.'))
|
||||
.map((fileName) => {
|
||||
const decoded = decodeChapterFileName(fileName);
|
||||
if (!decoded) return null;
|
||||
return {
|
||||
index: decoded.index,
|
||||
title: decoded.title,
|
||||
format: decoded.format,
|
||||
fileName,
|
||||
} satisfies ChapterObject;
|
||||
})
|
||||
.filter((value): value is ChapterObject => Boolean(value))
|
||||
.sort((a, b) => a.index - b.index);
|
||||
|
||||
const deduped = new Map<number, ChapterObject>();
|
||||
for (const chapter of chapters) {
|
||||
const existing = deduped.get(chapter.index);
|
||||
if (!existing) {
|
||||
deduped.set(chapter.index, chapter);
|
||||
continue;
|
||||
}
|
||||
if (chapter.fileName > existing.fileName) {
|
||||
deduped.set(chapter.index, chapter);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(deduped.values()).sort((a, b) => a.index - b.index);
|
||||
}
|
||||
|
||||
function streamBuffer(buffer: Buffer): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(buffer));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runFFmpeg(args: string[], signal?: AbortSignal): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn(getFFmpegPath(), args);
|
||||
let finished = false;
|
||||
|
||||
const onAbort = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
try {
|
||||
ffmpeg.kill('SIGKILL');
|
||||
} catch {}
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
ffmpeg.stderr.on('data', (data) => {
|
||||
console.error(`ffmpeg stderr: ${data}`);
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`FFmpeg process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function getAudioDuration(filePath: string, signal?: AbortSignal): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ffprobe = spawn('ffprobe', [
|
||||
'-i', filePath,
|
||||
'-show_entries', 'format=duration',
|
||||
'-v', 'quiet',
|
||||
'-of', 'csv=p=0'
|
||||
const ffprobe = spawn(getFFprobePath(), [
|
||||
'-i',
|
||||
filePath,
|
||||
'-show_entries',
|
||||
'format=duration',
|
||||
'-v',
|
||||
'quiet',
|
||||
'-of',
|
||||
'csv=p=0',
|
||||
]);
|
||||
|
||||
let output = '';
|
||||
|
|
@ -44,7 +179,7 @@ async function getAudioDuration(filePath: string, signal?: AbortSignal): Promise
|
|||
finished = true;
|
||||
try {
|
||||
ffprobe.kill('SIGKILL');
|
||||
} catch { }
|
||||
} catch {}
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
||||
|
|
@ -85,95 +220,25 @@ async function getAudioDuration(filePath: string, signal?: AbortSignal): Promise
|
|||
});
|
||||
}
|
||||
|
||||
async function runFFmpeg(args: string[], signal?: AbortSignal): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn('ffmpeg', args);
|
||||
|
||||
let finished = false;
|
||||
|
||||
const onAbort = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
try {
|
||||
ffmpeg.kill('SIGKILL');
|
||||
} catch { }
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
ffmpeg.stderr.on('data', (data) => {
|
||||
console.error(`ffmpeg stderr: ${data}`);
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`FFmpeg process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function buildAtempoFilter(speed: number): string {
|
||||
const clamped = Math.max(0.5, Math.min(speed, 3));
|
||||
// atempo supports 0.5..2.0 per filter; chain for >2.0
|
||||
if (clamped <= 2) return `atempo=${clamped.toFixed(3)}`;
|
||||
const second = clamped / 2;
|
||||
return `atempo=2.0,atempo=${second.toFixed(3)}`;
|
||||
}
|
||||
|
||||
const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
function isSafeId(value: string): boolean {
|
||||
return SAFE_ID_REGEX.test(value);
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
try {
|
||||
// Parse the request body
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const data: ConversionRequest = await request.json();
|
||||
const requestedFormat = data.format || 'm4b';
|
||||
|
||||
await ensureAudiobooksV1Ready();
|
||||
if (!(await isAudiobooksV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
const userId = ctxOrRes.userId;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const storageUserId = userId ?? getUnclaimedUserIdForNamespace(testNamespace);
|
||||
|
||||
// Generate or use existing book ID
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const storageUserId = ctxOrRes.userId ?? getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const bookId = data.bookId || randomUUID();
|
||||
|
||||
if (!isSafeId(bookId)) {
|
||||
return NextResponse.json({ error: 'Invalid bookId parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
// DB Check / Insert Audiobook
|
||||
await db
|
||||
.insert(audiobooks)
|
||||
.values({
|
||||
|
|
@ -183,57 +248,36 @@ export async function POST(request: NextRequest) {
|
|||
})
|
||||
.onConflictDoNothing();
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId,
|
||||
authEnabled: ctxOrRes.authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
// Create intermediate directory
|
||||
await mkdir(intermediateDir, { recursive: true });
|
||||
|
||||
const existingChapters = await listStoredChapters(intermediateDir, request.signal);
|
||||
const objects = await listAudiobookObjects(bookId, storageUserId, testNamespace);
|
||||
const objectNames = objects.map((item) => item.fileName);
|
||||
const existingChapters = listChapterObjects(objectNames);
|
||||
const hasChapters = existingChapters.length > 0;
|
||||
|
||||
const metaPath = join(intermediateDir, 'audiobook.meta.json');
|
||||
const incomingSettings = data.settings;
|
||||
let existingSettings: AudiobookGenerationSettings | null = null;
|
||||
try {
|
||||
existingSettings = JSON.parse(await readFile(metaPath, 'utf8')) as AudiobookGenerationSettings;
|
||||
} catch {
|
||||
existingSettings = JSON.parse((await getAudiobookObjectBuffer(bookId, storageUserId, 'audiobook.meta.json', testNamespace)).toString('utf8')) as AudiobookGenerationSettings;
|
||||
} catch (error) {
|
||||
if (!isMissingBlobError(error)) throw error;
|
||||
existingSettings = null;
|
||||
}
|
||||
|
||||
// Only enforce mismatch check if we already have generated chapters.
|
||||
// If no chapters exist, we can overwrite/ignore "existing" settings (which might be stale or partial).
|
||||
if (existingSettings && hasChapters) {
|
||||
if (incomingSettings) {
|
||||
const mismatch =
|
||||
existingSettings.ttsProvider !== incomingSettings.ttsProvider ||
|
||||
existingSettings.ttsModel !== incomingSettings.ttsModel ||
|
||||
existingSettings.voice !== incomingSettings.voice ||
|
||||
existingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
||||
existingSettings.postSpeed !== incomingSettings.postSpeed ||
|
||||
existingSettings.format !== incomingSettings.format;
|
||||
if (mismatch) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobook settings mismatch', settings: existingSettings },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
const incomingSettings = data.settings;
|
||||
if (existingSettings && hasChapters && incomingSettings) {
|
||||
const mismatch =
|
||||
existingSettings.ttsProvider !== incomingSettings.ttsProvider ||
|
||||
existingSettings.ttsModel !== incomingSettings.ttsModel ||
|
||||
existingSettings.voice !== incomingSettings.voice ||
|
||||
existingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
||||
existingSettings.postSpeed !== incomingSettings.postSpeed ||
|
||||
existingSettings.format !== incomingSettings.format;
|
||||
if (mismatch) {
|
||||
return NextResponse.json({ error: 'Audiobook settings mismatch', settings: existingSettings }, { status: 409 });
|
||||
}
|
||||
}
|
||||
// Note: We deliberately do NOT write the meta file here yet.
|
||||
// We wait until a chapter is successfully generated/saved below.
|
||||
const existingFormats = new Set(existingChapters.map((c) => c.format));
|
||||
|
||||
const existingFormats = new Set(existingChapters.map((chapter) => chapter.format));
|
||||
if (existingFormats.size > 1) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Mixed chapter formats detected; reset the audiobook to continue' },
|
||||
{ status: 400 },
|
||||
);
|
||||
return NextResponse.json({ error: 'Mixed chapter formats detected; reset the audiobook to continue' }, { status: 400 });
|
||||
}
|
||||
|
||||
const format: TTSAudiobookFormat =
|
||||
|
|
@ -244,8 +288,6 @@ export async function POST(request: NextRequest) {
|
|||
const rawPostSpeed = incomingSettings?.postSpeed ?? existingSettings?.postSpeed ?? 1;
|
||||
const postSpeed = Number.isFinite(Number(rawPostSpeed)) ? Number(rawPostSpeed) : 1;
|
||||
|
||||
// Use provided chapter index or find the next available index robustly (handles gaps)
|
||||
// Use provided chapter index or find the next available index robustly (handles gaps)
|
||||
let chapterIndex: number;
|
||||
if (data.chapterIndex !== undefined) {
|
||||
const normalized = Number(data.chapterIndex);
|
||||
|
|
@ -255,7 +297,6 @@ export async function POST(request: NextRequest) {
|
|||
chapterIndex = normalized;
|
||||
} else {
|
||||
const indices = existingChapters.map((c) => c.index);
|
||||
// Find smallest non-negative integer not present
|
||||
let next = 0;
|
||||
for (const idx of indices) {
|
||||
if (idx === next) {
|
||||
|
|
@ -267,76 +308,82 @@ export async function POST(request: NextRequest) {
|
|||
chapterIndex = next;
|
||||
}
|
||||
|
||||
// Write input file (MP3 from TTS)
|
||||
const inputPath = join(intermediateDir, `${chapterIndex}-input.mp3`);
|
||||
const chapterOutputTempPath = join(intermediateDir, `${chapterIndex}-chapter.tmp.${format}`);
|
||||
workDir = await mkdtemp(join(tmpdir(), 'openreader-audiobook-'));
|
||||
const inputPath = join(workDir, `${chapterIndex}-input.mp3`);
|
||||
const chapterOutputTempPath = join(workDir, `${chapterIndex}-chapter.tmp.${format}`);
|
||||
const titleTag = encodeChapterTitleTag(chapterIndex, data.chapterTitle);
|
||||
|
||||
// Write the chapter audio to a temp file
|
||||
await writeFile(inputPath, Buffer.from(new Uint8Array(data.buffer)));
|
||||
|
||||
// We intentionally do not delete the existing chapter file up-front. This avoids a long
|
||||
// window where the chapter is "missing" while ffmpeg is running (which can lead to
|
||||
// partial/stale "complete.*" downloads). We clean up duplicates and invalidate the
|
||||
// combined output only after the new chapter is written successfully.
|
||||
|
||||
if (format === 'mp3') {
|
||||
// For MP3, re-encode to ensure proper headers and consistent format
|
||||
await runFFmpeg([
|
||||
'-y', // Overwrite output file without asking
|
||||
'-i', inputPath,
|
||||
...(postSpeed !== 1 ? ['-filter:a', buildAtempoFilter(postSpeed)] : []),
|
||||
'-c:a', 'libmp3lame',
|
||||
'-b:a', '64k',
|
||||
'-metadata', `title=${titleTag}`,
|
||||
chapterOutputTempPath
|
||||
], request.signal);
|
||||
await runFFmpeg(
|
||||
[
|
||||
'-y',
|
||||
'-i',
|
||||
inputPath,
|
||||
...(postSpeed !== 1 ? ['-filter:a', buildAtempoFilter(postSpeed)] : []),
|
||||
'-c:a',
|
||||
'libmp3lame',
|
||||
'-b:a',
|
||||
'64k',
|
||||
'-metadata',
|
||||
`title=${titleTag}`,
|
||||
chapterOutputTempPath,
|
||||
],
|
||||
request.signal,
|
||||
);
|
||||
} else {
|
||||
// Convert MP3 to M4B container with proper encoding and metadata
|
||||
await runFFmpeg([
|
||||
'-y', // Overwrite output file without asking
|
||||
'-i', inputPath,
|
||||
...(postSpeed !== 1 ? ['-filter:a', buildAtempoFilter(postSpeed)] : []),
|
||||
'-c:a', 'aac',
|
||||
'-b:a', '64k',
|
||||
'-metadata', `title=${titleTag}`,
|
||||
'-f', 'mp4',
|
||||
chapterOutputTempPath
|
||||
], request.signal);
|
||||
await runFFmpeg(
|
||||
[
|
||||
'-y',
|
||||
'-i',
|
||||
inputPath,
|
||||
...(postSpeed !== 1 ? ['-filter:a', buildAtempoFilter(postSpeed)] : []),
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
'64k',
|
||||
'-metadata',
|
||||
`title=${titleTag}`,
|
||||
'-f',
|
||||
'mp4',
|
||||
chapterOutputTempPath,
|
||||
],
|
||||
request.signal,
|
||||
);
|
||||
}
|
||||
|
||||
const probe = await ffprobeAudio(chapterOutputTempPath, request.signal);
|
||||
const duration = probe.durationSec ?? (await getAudioDuration(chapterOutputTempPath, request.signal));
|
||||
|
||||
const finalChapterPath = join(intermediateDir, encodeChapterFileName(chapterIndex, data.chapterTitle, format));
|
||||
await unlink(finalChapterPath).catch(() => { });
|
||||
await rename(chapterOutputTempPath, finalChapterPath);
|
||||
const finalChapterName = encodeChapterFileName(chapterIndex, data.chapterTitle, format);
|
||||
const finalChapterBytes = await readFile(chapterOutputTempPath);
|
||||
await putAudiobookObject(bookId, storageUserId, finalChapterName, finalChapterBytes, chapterFileMimeType(format), testNamespace);
|
||||
|
||||
// Remove any existing chapter files for this index (e.g., if the title changed and the
|
||||
// filename changed) and invalidate the combined output now that the chapter is updated.
|
||||
const chapterPrefix = `${String(chapterIndex + 1).padStart(4, '0')}__`;
|
||||
const finalChapterName = basename(finalChapterPath);
|
||||
const existingFiles = await readdir(intermediateDir).catch(() => []);
|
||||
for (const file of existingFiles) {
|
||||
if (!file.startsWith(chapterPrefix)) continue;
|
||||
if (!file.endsWith('.mp3') && !file.endsWith('.m4b')) continue;
|
||||
if (file === finalChapterName) continue;
|
||||
await unlink(join(intermediateDir, file)).catch(() => { });
|
||||
for (const fileName of objectNames) {
|
||||
if (!fileName.startsWith(chapterPrefix)) continue;
|
||||
if (!fileName.endsWith('.mp3') && !fileName.endsWith('.m4b')) continue;
|
||||
if (fileName === finalChapterName) continue;
|
||||
await deleteAudiobookObject(bookId, storageUserId, fileName, testNamespace).catch(() => {});
|
||||
}
|
||||
await unlink(join(intermediateDir, 'complete.mp3')).catch(() => { });
|
||||
await unlink(join(intermediateDir, 'complete.m4b')).catch(() => { });
|
||||
await unlink(join(intermediateDir, 'complete.mp3.manifest.json')).catch(() => { });
|
||||
await unlink(join(intermediateDir, 'complete.m4b.manifest.json')).catch(() => { });
|
||||
|
||||
// Ensure meta exists after first successful chapter.
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.mp3', testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.m4b', testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.mp3.manifest.json', testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, storageUserId, 'complete.m4b.manifest.json', testNamespace).catch(() => {});
|
||||
|
||||
if (!existingSettings && incomingSettings) {
|
||||
await writeFile(metaPath, JSON.stringify(incomingSettings, null, 2)).catch(() => { });
|
||||
await putAudiobookObject(
|
||||
bookId,
|
||||
storageUserId,
|
||||
'audiobook.meta.json',
|
||||
Buffer.from(JSON.stringify(incomingSettings, null, 2), 'utf8'),
|
||||
'application/json; charset=utf-8',
|
||||
testNamespace,
|
||||
);
|
||||
}
|
||||
|
||||
// Clean up input file
|
||||
await unlink(inputPath).catch(console.error);
|
||||
|
||||
// Insert Chapter Record (Denormalized)
|
||||
await db
|
||||
.insert(audiobookChapters)
|
||||
.values({
|
||||
|
|
@ -360,26 +407,24 @@ export async function POST(request: NextRequest) {
|
|||
duration,
|
||||
status: 'completed' as const,
|
||||
bookId,
|
||||
format
|
||||
format,
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
if ((error as Error)?.message === 'ABORTED' || request.signal.aborted) {
|
||||
return NextResponse.json(
|
||||
{ error: 'cancelled' },
|
||||
{ status: 499 }
|
||||
);
|
||||
return NextResponse.json({ error: 'cancelled' }, { status: 499 });
|
||||
}
|
||||
console.error('Error processing audio chapter:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to process audio chapter' },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Failed to process audio chapter' }, { status: 500 });
|
||||
} finally {
|
||||
if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const bookId = request.nextUrl.searchParams.get('bookId');
|
||||
const requestedFormat = request.nextUrl.searchParams.get('format') as TTSAudiobookFormat | null;
|
||||
if (!bookId) {
|
||||
|
|
@ -389,25 +434,15 @@ export async function GET(request: NextRequest) {
|
|||
return NextResponse.json({ error: 'Invalid bookId parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
await ensureAudiobooksV1Ready();
|
||||
if (!(await isAudiobooksV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
const { userId, authEnabled } = ctxOrRes;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const storageUserId = userId ?? unclaimedUserId;
|
||||
const allowedUserIds = authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
// Check if audiobook exists for user OR is unclaimed (similar to documents)
|
||||
const [existingBook] = await db
|
||||
.select({ userId: audiobooks.userId })
|
||||
.from(audiobooks)
|
||||
|
|
@ -416,193 +451,163 @@ export async function GET(request: NextRequest) {
|
|||
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: existingBook.userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
if (!existsSync(intermediateDir)) {
|
||||
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);
|
||||
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const stored = await listStoredChapters(intermediateDir, request.signal);
|
||||
const chapters = stored.map((chapter) => ({
|
||||
title: chapter.title,
|
||||
duration: chapter.durationSec ?? 0,
|
||||
index: chapter.index,
|
||||
format: chapter.format,
|
||||
filePath: chapter.filePath,
|
||||
}));
|
||||
|
||||
const objects = await listAudiobookObjects(bookId, existingBook.userId, testNamespace);
|
||||
const objectNames = objects.map((item) => item.fileName);
|
||||
const chapters = listChapterObjects(objectNames);
|
||||
if (chapters.length === 0) {
|
||||
return NextResponse.json({ error: 'No chapters found' }, { status: 404 });
|
||||
}
|
||||
|
||||
const chapterFormats = new Set(chapters.map((chapter) => chapter.format));
|
||||
if (chapterFormats.size > 1) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Mixed chapter formats detected; reset the audiobook to continue' },
|
||||
{ status: 400 },
|
||||
);
|
||||
return NextResponse.json({ error: 'Mixed chapter formats detected; reset the audiobook to continue' }, { status: 400 });
|
||||
}
|
||||
|
||||
// Sort chapters by index
|
||||
chapters.sort((a, b) => a.index - b.index);
|
||||
const format: TTSAudiobookFormat = requestedFormat ?? (chapters[0]?.format as TTSAudiobookFormat) ?? 'm4b';
|
||||
const outputPath = join(intermediateDir, `complete.${format}`);
|
||||
const manifestPath = join(intermediateDir, `complete.${format}.manifest.json`);
|
||||
const metadataPath = join(intermediateDir, 'metadata.txt');
|
||||
const listPath = join(intermediateDir, 'list.txt');
|
||||
const format: TTSAudiobookFormat = requestedFormat ?? chapters[0].format;
|
||||
const completeName = `complete.${format}`;
|
||||
const manifestName = `${completeName}.manifest.json`;
|
||||
const signature = chapters.map((chapter) => ({ index: chapter.index, fileName: chapter.fileName }));
|
||||
|
||||
const signature = chapters.map((chapter) => ({
|
||||
index: chapter.index,
|
||||
fileName: basename(chapter.filePath),
|
||||
}));
|
||||
|
||||
if (existsSync(outputPath)) {
|
||||
let cached: typeof signature | null = null;
|
||||
if (objectNames.includes(completeName) && objectNames.includes(manifestName)) {
|
||||
try {
|
||||
cached = JSON.parse(await readFile(manifestPath, 'utf8')) as typeof signature;
|
||||
} catch {
|
||||
cached = null;
|
||||
}
|
||||
|
||||
if (cached && JSON.stringify(cached) === JSON.stringify(signature)) {
|
||||
return streamFile(outputPath, format);
|
||||
}
|
||||
|
||||
await unlink(outputPath).catch(() => { });
|
||||
await unlink(manifestPath).catch(() => { });
|
||||
}
|
||||
|
||||
// Ensure we have chapter durations for chapter markers / ordering.
|
||||
for (const chapter of chapters) {
|
||||
if (chapter.duration && chapter.duration > 0) continue;
|
||||
try {
|
||||
const probe = await ffprobeAudio(chapter.filePath, request.signal);
|
||||
if (probe.durationSec && probe.durationSec > 0) {
|
||||
chapter.duration = probe.durationSec;
|
||||
continue;
|
||||
const manifest = JSON.parse((await getAudiobookObjectBuffer(bookId, existingBook.userId, manifestName, testNamespace)).toString('utf8'));
|
||||
if (JSON.stringify(manifest) === JSON.stringify(signature)) {
|
||||
const cached = await getAudiobookObjectBuffer(bookId, existingBook.userId, completeName, testNamespace);
|
||||
return new NextResponse(streamBuffer(cached), {
|
||||
headers: {
|
||||
'Content-Type': chapterFileMimeType(format),
|
||||
'Content-Disposition': `attachment; filename="audiobook.${format}"`,
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch { }
|
||||
|
||||
try {
|
||||
chapter.duration = await getAudioDuration(chapter.filePath, request.signal);
|
||||
} catch {
|
||||
chapter.duration = 0;
|
||||
// Force regeneration below.
|
||||
}
|
||||
|
||||
await deleteAudiobookObject(bookId, existingBook.userId, completeName, testNamespace).catch(() => {});
|
||||
await deleteAudiobookObject(bookId, existingBook.userId, manifestName, testNamespace).catch(() => {});
|
||||
}
|
||||
|
||||
const chapterRows = await db
|
||||
.select({ chapterIndex: audiobookChapters.chapterIndex, duration: audiobookChapters.duration })
|
||||
.from(audiobookChapters)
|
||||
.where(and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, existingBook.userId)));
|
||||
const durationByIndex = new Map<number, number>();
|
||||
for (const row of chapterRows) {
|
||||
durationByIndex.set(row.chapterIndex, Number(row.duration ?? 0));
|
||||
}
|
||||
|
||||
workDir = await mkdtemp(join(tmpdir(), 'openreader-audiobook-combine-'));
|
||||
const metadataPath = join(workDir, 'metadata.txt');
|
||||
const listPath = join(workDir, 'list.txt');
|
||||
const outputPath = join(workDir, completeName);
|
||||
|
||||
const localChapters: Array<{ index: number; title: string; localPath: string; duration: number }> = [];
|
||||
for (const chapter of chapters) {
|
||||
const localPath = join(workDir, chapter.fileName);
|
||||
const bytes = await getAudiobookObjectBuffer(bookId, existingBook.userId, chapter.fileName, testNamespace);
|
||||
await writeFile(localPath, bytes);
|
||||
|
||||
let duration = durationByIndex.get(chapter.index) ?? 0;
|
||||
if (!duration || duration <= 0) {
|
||||
try {
|
||||
const probe = await ffprobeAudio(localPath, request.signal);
|
||||
if (probe.durationSec && probe.durationSec > 0) {
|
||||
duration = probe.durationSec;
|
||||
} else {
|
||||
duration = await getAudioDuration(localPath, request.signal);
|
||||
}
|
||||
} catch {
|
||||
duration = 0;
|
||||
}
|
||||
}
|
||||
|
||||
localChapters.push({
|
||||
index: chapter.index,
|
||||
title: chapter.title,
|
||||
localPath,
|
||||
duration,
|
||||
});
|
||||
}
|
||||
|
||||
// Create chapter metadata file for M4B
|
||||
const metadata: string[] = [];
|
||||
let currentTime = 0;
|
||||
|
||||
for (const chapter of chapters) {
|
||||
for (const chapter of localChapters) {
|
||||
const startMs = Math.floor(currentTime * 1000);
|
||||
currentTime += chapter.duration;
|
||||
const endMs = Math.floor(currentTime * 1000);
|
||||
|
||||
metadata.push('[CHAPTER]', 'TIMEBASE=1/1000', `START=${startMs}`, `END=${endMs}`, `title=${escapeFFMetadata(chapter.title)}`);
|
||||
}
|
||||
|
||||
await writeFile(metadataPath, ';FFMETADATA1\n' + metadata.join('\n'));
|
||||
|
||||
// Create list file for concat
|
||||
await writeFile(
|
||||
listPath,
|
||||
chapters.map(c => `file '${c.filePath}'`).join('\n')
|
||||
localChapters
|
||||
.map((chapter) => `file '${chapter.localPath.replace(/'/g, "'\\''")}'`)
|
||||
.join('\n'),
|
||||
);
|
||||
|
||||
if (format === 'mp3') {
|
||||
// For MP3, re-encode to properly rebuild headers and duration metadata
|
||||
// Using libmp3lame to ensure proper MP3 structure
|
||||
await runFFmpeg([
|
||||
'-f', 'concat',
|
||||
'-safe', '0',
|
||||
'-i', listPath,
|
||||
'-c:a', 'libmp3lame',
|
||||
'-b:a', '64k',
|
||||
outputPath
|
||||
], request.signal);
|
||||
await runFFmpeg(['-f', 'concat', '-safe', '0', '-i', listPath, '-c:a', 'libmp3lame', '-b:a', '64k', outputPath], request.signal);
|
||||
} else {
|
||||
// Combine all files into a single M4B with chapter metadata
|
||||
await runFFmpeg([
|
||||
'-f', 'concat',
|
||||
'-safe', '0',
|
||||
'-i', listPath,
|
||||
'-i', metadataPath,
|
||||
'-map_metadata', '1',
|
||||
'-c:a', 'aac',
|
||||
'-b:a', '64k',
|
||||
'-f', 'mp4',
|
||||
outputPath
|
||||
], request.signal);
|
||||
}
|
||||
|
||||
// Clean up temporary files (but keep the chapters and complete file)
|
||||
await Promise.all([
|
||||
unlink(metadataPath).catch(console.error),
|
||||
unlink(listPath).catch(console.error)
|
||||
]);
|
||||
|
||||
await writeFile(manifestPath, JSON.stringify(signature, null, 2)).catch(() => { });
|
||||
|
||||
// Stream the file back to the client
|
||||
return streamFile(outputPath, format);
|
||||
|
||||
} catch (error) {
|
||||
if ((error as Error)?.message === 'ABORTED' || request.signal.aborted) {
|
||||
return NextResponse.json(
|
||||
{ error: 'cancelled' },
|
||||
{ status: 499 }
|
||||
await runFFmpeg(
|
||||
[
|
||||
'-f',
|
||||
'concat',
|
||||
'-safe',
|
||||
'0',
|
||||
'-i',
|
||||
listPath,
|
||||
'-i',
|
||||
metadataPath,
|
||||
'-map_metadata',
|
||||
'1',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
'64k',
|
||||
'-f',
|
||||
'mp4',
|
||||
outputPath,
|
||||
],
|
||||
request.signal,
|
||||
);
|
||||
}
|
||||
console.error('Error creating M4B:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to create M4B file' },
|
||||
{ status: 500 }
|
||||
|
||||
const outputBytes = await readFile(outputPath);
|
||||
await putAudiobookObject(bookId, existingBook.userId, completeName, outputBytes, chapterFileMimeType(format), testNamespace);
|
||||
await putAudiobookObject(
|
||||
bookId,
|
||||
existingBook.userId,
|
||||
manifestName,
|
||||
Buffer.from(JSON.stringify(signature, null, 2), 'utf8'),
|
||||
'application/json; charset=utf-8',
|
||||
testNamespace,
|
||||
);
|
||||
|
||||
return new NextResponse(streamBuffer(outputBytes), {
|
||||
headers: {
|
||||
'Content-Type': chapterFileMimeType(format),
|
||||
'Content-Disposition': `attachment; filename="audiobook.${format}"`,
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if ((error as Error)?.message === 'ABORTED' || request.signal.aborted) {
|
||||
return NextResponse.json({ error: 'cancelled' }, { status: 499 });
|
||||
}
|
||||
console.error('Error creating full audiobook:', error);
|
||||
return NextResponse.json({ error: 'Failed to create full audiobook file' }, { status: 500 });
|
||||
} finally {
|
||||
if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to stream file
|
||||
function streamFile(filePath: string, format: string) {
|
||||
const stream = createReadStream(filePath);
|
||||
|
||||
const readableWebStream = new ReadableStream({
|
||||
start(controller) {
|
||||
stream.on('data', (chunk) => {
|
||||
controller.enqueue(chunk);
|
||||
});
|
||||
stream.on('end', () => {
|
||||
controller.close();
|
||||
});
|
||||
stream.on('error', (err) => {
|
||||
controller.error(err);
|
||||
});
|
||||
},
|
||||
cancel() {
|
||||
stream.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
const mimeType = format === 'mp3' ? 'audio/mpeg' : 'audio/mp4';
|
||||
|
||||
return new NextResponse(readableWebStream, {
|
||||
headers: {
|
||||
'Content-Type': mimeType,
|
||||
'Content-Disposition': `attachment; filename="audiobook.${format}"`,
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
});
|
||||
}
|
||||
export async function DELETE(request: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const bookId = request.nextUrl.searchParams.get('bookId');
|
||||
if (!bookId) {
|
||||
return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 });
|
||||
|
|
@ -611,23 +616,11 @@ export async function DELETE(request: NextRequest) {
|
|||
return NextResponse.json({ error: 'Invalid bookId parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
await ensureAudiobooksV1Ready();
|
||||
if (!(await isAudiobooksV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
const { userId, authEnabled } = ctxOrRes;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const storageUserId = userId ?? getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const storageUserId = ctxOrRes.userId ?? getUnclaimedUserIdForNamespace(testNamespace);
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
// Delete from DB - with composite PK, we delete by both id and userId
|
||||
const [existingBook] = await db
|
||||
.select({ userId: audiobooks.userId })
|
||||
.from(audiobooks)
|
||||
|
|
@ -637,36 +630,16 @@ export async function DELETE(request: NextRequest) {
|
|||
return NextResponse.json({ error: 'Book not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
// Delete chapters first (no foreign key constraint with composite PK)
|
||||
await db
|
||||
.delete(audiobookChapters)
|
||||
.where(and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, storageUserId)));
|
||||
|
||||
await db.delete(audiobooks).where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId)));
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
|
||||
// If directory doesn't exist, consider it already reset
|
||||
if (!existsSync(intermediateDir)) {
|
||||
return NextResponse.json({ success: true, existed: false });
|
||||
}
|
||||
|
||||
// Recursively delete the entire audiobook directory
|
||||
await rm(intermediateDir, { recursive: true, force: true });
|
||||
|
||||
return NextResponse.json({ success: true, existed: true });
|
||||
const deleted = await deleteAudiobookPrefix(audiobookPrefix(bookId, storageUserId, testNamespace)).catch(() => 0);
|
||||
return NextResponse.json({ success: true, existed: deleted > 0 });
|
||||
} catch (error) {
|
||||
console.error('Error resetting audiobook:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to reset audiobook' },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Failed to reset audiobook' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,15 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { getAudiobooksRootDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore';
|
||||
import { listStoredChapters } from '@/lib/server/audiobook';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudiobookFormat, TTSAudiobookChapter } from '@/types/tts';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { and, eq, inArray } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { audiobooks } from '@/db/schema';
|
||||
import { eq, and, inArray } from 'drizzle-orm';
|
||||
import { ensureDbIndexed } from '@/lib/server/db-indexing';
|
||||
import { audiobooks, audiobookChapters } from '@/db/schema';
|
||||
import { requireAuthContext } from '@/lib/server/auth';
|
||||
import { getAudiobookObjectBuffer, isMissingBlobError, listAudiobookObjects } from '@/lib/server/audiobooks-blobstore';
|
||||
import { decodeChapterFileName } from '@/lib/server/audiobook';
|
||||
import { pruneAudiobookChaptersNotOnDisk } from '@/lib/server/audiobook-prune';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { pruneAudiobookChaptersNotOnDisk, pruneAudiobookIfMissingDir } from '@/lib/server/audiobook-prune';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -22,21 +19,55 @@ function isSafeId(value: string): boolean {
|
|||
return SAFE_ID_REGEX.test(value);
|
||||
}
|
||||
|
||||
function s3NotConfiguredResponse(): NextResponse {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
type ChapterObject = {
|
||||
index: number;
|
||||
title: string;
|
||||
format: TTSAudiobookFormat;
|
||||
fileName: string;
|
||||
};
|
||||
|
||||
function listChapterObjects(fileNames: string[]): ChapterObject[] {
|
||||
const chapters = fileNames
|
||||
.map((fileName) => {
|
||||
const decoded = decodeChapterFileName(fileName);
|
||||
if (!decoded) return null;
|
||||
return {
|
||||
index: decoded.index,
|
||||
title: decoded.title,
|
||||
format: decoded.format,
|
||||
fileName,
|
||||
} satisfies ChapterObject;
|
||||
})
|
||||
.filter((value): value is ChapterObject => Boolean(value))
|
||||
.sort((a, b) => a.index - b.index);
|
||||
|
||||
const deduped = new Map<number, ChapterObject>();
|
||||
for (const chapter of chapters) {
|
||||
const current = deduped.get(chapter.index);
|
||||
if (!current || chapter.fileName > current.fileName) {
|
||||
deduped.set(chapter.index, chapter);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(deduped.values()).sort((a, b) => a.index - b.index);
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) return s3NotConfiguredResponse();
|
||||
|
||||
const bookId = request.nextUrl.searchParams.get('bookId');
|
||||
if (!bookId || !isSafeId(bookId)) {
|
||||
return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
await ensureAudiobooksV1Ready();
|
||||
if (!(await isAudiobooksV1Ready())) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const ctxOrRes = await requireAuthContext(request);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
|
|
@ -46,15 +77,12 @@ export async function GET(request: NextRequest) {
|
|||
const storageUserId = userId ?? unclaimedUserId;
|
||||
const allowedUserIds = authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
|
||||
|
||||
await ensureDbIndexed();
|
||||
|
||||
// Check if audiobook exists for user OR is unclaimed (similar to documents)
|
||||
const [existingBook] = await db
|
||||
.select({ userId: audiobooks.userId })
|
||||
.from(audiobooks)
|
||||
.where(and(eq(audiobooks.id, bookId), inArray(audiobooks.userId, allowedUserIds)));
|
||||
|
||||
if (!existingBook) {
|
||||
// Book doesn't exist for this user or unclaimed - return empty state
|
||||
return NextResponse.json({
|
||||
chapters: [],
|
||||
exists: false,
|
||||
|
|
@ -64,49 +92,56 @@ export async function GET(request: NextRequest) {
|
|||
});
|
||||
}
|
||||
|
||||
const intermediateDir = join(
|
||||
getAudiobooksRootDir({
|
||||
userId: existingBook.userId,
|
||||
authEnabled,
|
||||
namespace: testNamespace,
|
||||
}),
|
||||
`${bookId}-audiobook`,
|
||||
);
|
||||
const objects = await listAudiobookObjects(bookId, existingBook.userId, testNamespace);
|
||||
const objectNames = objects.map((object) => object.fileName);
|
||||
const chapterObjects = listChapterObjects(objectNames);
|
||||
|
||||
if (!existsSync(intermediateDir)) {
|
||||
await pruneAudiobookIfMissingDir(bookId, existingBook.userId, false);
|
||||
return NextResponse.json({
|
||||
chapters: [],
|
||||
exists: false,
|
||||
hasComplete: false,
|
||||
bookId: null,
|
||||
settings: null,
|
||||
});
|
||||
}
|
||||
|
||||
const stored = await listStoredChapters(intermediateDir, request.signal);
|
||||
await pruneAudiobookChaptersNotOnDisk(
|
||||
bookId,
|
||||
existingBook.userId,
|
||||
stored.map((c) => c.index),
|
||||
chapterObjects.map((chapter) => chapter.index),
|
||||
);
|
||||
const chapters: TTSAudiobookChapter[] = stored.map((chapter) => ({
|
||||
|
||||
const chapterRows = await db
|
||||
.select({ chapterIndex: audiobookChapters.chapterIndex, duration: audiobookChapters.duration })
|
||||
.from(audiobookChapters)
|
||||
.where(and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, existingBook.userId)));
|
||||
const durationByIndex = new Map<number, number>();
|
||||
for (const row of chapterRows) {
|
||||
durationByIndex.set(row.chapterIndex, Number(row.duration ?? 0));
|
||||
}
|
||||
|
||||
const chapters: TTSAudiobookChapter[] = chapterObjects.map((chapter) => ({
|
||||
index: chapter.index,
|
||||
title: chapter.title,
|
||||
duration: chapter.durationSec,
|
||||
duration: durationByIndex.get(chapter.index),
|
||||
status: 'completed',
|
||||
bookId,
|
||||
format: chapter.format as TTSAudiobookFormat,
|
||||
format: chapter.format,
|
||||
}));
|
||||
|
||||
let settings: AudiobookGenerationSettings | null = null;
|
||||
try {
|
||||
settings = JSON.parse(await readFile(join(intermediateDir, 'audiobook.meta.json'), 'utf8')) as AudiobookGenerationSettings;
|
||||
} catch {
|
||||
settings = JSON.parse((await getAudiobookObjectBuffer(bookId, existingBook.userId, 'audiobook.meta.json', testNamespace)).toString('utf8')) as AudiobookGenerationSettings;
|
||||
} catch (error) {
|
||||
if (!isMissingBlobError(error)) throw error;
|
||||
settings = null;
|
||||
}
|
||||
|
||||
const hasComplete = existsSync(join(intermediateDir, 'complete.mp3')) || existsSync(join(intermediateDir, 'complete.m4b'));
|
||||
const hasComplete = objectNames.includes('complete.mp3') || objectNames.includes('complete.m4b');
|
||||
const exists = chapters.length > 0 || hasComplete || settings !== null;
|
||||
|
||||
if (!exists) {
|
||||
await db.delete(audiobookChapters).where(and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, existingBook.userId)));
|
||||
await db.delete(audiobooks).where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, existingBook.userId)));
|
||||
return NextResponse.json({
|
||||
chapters: [],
|
||||
exists: false,
|
||||
hasComplete: false,
|
||||
bookId: null,
|
||||
settings: null,
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
chapters,
|
||||
|
|
@ -115,12 +150,8 @@ export async function GET(request: NextRequest) {
|
|||
bookId,
|
||||
settings,
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error fetching chapters:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch chapters' },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ error: 'Failed to fetch chapters' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,144 +0,0 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir, readdir, rename, rm } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
AUDIOBOOKS_V1_DIR,
|
||||
ensureAudiobooksV1Ready,
|
||||
ensureDocumentsV1Ready,
|
||||
isAudiobooksV1Ready,
|
||||
isDocumentsV1Ready,
|
||||
} from '@/lib/server/docstore';
|
||||
import { auth } from '@/lib/server/auth';
|
||||
|
||||
type Mapping = { oldId: string; id: string };
|
||||
|
||||
function isSafeId(value: string): boolean {
|
||||
return /^[a-zA-Z0-9._-]{1,128}$/.test(value);
|
||||
}
|
||||
|
||||
async function mergeDirectoryContents(sourceDir: string, targetDir: string): Promise<{ moved: number; skipped: number }> {
|
||||
let moved = 0;
|
||||
let skipped = 0;
|
||||
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(sourceDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const sourcePath = join(sourceDir, entry.name);
|
||||
const targetPath = join(targetDir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await mkdir(targetPath, { recursive: true });
|
||||
const nested = await mergeDirectoryContents(sourcePath, targetPath);
|
||||
moved += nested.moved;
|
||||
skipped += nested.skipped;
|
||||
|
||||
try {
|
||||
const remaining = await readdir(sourcePath);
|
||||
if (remaining.length === 0) {
|
||||
await rm(sourcePath);
|
||||
}
|
||||
} catch { }
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile()) continue;
|
||||
|
||||
if (existsSync(targetPath)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await rename(sourcePath, targetPath);
|
||||
moved++;
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
async function rekeyAudiobooksV1(mappings: Mapping[]): Promise<{ renamed: number; merged: number; skipped: number }> {
|
||||
let renamed = 0;
|
||||
let merged = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const mapping of mappings) {
|
||||
if (mapping.oldId === mapping.id) continue;
|
||||
const sourceDir = join(AUDIOBOOKS_V1_DIR, `${mapping.oldId}-audiobook`);
|
||||
if (!existsSync(sourceDir)) continue;
|
||||
|
||||
const targetDir = join(AUDIOBOOKS_V1_DIR, `${mapping.id}-audiobook`);
|
||||
if (!existsSync(targetDir)) {
|
||||
try {
|
||||
await rename(sourceDir, targetDir);
|
||||
renamed++;
|
||||
continue;
|
||||
} catch {
|
||||
// Fall through to merge.
|
||||
}
|
||||
}
|
||||
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
const res = await mergeDirectoryContents(sourceDir, targetDir);
|
||||
if (res.moved > 0) merged++;
|
||||
skipped += res.skipped;
|
||||
|
||||
try {
|
||||
const remaining = await readdir(sourceDir);
|
||||
if (remaining.length === 0) {
|
||||
await rm(sourceDir);
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
|
||||
return { renamed, merged, skipped };
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
// Auth check - require session
|
||||
const session = await auth?.api.getSession({ headers: request.headers });
|
||||
if (auth && !session?.user) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
const raw = (await request.json().catch(() => null)) as { mappings?: Mapping[] } | null;
|
||||
const mappings = (raw?.mappings ?? []).filter(
|
||||
(m): m is Mapping => Boolean(m && typeof m.oldId === 'string' && typeof m.id === 'string'),
|
||||
);
|
||||
|
||||
for (const mapping of mappings) {
|
||||
if (!isSafeId(mapping.oldId) || !isSafeId(mapping.id)) {
|
||||
return NextResponse.json({ error: 'Invalid document id mapping' }, { status: 400 });
|
||||
}
|
||||
}
|
||||
|
||||
const documentsMigrated = await ensureDocumentsV1Ready();
|
||||
const audiobooksMigrated = await ensureAudiobooksV1Ready();
|
||||
const rekey = await rekeyAudiobooksV1(mappings);
|
||||
|
||||
const documentsReady = await isDocumentsV1Ready();
|
||||
const audiobooksReady = await isAudiobooksV1Ready();
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
documentsReady,
|
||||
audiobooksReady,
|
||||
documentsMigrated,
|
||||
audiobooksMigrated,
|
||||
rekey,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error running v1 migrations:', error);
|
||||
return NextResponse.json({ error: 'Failed to run v1 migrations' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1,275 +0,0 @@
|
|||
import { createHash } from 'crypto';
|
||||
import { existsSync } from 'fs';
|
||||
import { readdir, readFile, stat, unlink } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { auth } from '@/lib/server/auth';
|
||||
import { DOCUMENTS_V1_DIR } from '@/lib/server/docstore';
|
||||
import { isValidDocumentId, putDocumentBlob } from '@/lib/server/documents-blobstore';
|
||||
import { toDocumentTypeFromName } from '@/lib/server/documents-utils';
|
||||
import { contentTypeForName } from '@/lib/server/library';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import type { DocumentType } from '@/types/documents';
|
||||
import {
|
||||
applyOpenReaderTestNamespacePath,
|
||||
getOpenReaderTestNamespace,
|
||||
getUnclaimedUserIdForNamespace,
|
||||
} from '@/lib/server/test-namespace';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
type V2Body = {
|
||||
deleteLocal?: boolean;
|
||||
dryRun?: boolean;
|
||||
};
|
||||
|
||||
type LegacyDocumentCandidate = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
};
|
||||
|
||||
function isPreconditionFailed(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
|
||||
}
|
||||
|
||||
function extractIdFromFileName(fileName: string): string | null {
|
||||
const match = /^([a-f0-9]{64})__/i.exec(fileName);
|
||||
if (!match) return null;
|
||||
const id = match[1].toLowerCase();
|
||||
return isValidDocumentId(id) ? id : null;
|
||||
}
|
||||
|
||||
function decodeNameFromFileName(fileName: string, id: string): string {
|
||||
const prefix = `${id}__`;
|
||||
if (!fileName.startsWith(prefix)) return `${id}.bin`;
|
||||
const encoded = fileName.slice(prefix.length);
|
||||
try {
|
||||
return decodeURIComponent(encoded);
|
||||
} catch {
|
||||
return `${id}.bin`;
|
||||
}
|
||||
}
|
||||
|
||||
function sniffBinaryDocumentType(bytes: Buffer): Exclude<DocumentType, 'html'> | null {
|
||||
// PDF signature: "%PDF-"
|
||||
if (bytes.length >= 5 && bytes.subarray(0, 5).toString('ascii') === '%PDF-') {
|
||||
return 'pdf';
|
||||
}
|
||||
|
||||
// ZIP signatures: PK..
|
||||
const isZip =
|
||||
bytes.length >= 4 &&
|
||||
bytes[0] === 0x50 &&
|
||||
bytes[1] === 0x4b &&
|
||||
(bytes[2] === 0x03 || bytes[2] === 0x05 || bytes[2] === 0x07) &&
|
||||
(bytes[3] === 0x04 || bytes[3] === 0x06 || bytes[3] === 0x08);
|
||||
if (!isZip) return null;
|
||||
|
||||
// EPUB/DOCX markers usually appear in ZIP local headers near the start.
|
||||
const probe = bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString('latin1');
|
||||
if (probe.includes('application/epub+zip') || probe.includes('META-INF/container.xml')) {
|
||||
return 'epub';
|
||||
}
|
||||
if (probe.includes('[Content_Types].xml') && probe.includes('word/')) {
|
||||
return 'docx';
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeNameForType(name: string, id: string, type: DocumentType): string {
|
||||
if (type === 'html') return name;
|
||||
const expectedExt = type === 'pdf' ? '.pdf' : type === 'epub' ? '.epub' : '.docx';
|
||||
if (name.toLowerCase().endsWith(expectedExt)) return name;
|
||||
const base = name.replace(/\.bin$/i, '');
|
||||
return `${base || id}${expectedExt}`;
|
||||
}
|
||||
|
||||
function contentTypeForDocument(type: DocumentType, name: string): string {
|
||||
if (type === 'pdf') return 'application/pdf';
|
||||
if (type === 'epub') return 'application/epub+zip';
|
||||
if (type === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
||||
return contentTypeForName(name);
|
||||
}
|
||||
|
||||
async function migrateDocumentRowsToBlobHandle(dryRun: boolean): Promise<number> {
|
||||
const rows = (await db.select().from(documents)) as Array<{
|
||||
id: string;
|
||||
userId: string;
|
||||
filePath: string;
|
||||
}>;
|
||||
|
||||
let updated = 0;
|
||||
for (const row of rows) {
|
||||
if (!row.id || !row.userId || row.filePath === row.id) continue;
|
||||
updated++;
|
||||
if (dryRun) continue;
|
||||
await db
|
||||
.update(documents)
|
||||
.set({ filePath: row.id })
|
||||
.where(and(eq(documents.id, row.id), eq(documents.userId, row.userId)));
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
async function seedMissingDocumentRows(
|
||||
userId: string,
|
||||
candidates: LegacyDocumentCandidate[],
|
||||
dryRun: boolean,
|
||||
): Promise<number> {
|
||||
if (candidates.length === 0) return 0;
|
||||
|
||||
const existingRows = (await db.select().from(documents).where(eq(documents.userId, userId))) as Array<{
|
||||
id: string;
|
||||
}>;
|
||||
const existingIds = new Set(existingRows.map((row) => row.id));
|
||||
|
||||
const seen = new Set<string>();
|
||||
const toInsert: LegacyDocumentCandidate[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (seen.has(candidate.id)) continue;
|
||||
seen.add(candidate.id);
|
||||
if (existingIds.has(candidate.id)) continue;
|
||||
toInsert.push(candidate);
|
||||
}
|
||||
|
||||
if (toInsert.length === 0) return 0;
|
||||
if (dryRun) return toInsert.length;
|
||||
|
||||
await db.insert(documents).values(
|
||||
toInsert.map((candidate) => ({
|
||||
id: candidate.id,
|
||||
userId,
|
||||
name: candidate.name,
|
||||
type: candidate.type,
|
||||
size: candidate.size,
|
||||
lastModified: candidate.lastModified,
|
||||
filePath: candidate.id,
|
||||
})),
|
||||
).onConflictDoNothing();
|
||||
|
||||
return toInsert.length;
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth?.api.getSession({ headers: request.headers });
|
||||
if (auth && !session?.user) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'S3 is not configured. Set S3_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const raw = (await request.json().catch(() => ({}))) as V2Body;
|
||||
const dryRun = raw.dryRun === true;
|
||||
const deleteLocal = raw.deleteLocal === true;
|
||||
const testNamespace = getOpenReaderTestNamespace(request.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const docsDir = applyOpenReaderTestNamespacePath(DOCUMENTS_V1_DIR, testNamespace);
|
||||
|
||||
if (!existsSync(docsDir)) {
|
||||
const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
dryRun,
|
||||
deleteLocal,
|
||||
docsDir,
|
||||
filesScanned: 0,
|
||||
uploaded: 0,
|
||||
alreadyPresent: 0,
|
||||
skippedInvalid: 0,
|
||||
deletedLocal: 0,
|
||||
dbRowsUpdated: rowsUpdated,
|
||||
dbRowsSeeded: 0,
|
||||
});
|
||||
}
|
||||
|
||||
const entries = await readdir(docsDir, { withFileTypes: true });
|
||||
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
||||
|
||||
let uploaded = 0;
|
||||
let alreadyPresent = 0;
|
||||
let skippedInvalid = 0;
|
||||
let deletedLocal = 0;
|
||||
const candidates: LegacyDocumentCandidate[] = [];
|
||||
|
||||
for (const fileName of files) {
|
||||
const fullPath = path.join(docsDir, fileName);
|
||||
const bytes = await readFile(fullPath);
|
||||
const fileStats = await stat(fullPath);
|
||||
|
||||
const extractedId = extractIdFromFileName(fileName);
|
||||
const id = extractedId ?? createHash('sha256').update(bytes).digest('hex');
|
||||
if (!isValidDocumentId(id)) {
|
||||
skippedInvalid++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const inferredName = decodeNameFromFileName(fileName, id);
|
||||
const inferredType = toDocumentTypeFromName(inferredName);
|
||||
const type = inferredType === 'html' ? (sniffBinaryDocumentType(bytes) ?? inferredType) : inferredType;
|
||||
const normalizedName = normalizeNameForType(inferredName, id, type);
|
||||
const contentType = contentTypeForDocument(type, normalizedName);
|
||||
const lastModified = Number.isFinite(fileStats.mtimeMs) ? Math.floor(fileStats.mtimeMs) : Date.now();
|
||||
|
||||
candidates.push({
|
||||
id,
|
||||
name: normalizedName,
|
||||
type,
|
||||
size: bytes.length,
|
||||
lastModified,
|
||||
});
|
||||
|
||||
if (!dryRun) {
|
||||
try {
|
||||
await putDocumentBlob(id, bytes, contentType, testNamespace);
|
||||
uploaded++;
|
||||
} catch (error) {
|
||||
if (isPreconditionFailed(error)) {
|
||||
alreadyPresent++;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
await unlink(fullPath).catch(() => {});
|
||||
deletedLocal++;
|
||||
}
|
||||
}
|
||||
|
||||
const rowsUpdated = await migrateDocumentRowsToBlobHandle(dryRun);
|
||||
const rowsSeeded = await seedMissingDocumentRows(unclaimedUserId, candidates, dryRun);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
dryRun,
|
||||
deleteLocal,
|
||||
docsDir,
|
||||
filesScanned: files.length,
|
||||
uploaded,
|
||||
alreadyPresent,
|
||||
skippedInvalid,
|
||||
deletedLocal,
|
||||
dbRowsUpdated: rowsUpdated,
|
||||
dbRowsSeeded: rowsSeeded,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error running v2 migrations:', error);
|
||||
return NextResponse.json({ error: 'Failed to run v2 migrations' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +1,12 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { claimAnonymousData } from '@/lib/server/claim-data';
|
||||
import { auth } from '@/lib/server/auth';
|
||||
import { ensureDbIndexed, getUnclaimedCounts } from '@/lib/server/db-indexing';
|
||||
import { isDocumentsV1Ready } from '@/lib/server/docstore';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { count, ne } from 'drizzle-orm';
|
||||
import { audiobooks, documents } from '@/db/schema';
|
||||
import { count, eq, ne } from 'drizzle-orm';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
|
||||
async function checkClaimMigrationReadiness(): Promise<NextResponse | null> {
|
||||
const documentsV1Ready = await isDocumentsV1Ready();
|
||||
if (!documentsV1Ready) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Document migration is not ready. Run startup migrations first.' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const [legacyRows] = await db
|
||||
.select({ count: count() })
|
||||
.from(documents)
|
||||
|
|
@ -31,6 +22,22 @@ async function checkClaimMigrationReadiness(): Promise<NextResponse | null> {
|
|||
return null;
|
||||
}
|
||||
|
||||
async function getClaimableCounts(unclaimedUserId: string): Promise<{ documents: number; audiobooks: number }> {
|
||||
const [docCount] = await db
|
||||
.select({ count: count() })
|
||||
.from(documents)
|
||||
.where(eq(documents.userId, unclaimedUserId));
|
||||
const [bookCount] = await db
|
||||
.select({ count: count() })
|
||||
.from(audiobooks)
|
||||
.where(eq(audiobooks.userId, unclaimedUserId));
|
||||
|
||||
return {
|
||||
documents: Number(docCount?.count ?? 0),
|
||||
audiobooks: Number(bookCount?.count ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const session = await auth?.api.getSession({ headers: req.headers });
|
||||
|
|
@ -41,8 +48,9 @@ export async function GET(req: NextRequest) {
|
|||
const readiness = await checkClaimMigrationReadiness();
|
||||
if (readiness) return readiness;
|
||||
|
||||
await ensureDbIndexed();
|
||||
const counts = await getUnclaimedCounts();
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const counts = await getClaimableCounts(unclaimedUserId);
|
||||
return NextResponse.json({ success: true, ...counts });
|
||||
} catch (error) {
|
||||
console.error('Error checking claimable data:', error);
|
||||
|
|
@ -60,10 +68,11 @@ export async function POST(req: NextRequest) {
|
|||
const readiness = await checkClaimMigrationReadiness();
|
||||
if (readiness) return readiness;
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
|
||||
const userId = session.user.id;
|
||||
|
||||
await ensureDbIndexed();
|
||||
const result = await claimAnonymousData(userId);
|
||||
const result = await claimAnonymousData(userId, unclaimedUserId, testNamespace);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { spawn } from 'child_process';
|
|||
import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts';
|
||||
import { preprocessSentenceForAudio } from '@/lib/nlp';
|
||||
import { auth } from '@/lib/server/auth';
|
||||
import { getFFmpegPath } from '@/lib/server/ffmpeg-bin';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
|
|
@ -353,7 +354,7 @@ async function alignAudioWithText(
|
|||
await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer)));
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn('ffmpeg', [
|
||||
const ffmpeg = spawn(getFFmpegPath(), [
|
||||
'-y',
|
||||
'-i',
|
||||
inputPath,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import { resolveDocumentId } from '@/lib/dexie';
|
|||
import { RateLimitBanner } from '@/components/auth/RateLimitBanner';
|
||||
import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext';
|
||||
|
||||
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
|
||||
const canExportAudiobook = process.env.NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT !== 'false';
|
||||
|
||||
export default function EPUBPage() {
|
||||
const { id } = useParams();
|
||||
|
|
@ -158,7 +158,7 @@ export default function EPUBPage() {
|
|||
onZoomDecrease={() => setPadPct(p => Math.max(p - 10, 0))}
|
||||
onOpenSettings={() => setIsSettingsOpen(true)}
|
||||
onOpenAudiobook={() => setIsAudiobookModalOpen(true)}
|
||||
isDev={isDev}
|
||||
showAudiobookExport={canExportAudiobook}
|
||||
minZoom={0}
|
||||
maxZoom={100}
|
||||
/>
|
||||
|
|
@ -177,7 +177,7 @@ export default function EPUBPage() {
|
|||
</div>
|
||||
)}
|
||||
</div>
|
||||
{isDev && (
|
||||
{canExportAudiobook && (
|
||||
<AudiobookExportModal
|
||||
isOpen={isAudiobookModalOpen}
|
||||
setIsOpen={setIsAudiobookModalOpen}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import { resolveDocumentId } from '@/lib/dexie';
|
|||
import { RateLimitBanner } from '@/components/auth/RateLimitBanner';
|
||||
import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext';
|
||||
|
||||
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
|
||||
const canExportAudiobook = process.env.NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT !== 'false';
|
||||
|
||||
// Dynamic import for client-side rendering only
|
||||
const PDFViewer = dynamic(
|
||||
|
|
@ -154,7 +154,7 @@ export default function PDFViewerPage() {
|
|||
onZoomDecrease={handleZoomOut}
|
||||
onOpenSettings={() => setIsSettingsOpen(true)}
|
||||
onOpenAudiobook={() => setIsAudiobookModalOpen(true)}
|
||||
isDev={isDev}
|
||||
showAudiobookExport={canExportAudiobook}
|
||||
minZoom={50}
|
||||
maxZoom={300}
|
||||
/>
|
||||
|
|
@ -171,7 +171,7 @@ export default function PDFViewerPage() {
|
|||
<PDFViewer zoomLevel={zoomLevel} />
|
||||
)}
|
||||
</div>
|
||||
{isDev && (
|
||||
{canExportAudiobook && (
|
||||
<AudiobookExportModal
|
||||
isOpen={isAudiobookModalOpen}
|
||||
setIsOpen={setIsAudiobookModalOpen}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ interface DocumentHeaderMenuProps {
|
|||
onZoomDecrease: () => void;
|
||||
onOpenSettings: () => void;
|
||||
onOpenAudiobook?: () => void;
|
||||
isDev?: boolean;
|
||||
showAudiobookExport?: boolean;
|
||||
minZoom?: number;
|
||||
maxZoom?: number;
|
||||
}
|
||||
|
|
@ -23,7 +23,7 @@ export function DocumentHeaderMenu({
|
|||
onZoomDecrease,
|
||||
onOpenSettings,
|
||||
onOpenAudiobook,
|
||||
isDev,
|
||||
showAudiobookExport,
|
||||
minZoom = 0,
|
||||
maxZoom = 100
|
||||
}: DocumentHeaderMenuProps) {
|
||||
|
|
@ -38,7 +38,7 @@ export function DocumentHeaderMenu({
|
|||
min={minZoom}
|
||||
max={maxZoom}
|
||||
/>
|
||||
{isDev && onOpenAudiobook && (
|
||||
{showAudiobookExport && onOpenAudiobook && (
|
||||
<button
|
||||
onClick={onOpenAudiobook}
|
||||
className="inline-flex items-center py-1 px-2 rounded-md border border-offbase bg-base text-foreground text-xs hover:bg-offbase transition-all duration-200 ease-in-out hover:scale-[1.09] hover:text-accent"
|
||||
|
|
@ -102,7 +102,7 @@ export function DocumentHeaderMenu({
|
|||
|
||||
{/* Actions Section */}
|
||||
<div className="p-1">
|
||||
{isDev && onOpenAudiobook && (
|
||||
{showAudiobookExport && onOpenAudiobook && (
|
||||
<MenuItem>
|
||||
{({ active }) => (
|
||||
<button
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import type { TTSAudiobookChapter } from '@/types/tts';
|
|||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
|
||||
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
|
||||
const canExportAudiobook = process.env.NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT !== 'false';
|
||||
const canWordHighlight = isDev || process.env.NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT === 'true';
|
||||
|
||||
const viewTypeTextMapping = [
|
||||
{ id: 'single', name: 'Single Page' },
|
||||
|
|
@ -151,9 +153,9 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-background
|
||||
disabled:opacity-50 disabled:cursor-not-allowed disabled:hover:scale-[1] disabled:hover:bg-accent"
|
||||
onClick={() => setIsAudiobookModalOpen(true)}
|
||||
disabled={!isDev}
|
||||
disabled={!canExportAudiobook}
|
||||
>
|
||||
Export Audiobook {!isDev && '(requires self-hosted)'}
|
||||
Export Audiobook {!canExportAudiobook && '(disabled by configuration)'}
|
||||
</Button>
|
||||
</div>}
|
||||
|
||||
|
|
@ -355,7 +357,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
<input
|
||||
type="checkbox"
|
||||
checked={pdfWordHighlightEnabled && pdfHighlightEnabled}
|
||||
disabled={!pdfHighlightEnabled || !isDev}
|
||||
disabled={!pdfHighlightEnabled || !canWordHighlight}
|
||||
onChange={(e) =>
|
||||
updateConfigKey('pdfWordHighlightEnabled', e.target.checked)
|
||||
}
|
||||
|
|
@ -366,7 +368,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Highlight individual words using audio timestamps generated by whisper.cpp {!isDev && '(requires self-hosted)'}
|
||||
Highlight individual words using audio timestamps generated by whisper.cpp {!canWordHighlight && '(disabled by configuration)'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -392,7 +394,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
<input
|
||||
type="checkbox"
|
||||
checked={epubWordHighlightEnabled && epubHighlightEnabled}
|
||||
disabled={!epubHighlightEnabled || !isDev}
|
||||
disabled={!epubHighlightEnabled || !canWordHighlight}
|
||||
onChange={(e) =>
|
||||
updateConfigKey('epubWordHighlightEnabled', e.target.checked)
|
||||
}
|
||||
|
|
@ -403,7 +405,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Highlight individual words using audio timestamps generated by whisper.cpp {!isDev && '(requires self-hosted)'}
|
||||
Highlight individual words using audio timestamps generated by whisper.cpp {!canWordHighlight && '(disabled by configuration)'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import { createContext, useContext, useEffect, useMemo, useRef, useState, ReactNode } from 'react';
|
||||
import { useLiveQuery } from 'dexie-react-hooks';
|
||||
import { db, getDocumentIdMappings, initDB, migrateLegacyDexieDocumentIdsToSha, updateAppConfig } from '@/lib/dexie';
|
||||
import { db, initDB, migrateLegacyDexieDocumentIdsToSha, updateAppConfig } from '@/lib/dexie';
|
||||
import { APP_CONFIG_DEFAULTS, type ViewType, type SavedVoices, type AppConfigValues, type AppConfigRow } from '@/types/config';
|
||||
import toast from 'react-hot-toast';
|
||||
export type { ViewType } from '@/types/config';
|
||||
|
|
@ -99,58 +99,6 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
|||
const run = async () => {
|
||||
try {
|
||||
await migrateLegacyDexieDocumentIdsToSha();
|
||||
const mappings = await getDocumentIdMappings();
|
||||
|
||||
// Run server-side v1 migrations proactively, since the client may now
|
||||
// reference SHA-based IDs immediately after the Dexie migration.
|
||||
const response = await fetch('/api/migrations/v1', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ mappings }),
|
||||
}).catch(() => null);
|
||||
|
||||
if (response?.ok) {
|
||||
const data = await response.json();
|
||||
const v2ApplyResponse = await fetch('/api/migrations/v2', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ dryRun: false, deleteLocal: false }),
|
||||
}).catch(() => null);
|
||||
const v2ApplyData = v2ApplyResponse?.ok ? await v2ApplyResponse.json().catch(() => null) : null;
|
||||
|
||||
const didMigrateV1 =
|
||||
data.documentsMigrated ||
|
||||
data.audiobooksMigrated ||
|
||||
(data.rekey?.renamed ?? 0) > 0 ||
|
||||
(data.rekey?.merged ?? 0) > 0;
|
||||
|
||||
if (v2ApplyData) {
|
||||
const uploaded = Number(v2ApplyData.uploaded ?? 0);
|
||||
const alreadyPresent = Number(v2ApplyData.alreadyPresent ?? 0);
|
||||
const dbRowsUpdated = Number(v2ApplyData.dbRowsUpdated ?? 0);
|
||||
const dbRowsSeeded = Number(v2ApplyData.dbRowsSeeded ?? 0);
|
||||
const deletedLocal = Number(v2ApplyData.deletedLocal ?? 0);
|
||||
const dbRowsMigrated = dbRowsUpdated + dbRowsSeeded;
|
||||
const didMigrateV2 = uploaded > 0 || dbRowsMigrated > 0 || deletedLocal > 0;
|
||||
|
||||
if (didMigrateV2) {
|
||||
toast.success(
|
||||
`Legacy document migration complete: ${uploaded} uploaded, ${alreadyPresent} already in S3, ${dbRowsMigrated} DB row(s) migrated.`,
|
||||
{ duration: 6000, icon: '📦' },
|
||||
);
|
||||
window.dispatchEvent(new CustomEvent('openreader:documentsChanged', {
|
||||
detail: { reason: 'migration-v2-complete' },
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if (didMigrateV1) {
|
||||
toast.success('Library migration complete', {
|
||||
duration: 5000,
|
||||
icon: '📦',
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Startup migrations failed:', error);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import { readdir } from 'fs/promises';
|
||||
import { getFFprobePath } from '@/lib/server/ffmpeg-bin';
|
||||
|
||||
export type StoredChapter = {
|
||||
index: number;
|
||||
|
|
@ -78,7 +79,7 @@ type ProbeResult = {
|
|||
|
||||
export async function ffprobeAudio(filePath: string, signal?: AbortSignal): Promise<ProbeResult> {
|
||||
return new Promise<ProbeResult>((resolve, reject) => {
|
||||
const ffprobe = spawn('ffprobe', [
|
||||
const ffprobe = spawn(getFFprobePath(), [
|
||||
'-v',
|
||||
'quiet',
|
||||
'-print_format',
|
||||
|
|
|
|||
251
src/lib/server/audiobooks-blobstore.ts
Normal file
251
src/lib/server/audiobooks-blobstore.ts
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
import {
|
||||
DeleteObjectCommand,
|
||||
DeleteObjectsCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getS3Client, getS3Config } from '@/lib/server/s3';
|
||||
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const SAFE_BOOK_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
const SAFE_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/;
|
||||
|
||||
export type AudiobookBlobObject = {
|
||||
key: string;
|
||||
fileName: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
eTag: string | null;
|
||||
};
|
||||
|
||||
function sanitizeNamespace(namespace: string | null): string | null {
|
||||
if (!namespace) return null;
|
||||
if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null;
|
||||
return namespace;
|
||||
}
|
||||
|
||||
function assertSafeBookId(bookId: string): void {
|
||||
if (!SAFE_BOOK_ID_REGEX.test(bookId)) {
|
||||
throw new Error(`Invalid audiobook id: ${bookId}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertSafeUserId(userId: string): void {
|
||||
if (!SAFE_USER_ID_REGEX.test(userId)) {
|
||||
throw new Error(`Invalid user id for audiobook storage scope: ${userId}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertSafeFileName(fileName: string): void {
|
||||
if (!fileName || fileName === '.' || fileName === '..' || fileName.includes('/') || fileName.includes('\\')) {
|
||||
throw new Error(`Invalid audiobook file name: ${fileName}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream {
|
||||
return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function';
|
||||
}
|
||||
|
||||
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
if (Buffer.isBuffer(chunk)) {
|
||||
chunks.push(chunk);
|
||||
} else if (typeof chunk === 'string') {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
} else {
|
||||
chunks.push(Buffer.from(chunk as Uint8Array));
|
||||
}
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async function bodyToBuffer(body: unknown): Promise<Buffer> {
|
||||
if (!body) return Buffer.alloc(0);
|
||||
|
||||
if (body instanceof Uint8Array) return Buffer.from(body);
|
||||
if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
||||
if (body instanceof ArrayBuffer) return Buffer.from(body);
|
||||
|
||||
if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) {
|
||||
const maybe = body as { transformToByteArray?: () => Promise<Uint8Array> };
|
||||
if (typeof maybe.transformToByteArray === 'function') {
|
||||
return Buffer.from(await maybe.transformToByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
if (isNodeReadableStream(body)) {
|
||||
return streamToBuffer(body);
|
||||
}
|
||||
|
||||
throw new Error('Unsupported S3 response body type');
|
||||
}
|
||||
|
||||
export function isPreconditionFailed(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||
return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed';
|
||||
}
|
||||
|
||||
export function isMissingBlobError(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } };
|
||||
if (maybe.$metadata?.httpStatusCode === 404) return true;
|
||||
if (maybe.name === 'NotFound' || maybe.name === 'NoSuchKey') return true;
|
||||
if (maybe.Code === 'NotFound' || maybe.Code === 'NoSuchKey') return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function audiobookPrefix(bookId: string, userId: string, namespace: string | null): string {
|
||||
assertSafeBookId(bookId);
|
||||
assertSafeUserId(userId);
|
||||
const cfg = getS3Config();
|
||||
const ns = sanitizeNamespace(namespace);
|
||||
const nsSegment = ns ? `ns/${ns}/` : '';
|
||||
return `${cfg.prefix}/audiobooks_v1/${nsSegment}users/${encodeURIComponent(userId)}/${bookId}-audiobook/`;
|
||||
}
|
||||
|
||||
export function audiobookKey(bookId: string, userId: string, fileName: string, namespace: string | null): string {
|
||||
assertSafeFileName(fileName);
|
||||
return `${audiobookPrefix(bookId, userId, namespace)}${fileName}`;
|
||||
}
|
||||
|
||||
export async function listAudiobookObjects(bookId: string, userId: string, namespace: string | null): Promise<AudiobookBlobObject[]> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const prefix = audiobookPrefix(bookId, userId, namespace);
|
||||
let continuationToken: string | undefined;
|
||||
const objects: AudiobookBlobObject[] = [];
|
||||
|
||||
do {
|
||||
const listRes = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: cfg.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
}),
|
||||
);
|
||||
|
||||
for (const entry of listRes.Contents ?? []) {
|
||||
const key = entry.Key;
|
||||
if (!key || !key.startsWith(prefix)) continue;
|
||||
const fileName = key.slice(prefix.length);
|
||||
if (!fileName || fileName.includes('/')) continue;
|
||||
objects.push({
|
||||
key,
|
||||
fileName,
|
||||
size: Number(entry.Size ?? 0),
|
||||
lastModified: entry.LastModified?.getTime() ?? 0,
|
||||
eTag: entry.ETag ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
return objects;
|
||||
}
|
||||
|
||||
export async function headAudiobookObject(
|
||||
bookId: string,
|
||||
userId: string,
|
||||
fileName: string,
|
||||
namespace: string | null,
|
||||
): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = audiobookKey(bookId, userId, fileName, namespace);
|
||||
const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
return {
|
||||
contentLength: Number(res.ContentLength ?? 0),
|
||||
contentType: res.ContentType ?? null,
|
||||
eTag: res.ETag ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getAudiobookObjectBuffer(bookId: string, userId: string, fileName: string, namespace: string | null): Promise<Buffer> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = audiobookKey(bookId, userId, fileName, namespace);
|
||||
const res = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
return bodyToBuffer(res.Body);
|
||||
}
|
||||
|
||||
export async function getAudiobookObjectStream(bookId: string, userId: string, fileName: string, namespace: string | null): Promise<unknown> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = audiobookKey(bookId, userId, fileName, namespace);
|
||||
const res = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
return res.Body;
|
||||
}
|
||||
|
||||
export async function putAudiobookObject(
|
||||
bookId: string,
|
||||
userId: string,
|
||||
fileName: string,
|
||||
body: Buffer,
|
||||
contentType: string,
|
||||
namespace: string | null,
|
||||
options?: { ifNoneMatch?: boolean },
|
||||
): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = audiobookKey(bookId, userId, fileName, namespace);
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function deleteAudiobookObject(bookId: string, userId: string, fileName: string, namespace: string | null): Promise<void> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const key = audiobookKey(bookId, userId, fileName, namespace);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
}
|
||||
|
||||
export async function deleteAudiobookPrefix(prefix: string): Promise<number> {
|
||||
const cfg = getS3Config();
|
||||
const client = getS3Client();
|
||||
const cleanedPrefix = prefix.replace(/^\/+/, '');
|
||||
let deleted = 0;
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const listRes = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: cfg.bucket,
|
||||
Prefix: cleanedPrefix,
|
||||
ContinuationToken: continuationToken,
|
||||
}),
|
||||
);
|
||||
|
||||
const keys = (listRes.Contents ?? [])
|
||||
.map((item) => item.Key)
|
||||
.filter((value): value is string => typeof value === 'string' && value.length > 0);
|
||||
|
||||
if (keys.length > 0) {
|
||||
const deleteRes = await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Delete: {
|
||||
Objects: keys.map((Key) => ({ Key })),
|
||||
Quiet: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
deleted += deleteRes.Deleted?.length ?? 0;
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
|
@ -1,85 +1,85 @@
|
|||
import { db } from '@/db';
|
||||
import { documents, audiobooks, audiobookChapters } from '@/db/schema';
|
||||
import { eq, and } from 'drizzle-orm';
|
||||
import fs from 'fs/promises';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { UNCLAIMED_USER_ID } from './docstore';
|
||||
import {
|
||||
UNCLAIMED_USER_ID,
|
||||
getUserAudiobookDir,
|
||||
moveAudiobookToUser,
|
||||
listUserAudiobookIds,
|
||||
} from './docstore';
|
||||
deleteAudiobookObject,
|
||||
getAudiobookObjectBuffer,
|
||||
listAudiobookObjects,
|
||||
putAudiobookObject,
|
||||
} from './audiobooks-blobstore';
|
||||
import { isS3Configured } from './s3';
|
||||
|
||||
import { isAuthEnabled } from '@/lib/server/auth-config';
|
||||
|
||||
export async function claimAnonymousData(userId: string) {
|
||||
if (!isAuthEnabled() || !userId) return { documents: 0, audiobooks: 0 };
|
||||
type AudiobookRow = {
|
||||
id: string;
|
||||
userId: string;
|
||||
title: string;
|
||||
author: string | null;
|
||||
description: string | null;
|
||||
coverPath: string | null;
|
||||
duration: number | null;
|
||||
createdAt: unknown;
|
||||
};
|
||||
|
||||
// Get list of unclaimed audiobook IDs before updating DB
|
||||
const unclaimedBookIds = await listUserAudiobookIds(UNCLAIMED_USER_ID);
|
||||
type AudiobookChapterRow = {
|
||||
id: string;
|
||||
bookId: string;
|
||||
userId: string;
|
||||
chapterIndex: number;
|
||||
title: string;
|
||||
duration: number | null;
|
||||
filePath: string;
|
||||
format: string;
|
||||
};
|
||||
|
||||
// Update Documents - documents use shared storage, only DB update needed
|
||||
const docResult = await db.update(documents)
|
||||
.set({ userId })
|
||||
.where(eq(documents.userId, UNCLAIMED_USER_ID))
|
||||
.returning({ id: documents.id });
|
||||
function contentTypeForAudiobookObject(fileName: string): string {
|
||||
if (fileName.endsWith('.mp3')) return 'audio/mpeg';
|
||||
if (fileName.endsWith('.m4b')) return 'audio/mp4';
|
||||
if (fileName.endsWith('.json')) return 'application/json; charset=utf-8';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
// For audiobooks, we need to:
|
||||
// 1. Move the physical folders from unclaimed to user's folder
|
||||
// 2. Update the DB records
|
||||
async function moveAudiobookBlobScope(
|
||||
bookId: string,
|
||||
fromUserId: string,
|
||||
toUserId: string,
|
||||
namespace: string | null,
|
||||
): Promise<void> {
|
||||
if (fromUserId === toUserId) return;
|
||||
|
||||
let audiobooksClaimedCount = 0;
|
||||
const userDir = getUserAudiobookDir(userId);
|
||||
await fs.mkdir(userDir, { recursive: true });
|
||||
const objects = await listAudiobookObjects(bookId, fromUserId, namespace);
|
||||
if (objects.length === 0) return;
|
||||
|
||||
for (const bookId of unclaimedBookIds) {
|
||||
try {
|
||||
// Move the audiobook folder
|
||||
const moved = await moveAudiobookToUser(bookId, UNCLAIMED_USER_ID, userId);
|
||||
if (moved) {
|
||||
// Update DB - delete old record and insert new one (composite PK requires this)
|
||||
const [oldRecord] = await db.select().from(audiobooks).where(
|
||||
and(eq(audiobooks.id, bookId), eq(audiobooks.userId, UNCLAIMED_USER_ID))
|
||||
);
|
||||
|
||||
if (oldRecord) {
|
||||
// Get chapters
|
||||
const oldChapters = await db.select().from(audiobookChapters).where(
|
||||
and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, UNCLAIMED_USER_ID))
|
||||
);
|
||||
|
||||
// Delete old records
|
||||
await db.delete(audiobookChapters).where(
|
||||
and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, UNCLAIMED_USER_ID))
|
||||
);
|
||||
await db.delete(audiobooks).where(
|
||||
and(eq(audiobooks.id, bookId), eq(audiobooks.userId, UNCLAIMED_USER_ID))
|
||||
);
|
||||
|
||||
// Insert new records with new userId
|
||||
await db.insert(audiobooks).values({
|
||||
...oldRecord,
|
||||
userId,
|
||||
});
|
||||
|
||||
for (const chapter of oldChapters) {
|
||||
await db.insert(audiobookChapters).values({
|
||||
...chapter,
|
||||
userId,
|
||||
});
|
||||
}
|
||||
|
||||
audiobooksClaimedCount++;
|
||||
console.log(`Claimed audiobook: ${bookId}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error claiming audiobook ${bookId}:`, err);
|
||||
}
|
||||
for (const object of objects) {
|
||||
const bytes = await getAudiobookObjectBuffer(bookId, fromUserId, object.fileName, namespace);
|
||||
await putAudiobookObject(
|
||||
bookId,
|
||||
toUserId,
|
||||
object.fileName,
|
||||
bytes,
|
||||
contentTypeForAudiobookObject(object.fileName),
|
||||
namespace,
|
||||
);
|
||||
}
|
||||
|
||||
for (const object of objects) {
|
||||
await deleteAudiobookObject(bookId, fromUserId, object.fileName, namespace).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
export async function claimAnonymousData(userId: string, unclaimedUserId: string = UNCLAIMED_USER_ID, namespace: string | null = null) {
|
||||
if (!isAuthEnabled() || !userId) return { documents: 0, audiobooks: 0 };
|
||||
|
||||
const [documentsClaimed, audiobooksClaimed] = await Promise.all([
|
||||
transferUserDocuments(unclaimedUserId, userId),
|
||||
transferUserAudiobooks(unclaimedUserId, userId, namespace),
|
||||
]);
|
||||
|
||||
return {
|
||||
documents: docResult.length,
|
||||
audiobooks: audiobooksClaimedCount
|
||||
documents: documentsClaimed,
|
||||
audiobooks: audiobooksClaimed,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -120,60 +120,44 @@ export async function transferUserDocuments(
|
|||
* Used when an anonymous user creates a real account.
|
||||
* @returns number of audiobooks transferred
|
||||
*/
|
||||
export async function transferUserAudiobooks(fromUserId: string, toUserId: string): Promise<number> {
|
||||
export async function transferUserAudiobooks(
|
||||
fromUserId: string,
|
||||
toUserId: string,
|
||||
namespace: string | null = null,
|
||||
): Promise<number> {
|
||||
if (!isAuthEnabled() || !fromUserId || !toUserId) return 0;
|
||||
if (fromUserId === toUserId) return 0;
|
||||
|
||||
const bookIds = await listUserAudiobookIds(fromUserId);
|
||||
let transferred = 0;
|
||||
const books = (await db
|
||||
.select()
|
||||
.from(audiobooks)
|
||||
.where(eq(audiobooks.userId, fromUserId))) as AudiobookRow[];
|
||||
if (books.length === 0) return 0;
|
||||
|
||||
const toUserDir = getUserAudiobookDir(toUserId);
|
||||
await fs.mkdir(toUserDir, { recursive: true });
|
||||
|
||||
for (const bookId of bookIds) {
|
||||
try {
|
||||
// Move the audiobook folder
|
||||
const moved = await moveAudiobookToUser(bookId, fromUserId, toUserId);
|
||||
if (moved) {
|
||||
// Update DB - delete old record and insert new one (composite PK)
|
||||
const [oldRecord] = await db.select().from(audiobooks).where(
|
||||
and(eq(audiobooks.id, bookId), eq(audiobooks.userId, fromUserId))
|
||||
);
|
||||
|
||||
if (oldRecord) {
|
||||
// Get chapters
|
||||
const oldChapters = await db.select().from(audiobookChapters).where(
|
||||
and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, fromUserId))
|
||||
);
|
||||
|
||||
// Delete old records
|
||||
await db.delete(audiobookChapters).where(
|
||||
and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, fromUserId))
|
||||
);
|
||||
await db.delete(audiobooks).where(
|
||||
and(eq(audiobooks.id, bookId), eq(audiobooks.userId, fromUserId))
|
||||
);
|
||||
|
||||
// Insert new records with new userId
|
||||
await db.insert(audiobooks).values({
|
||||
...oldRecord,
|
||||
userId: toUserId,
|
||||
});
|
||||
|
||||
for (const chapter of oldChapters) {
|
||||
await db.insert(audiobookChapters).values({
|
||||
...chapter,
|
||||
userId: toUserId,
|
||||
});
|
||||
}
|
||||
|
||||
transferred++;
|
||||
console.log(`Transferred audiobook ${bookId} from ${fromUserId} to ${toUserId}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error transferring audiobook ${bookId}:`, err);
|
||||
if (isS3Configured()) {
|
||||
for (const book of books) {
|
||||
await moveAudiobookBlobScope(book.id, fromUserId, toUserId, namespace);
|
||||
}
|
||||
}
|
||||
|
||||
return transferred;
|
||||
await db
|
||||
.insert(audiobooks)
|
||||
.values(books.map((book) => ({ ...book, userId: toUserId })))
|
||||
.onConflictDoNothing();
|
||||
|
||||
const chapters = (await db
|
||||
.select()
|
||||
.from(audiobookChapters)
|
||||
.where(eq(audiobookChapters.userId, fromUserId))) as AudiobookChapterRow[];
|
||||
if (chapters.length > 0) {
|
||||
await db
|
||||
.insert(audiobookChapters)
|
||||
.values(chapters.map((chapter) => ({ ...chapter, userId: toUserId })))
|
||||
.onConflictDoNothing();
|
||||
}
|
||||
|
||||
await db.delete(audiobookChapters).where(eq(audiobookChapters.userId, fromUserId));
|
||||
await db.delete(audiobooks).where(eq(audiobooks.userId, fromUserId));
|
||||
|
||||
return books.length;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,186 +0,0 @@
|
|||
import fs from 'fs/promises';
|
||||
import { existsSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { isAuthEnabled } from '@/lib/server/auth-config';
|
||||
import { db } from '@/db';
|
||||
import { audiobookChapters, audiobooks, documents } from '@/db/schema';
|
||||
import { and, count, eq } from 'drizzle-orm';
|
||||
import { listStoredChapters } from '@/lib/server/audiobook';
|
||||
import { AUDIOBOOKS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const MIGRATIONS_DIR = path.join(DOCSTORE_DIR, '.migrations');
|
||||
const STATE_PATH = path.join(MIGRATIONS_DIR, 'db-index.json');
|
||||
|
||||
type DbIndexState = {
|
||||
indexedAt: number;
|
||||
mode: 'auth' | 'noauth';
|
||||
};
|
||||
|
||||
let inflight: Promise<void> | null = null;
|
||||
let memoryIndexedMode: DbIndexState['mode'] | null = null;
|
||||
|
||||
async function readState(): Promise<DbIndexState | null> {
|
||||
try {
|
||||
const raw = await fs.readFile(STATE_PATH, 'utf8');
|
||||
return JSON.parse(raw) as DbIndexState;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeState(): Promise<void> {
|
||||
await fs.mkdir(MIGRATIONS_DIR, { recursive: true });
|
||||
const state: DbIndexState = {
|
||||
indexedAt: Date.now(),
|
||||
mode: isAuthEnabled() ? 'auth' : 'noauth',
|
||||
};
|
||||
await fs.writeFile(STATE_PATH, JSON.stringify(state, null, 2));
|
||||
}
|
||||
|
||||
async function hasAudiobookFilesystemContent(mode: DbIndexState['mode']): Promise<boolean> {
|
||||
const audiobookDir = mode === 'auth' ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR;
|
||||
try {
|
||||
const entries = await fs.readdir(audiobookDir, { withFileTypes: true });
|
||||
return entries.some((entry) => entry.isDirectory() && entry.name.endsWith('-audiobook'));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function isAudiobookIndexedForUser(id: string, userId: string): Promise<boolean> {
|
||||
const result = await db
|
||||
.select({ id: audiobooks.id })
|
||||
.from(audiobooks)
|
||||
.where(and(eq(audiobooks.id, id), eq(audiobooks.userId, userId)));
|
||||
return result.length > 0;
|
||||
}
|
||||
|
||||
async function migrateLegacyAudiobooksToUnclaimed(): Promise<number> {
|
||||
if (!existsSync(AUDIOBOOKS_V1_DIR)) return 0;
|
||||
|
||||
const unclaimedDir = getUnclaimedAudiobookDir();
|
||||
await fs.mkdir(unclaimedDir, { recursive: true });
|
||||
|
||||
const entries = await fs.readdir(AUDIOBOOKS_V1_DIR, { withFileTypes: true });
|
||||
let migrated = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
|
||||
const sourceDir = path.join(AUDIOBOOKS_V1_DIR, entry.name);
|
||||
const targetDir = path.join(unclaimedDir, entry.name);
|
||||
if (existsSync(targetDir)) continue;
|
||||
|
||||
try {
|
||||
await fs.rename(sourceDir, targetDir);
|
||||
migrated++;
|
||||
console.log(`Migrated legacy audiobook to unclaimed: ${entry.name}`);
|
||||
} catch (err) {
|
||||
console.error(`Error migrating legacy audiobook ${entry.name}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
return migrated;
|
||||
}
|
||||
|
||||
export async function getUnclaimedCounts(): Promise<{ documents: number; audiobooks: number }> {
|
||||
const [docCount] = await db.select({ count: count() }).from(documents).where(eq(documents.userId, UNCLAIMED_USER_ID));
|
||||
const [bookCount] = await db.select({ count: count() }).from(audiobooks).where(eq(audiobooks.userId, UNCLAIMED_USER_ID));
|
||||
|
||||
return {
|
||||
documents: Number(docCount?.count ?? 0),
|
||||
audiobooks: Number(bookCount?.count ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function scanAndPopulateAudiobookDb(): Promise<void> {
|
||||
const authEnabled = isAuthEnabled();
|
||||
console.log('Scanning file system for un-indexed audiobooks...');
|
||||
|
||||
if (authEnabled) {
|
||||
await migrateLegacyAudiobooksToUnclaimed();
|
||||
}
|
||||
|
||||
const audiobookScanDir = authEnabled ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR;
|
||||
if (!existsSync(audiobookScanDir)) return;
|
||||
|
||||
const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
|
||||
const bookId = entry.name.replace('-audiobook', '');
|
||||
if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue;
|
||||
|
||||
const dirPath = path.join(audiobookScanDir, entry.name);
|
||||
|
||||
let title = 'Unknown Title';
|
||||
try {
|
||||
const metaPath = path.join(dirPath, 'audiobook.meta.json');
|
||||
const metaContent = await fs.readFile(metaPath, 'utf8');
|
||||
JSON.parse(metaContent);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
const chapters = await listStoredChapters(dirPath);
|
||||
const totalDuration = chapters.reduce((acc, chapter) => acc + (chapter.durationSec || 0), 0);
|
||||
if (chapters.length > 0) title = chapters[0].title || title;
|
||||
|
||||
await db.insert(audiobooks).values({
|
||||
id: bookId,
|
||||
userId: UNCLAIMED_USER_ID,
|
||||
title,
|
||||
duration: totalDuration,
|
||||
});
|
||||
console.log(`Indexed audiobook: ${bookId}`);
|
||||
|
||||
for (const chapter of chapters) {
|
||||
await db.insert(audiobookChapters).values({
|
||||
id: `${bookId}-${chapter.index}`,
|
||||
bookId,
|
||||
userId: UNCLAIMED_USER_ID,
|
||||
chapterIndex: chapter.index,
|
||||
title: chapter.title,
|
||||
duration: chapter.durationSec || 0,
|
||||
filePath: chapter.filePath,
|
||||
format: chapter.format,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureAudiobooksIndexed(): Promise<void> {
|
||||
const mode: DbIndexState['mode'] = isAuthEnabled() ? 'auth' : 'noauth';
|
||||
if (memoryIndexedMode === mode) return;
|
||||
|
||||
inflight ??= (async () => {
|
||||
const hasState = existsSync(STATE_PATH) ? await readState() : null;
|
||||
if (hasState && hasState.mode === mode) {
|
||||
const [counts, fsHasAudiobooks] = await Promise.all([
|
||||
getUnclaimedCounts(),
|
||||
hasAudiobookFilesystemContent(mode),
|
||||
]);
|
||||
const audiobooksOk = counts.audiobooks > 0 || !fsHasAudiobooks;
|
||||
if (audiobooksOk) {
|
||||
memoryIndexedMode = mode;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await fs.mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
await scanAndPopulateAudiobookDb();
|
||||
await writeState();
|
||||
memoryIndexedMode = mode;
|
||||
})().finally(() => {
|
||||
inflight = null;
|
||||
});
|
||||
|
||||
await inflight;
|
||||
}
|
||||
|
||||
export async function ensureDbIndexed(): Promise<void> {
|
||||
await ensureAudiobooksIndexed();
|
||||
}
|
||||
|
|
@ -1,210 +1,11 @@
|
|||
import { createHash } from 'crypto';
|
||||
import { spawn } from 'child_process';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir, readdir, readFile, rename, rm, stat, unlink, utimes, writeFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { decodeChapterTitleTag, encodeChapterFileName, encodeChapterTitleTag, ffprobeAudio } from '@/lib/server/audiobook';
|
||||
|
||||
export const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
export const DOCUMENTS_V1_DIR = path.join(DOCSTORE_DIR, 'documents_v1');
|
||||
export const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1');
|
||||
export const AUDIOBOOKS_USERS_DIR = path.join(DOCSTORE_DIR, 'audiobooks_users_v1');
|
||||
|
||||
export const UNCLAIMED_USER_ID = 'unclaimed';
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
/**
|
||||
* Get the audiobook directory for a specific user when auth is enabled.
|
||||
* Returns path like: docstore/audiobooks_users_v1/{userId}
|
||||
*/
|
||||
export function getUserAudiobookDir(userId: string): string {
|
||||
// Sanitize userId to prevent path traversal
|
||||
const safeUserId = userId.replace(/[^a-zA-Z0-9._-]/g, '');
|
||||
if (!safeUserId || safeUserId === '.' || safeUserId === '..' || safeUserId.includes('..')) {
|
||||
throw new Error('Invalid userId for audiobook directory');
|
||||
}
|
||||
return path.join(AUDIOBOOKS_USERS_DIR, safeUserId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the unclaimed audiobooks directory for pre-auth content.
|
||||
* Returns path like: docstore/audiobooks_users_v1/unclaimed
|
||||
*/
|
||||
export function getUnclaimedAudiobookDir(): string {
|
||||
return path.join(AUDIOBOOKS_USERS_DIR, UNCLAIMED_USER_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the base audiobooks directory for request context, including optional test namespace.
|
||||
* - When auth is disabled or userId is absent: docstore/audiobooks_v1[/<namespace>]
|
||||
* - When auth is enabled: docstore/audiobooks_users_v1/{userId}[/<namespace>]
|
||||
*/
|
||||
export function getAudiobooksRootDir({
|
||||
userId,
|
||||
authEnabled,
|
||||
namespace,
|
||||
}: {
|
||||
userId: string | null;
|
||||
authEnabled: boolean;
|
||||
namespace: string | null;
|
||||
}): string {
|
||||
const baseDir = !authEnabled || !userId ? AUDIOBOOKS_V1_DIR : getUserAudiobookDir(userId);
|
||||
if (!namespace) return baseDir;
|
||||
|
||||
const safe = namespace.trim();
|
||||
if (!safe || safe === '.' || safe === '..' || safe.includes('..')) return baseDir;
|
||||
if (!SAFE_NAMESPACE_REGEX.test(safe)) return baseDir;
|
||||
|
||||
const resolved = path.resolve(baseDir, safe);
|
||||
if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Move an audiobook folder from one user's directory to another.
|
||||
* Used for claiming unclaimed audiobooks or transferring on account linking.
|
||||
* @returns true if moved successfully, false if source doesn't exist
|
||||
*/
|
||||
export async function moveAudiobookToUser(
|
||||
bookId: string,
|
||||
fromUserId: string,
|
||||
toUserId: string
|
||||
): Promise<boolean> {
|
||||
const sourceDir = path.join(getUserAudiobookDir(fromUserId), `${bookId}-audiobook`);
|
||||
const targetUserDir = getUserAudiobookDir(toUserId);
|
||||
const targetDir = path.join(targetUserDir, `${bookId}-audiobook`);
|
||||
|
||||
if (!existsSync(sourceDir)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Ensure target user directory exists
|
||||
await mkdir(targetUserDir, { recursive: true });
|
||||
|
||||
// If target already exists, we need to merge or skip
|
||||
if (existsSync(targetDir)) {
|
||||
// Target exists - merge contents (move files that don't exist in target)
|
||||
const result = await mergeDirectoryContents(sourceDir, targetDir);
|
||||
// Try to remove source if empty
|
||||
try {
|
||||
const remaining = await readdir(sourceDir);
|
||||
if (remaining.length === 0) {
|
||||
await rm(sourceDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return result.moved > 0 || result.skipped > 0;
|
||||
}
|
||||
|
||||
// Simple rename/move
|
||||
await rename(sourceDir, targetDir);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all audiobook IDs in a user's directory.
|
||||
*/
|
||||
export async function listUserAudiobookIds(userId: string): Promise<string[]> {
|
||||
const userDir = getUserAudiobookDir(userId);
|
||||
if (!existsSync(userDir)) return [];
|
||||
|
||||
const entries = await readdir(userDir, { withFileTypes: true });
|
||||
const bookIds: string[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
bookIds.push(entry.name.replace('-audiobook', ''));
|
||||
}
|
||||
|
||||
return bookIds;
|
||||
}
|
||||
|
||||
const MIGRATIONS_DIR = path.join(DOCSTORE_DIR, '.migrations');
|
||||
const MIGRATIONS_STATE_PATH = path.join(MIGRATIONS_DIR, 'state.json');
|
||||
|
||||
type MigrationState = {
|
||||
documentsV1Migrated?: boolean;
|
||||
audiobooksV1Migrated?: boolean;
|
||||
updatedAt?: number;
|
||||
};
|
||||
|
||||
type LegacyDocumentMetadata = {
|
||||
id: string;
|
||||
name: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
type: string;
|
||||
};
|
||||
|
||||
function isLegacyDocumentMetadata(value: unknown): value is LegacyDocumentMetadata {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const v = value as Record<string, unknown>;
|
||||
return (
|
||||
typeof v.id === 'string' &&
|
||||
typeof v.name === 'string' &&
|
||||
typeof v.size === 'number' &&
|
||||
typeof v.lastModified === 'number' &&
|
||||
typeof v.type === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
async function loadMigrationState(): Promise<MigrationState> {
|
||||
try {
|
||||
return JSON.parse(await readFile(MIGRATIONS_STATE_PATH, 'utf8')) as MigrationState;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async function saveMigrationState(update: Partial<MigrationState>): Promise<void> {
|
||||
const state = await loadMigrationState();
|
||||
const next: MigrationState = {
|
||||
documentsV1Migrated: state.documentsV1Migrated,
|
||||
audiobooksV1Migrated: state.audiobooksV1Migrated,
|
||||
...update,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await mkdir(MIGRATIONS_DIR, { recursive: true });
|
||||
await writeFile(MIGRATIONS_STATE_PATH, JSON.stringify(next, null, 2));
|
||||
}
|
||||
|
||||
async function hasLegacyDocumentFiles(): Promise<boolean> {
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(DOCSTORE_DIR, { withFileTypes: true });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (!entry.name.endsWith('.json')) continue;
|
||||
|
||||
const metadataPath = path.join(DOCSTORE_DIR, entry.name);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(metadataPath, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!isLegacyDocumentMetadata(parsed)) continue;
|
||||
|
||||
const contentPath = path.join(DOCSTORE_DIR, `${parsed.id}.${parsed.type}`);
|
||||
if (!existsSync(contentPath)) continue;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function isDocumentsV1Ready(): Promise<boolean> {
|
||||
if (!existsSync(DOCSTORE_DIR) || !existsSync(DOCUMENTS_V1_DIR)) return false;
|
||||
const state = await loadMigrationState();
|
||||
if (!state.documentsV1Migrated) return false;
|
||||
if (await hasLegacyDocumentFiles()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function safeDocumentName(rawName: string, fallback: string): string {
|
||||
const baseName = path.basename(rawName || fallback);
|
||||
|
|
@ -212,456 +13,15 @@ function safeDocumentName(rawName: string, fallback: string): string {
|
|||
}
|
||||
|
||||
export function getMigratedDocumentFileName(id: string, name: string): string {
|
||||
const normalizedName = safeDocumentName(name, `${id}.bin`);
|
||||
const prefix = `${id}__`;
|
||||
const encodedName = encodeURIComponent(name);
|
||||
const encodedName = encodeURIComponent(normalizedName);
|
||||
let targetFileName = `${prefix}${encodedName}`;
|
||||
|
||||
// Ensure total filename length is within safe limits (e.g. 240 chars).
|
||||
// If too long, use a deterministic hash of the name instead of the full encoded name.
|
||||
// Keep migrated document filenames under conservative filesystem length limits.
|
||||
if (targetFileName.length > 240) {
|
||||
const nameHash = createHash('sha256').update(name).digest('hex').slice(0, 32);
|
||||
const nameHash = createHash('sha256').update(normalizedName).digest('hex').slice(0, 32);
|
||||
targetFileName = `${prefix}truncated-${nameHash}`;
|
||||
}
|
||||
return targetFileName;
|
||||
}
|
||||
|
||||
export async function ensureDocumentsV1Ready(): Promise<boolean> {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
await mkdir(DOCUMENTS_V1_DIR, { recursive: true });
|
||||
|
||||
const state = await loadMigrationState();
|
||||
if (state.documentsV1Migrated && !(await hasLegacyDocumentFiles())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!(await hasLegacyDocumentFiles())) {
|
||||
await saveMigrationState({ documentsV1Migrated: true });
|
||||
return false;
|
||||
}
|
||||
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(DOCSTORE_DIR, { withFileTypes: true });
|
||||
} catch {
|
||||
entries = [];
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (!entry.name.endsWith('.json')) continue;
|
||||
|
||||
const metadataPath = path.join(DOCSTORE_DIR, entry.name);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(metadataPath, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!isLegacyDocumentMetadata(parsed)) continue;
|
||||
const metadata = parsed;
|
||||
|
||||
const contentPath = path.join(DOCSTORE_DIR, `${metadata.id}.${metadata.type}`);
|
||||
let contentStat: Awaited<ReturnType<typeof stat>>;
|
||||
try {
|
||||
contentStat = await stat(contentPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!contentStat.isFile()) continue;
|
||||
|
||||
const content = await readFile(contentPath);
|
||||
const id = createHash('sha256').update(content).digest('hex');
|
||||
const fallbackName = `${id}.${metadata.type}`;
|
||||
const name = safeDocumentName(metadata.name, fallbackName);
|
||||
|
||||
const targetFileName = getMigratedDocumentFileName(id, name);
|
||||
const targetPath = path.join(DOCUMENTS_V1_DIR, targetFileName);
|
||||
|
||||
if (!existsSync(targetPath)) {
|
||||
await writeFile(targetPath, content);
|
||||
if (Number.isFinite(metadata.lastModified) && metadata.lastModified > 0) {
|
||||
const stamp = new Date(metadata.lastModified);
|
||||
await utimes(targetPath, stamp, stamp).catch(() => { });
|
||||
}
|
||||
}
|
||||
|
||||
await unlink(metadataPath).catch(() => { });
|
||||
await unlink(contentPath).catch(() => { });
|
||||
}
|
||||
|
||||
await saveMigrationState({ documentsV1Migrated: !(await hasLegacyDocumentFiles()) });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function hasLegacyAudiobookDirs(): Promise<boolean> {
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(DOCSTORE_DIR, { withFileTypes: true });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
return entries.some((entry) => entry.isDirectory() && entry.name.endsWith('-audiobook'));
|
||||
}
|
||||
|
||||
async function hasLegacyAudiobookChapterLayout(): Promise<boolean> {
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(AUDIOBOOKS_V1_DIR, { withFileTypes: true });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
|
||||
const dir = path.join(AUDIOBOOKS_V1_DIR, entry.name);
|
||||
let files: string[] = [];
|
||||
try {
|
||||
files = await readdir(dir);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
// Per-audiobook settings file is the new format; ignore it.
|
||||
if (file === 'audiobook.meta.json') continue;
|
||||
|
||||
if (file.endsWith('.meta.json')) return true;
|
||||
if (/^\d+-chapter\.(mp3|m4b)$/i.test(file)) return true;
|
||||
if (/^[a-f0-9]{64}\.(mp3|m4b)$/i.test(file)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function isAudiobooksV1Ready(): Promise<boolean> {
|
||||
if (!existsSync(DOCSTORE_DIR) || !existsSync(AUDIOBOOKS_V1_DIR)) return false;
|
||||
const state = await loadMigrationState();
|
||||
if (!state.audiobooksV1Migrated) return false;
|
||||
const legacyDirsPresent = await hasLegacyAudiobookDirs();
|
||||
const legacyChaptersPresent = await hasLegacyAudiobookChapterLayout();
|
||||
if (legacyDirsPresent || legacyChaptersPresent) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
async function mergeDirectoryContents(sourceDir: string, targetDir: string): Promise<{ moved: number; skipped: number }> {
|
||||
let moved = 0;
|
||||
let skipped = 0;
|
||||
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(sourceDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const sourcePath = path.join(sourceDir, entry.name);
|
||||
const targetPath = path.join(targetDir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await mkdir(targetPath, { recursive: true });
|
||||
const nested = await mergeDirectoryContents(sourcePath, targetPath);
|
||||
moved += nested.moved;
|
||||
skipped += nested.skipped;
|
||||
|
||||
try {
|
||||
const remaining = await readdir(sourcePath);
|
||||
if (remaining.length === 0) {
|
||||
await rm(sourcePath);
|
||||
}
|
||||
} catch { }
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile()) continue;
|
||||
|
||||
if (existsSync(targetPath)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await rename(sourcePath, targetPath);
|
||||
moved++;
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
export async function ensureAudiobooksV1Ready(): Promise<boolean> {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
await mkdir(AUDIOBOOKS_V1_DIR, { recursive: true });
|
||||
// Also ensure the user-specific audiobooks directory exists for auth-enabled scenarios
|
||||
await mkdir(AUDIOBOOKS_USERS_DIR, { recursive: true });
|
||||
|
||||
const state = await loadMigrationState();
|
||||
const legacyDirsPresent = await hasLegacyAudiobookDirs();
|
||||
const legacyChaptersPresent = await hasLegacyAudiobookChapterLayout();
|
||||
|
||||
if (state.audiobooksV1Migrated && !legacyDirsPresent && !legacyChaptersPresent) {
|
||||
const stateRaw = state as unknown as Record<string, unknown>;
|
||||
const allowedKeys = new Set(['documentsV1Migrated', 'audiobooksV1Migrated', 'updatedAt']);
|
||||
const hasExtraKeys = Object.keys(stateRaw).some((key) => !allowedKeys.has(key));
|
||||
if (hasExtraKeys) {
|
||||
await saveMigrationState({ audiobooksV1Migrated: true });
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(DOCSTORE_DIR, { withFileTypes: true });
|
||||
} catch {
|
||||
entries = [];
|
||||
}
|
||||
|
||||
if (legacyDirsPresent) {
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
|
||||
const sourceDir = path.join(DOCSTORE_DIR, entry.name);
|
||||
const targetDir = path.join(AUDIOBOOKS_V1_DIR, entry.name);
|
||||
|
||||
try {
|
||||
if (!existsSync(targetDir)) {
|
||||
await rename(sourceDir, targetDir);
|
||||
continue;
|
||||
}
|
||||
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
await mergeDirectoryContents(sourceDir, targetDir);
|
||||
|
||||
try {
|
||||
const remaining = await readdir(sourceDir);
|
||||
if (remaining.length === 0) {
|
||||
await rm(sourceDir);
|
||||
} else {
|
||||
console.warn(`Legacy audiobook dir not fully migrated (kept): ${sourceDir}`);
|
||||
}
|
||||
} catch { }
|
||||
} catch (error) {
|
||||
console.error('Error migrating legacy audiobook directory:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (legacyDirsPresent || legacyChaptersPresent) {
|
||||
await normalizeAudiobookChapterLayout();
|
||||
}
|
||||
|
||||
const finalLegacyRemaining = await hasLegacyAudiobookDirs();
|
||||
const finalLegacyChaptersRemaining = await hasLegacyAudiobookChapterLayout();
|
||||
await saveMigrationState({ audiobooksV1Migrated: !finalLegacyRemaining && !finalLegacyChaptersRemaining });
|
||||
return true;
|
||||
}
|
||||
|
||||
type LegacyChapterMeta = {
|
||||
title?: string;
|
||||
duration?: number;
|
||||
index?: number;
|
||||
format?: string;
|
||||
};
|
||||
|
||||
async function runProcess(command: string, args: string[]): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args);
|
||||
let stderr = '';
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`${command} exited with code ${code}: ${stderr}`));
|
||||
});
|
||||
child.on('error', (err) => reject(err));
|
||||
});
|
||||
}
|
||||
|
||||
async function rewriteAudioTitleTag(inputPath: string, outputPath: string, format: 'mp3' | 'm4b', titleTag: string): Promise<void> {
|
||||
const baseArgs = ['-y', '-i', inputPath, '-metadata', `title=${titleTag}`];
|
||||
if (format === 'mp3') {
|
||||
await runProcess('ffmpeg', [...baseArgs, '-c', 'copy', '-write_id3v2', '1', '-id3v2_version', '3', outputPath]);
|
||||
return;
|
||||
}
|
||||
await runProcess('ffmpeg', [...baseArgs, '-c', 'copy', '-f', 'mp4', outputPath]);
|
||||
}
|
||||
|
||||
async function transcodeWithTitleTag(inputPath: string, outputPath: string, format: 'mp3' | 'm4b', titleTag: string): Promise<void> {
|
||||
if (format === 'mp3') {
|
||||
await runProcess('ffmpeg', [
|
||||
'-y',
|
||||
'-i',
|
||||
inputPath,
|
||||
'-c:a',
|
||||
'libmp3lame',
|
||||
'-b:a',
|
||||
'64k',
|
||||
'-metadata',
|
||||
`title=${titleTag}`,
|
||||
outputPath,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
await runProcess('ffmpeg', [
|
||||
'-y',
|
||||
'-i',
|
||||
inputPath,
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
'64k',
|
||||
'-metadata',
|
||||
`title=${titleTag}`,
|
||||
'-f',
|
||||
'mp4',
|
||||
outputPath,
|
||||
]);
|
||||
}
|
||||
|
||||
async function normalizeAudiobookDirectoryChapterLayout(intermediateDir: string): Promise<void> {
|
||||
let files: string[] = [];
|
||||
try {
|
||||
files = await readdir(intermediateDir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove any combined output files from older layouts.
|
||||
await unlink(path.join(intermediateDir, 'complete.mp3')).catch(() => { });
|
||||
await unlink(path.join(intermediateDir, 'complete.m4b')).catch(() => { });
|
||||
await unlink(path.join(intermediateDir, 'metadata.txt')).catch(() => { });
|
||||
await unlink(path.join(intermediateDir, 'list.txt')).catch(() => { });
|
||||
|
||||
const metaFiles = files.filter((file) => file.endsWith('.meta.json'));
|
||||
const migratedIndices = new Set<number>();
|
||||
|
||||
for (const metaFile of metaFiles) {
|
||||
const metaPath = path.join(intermediateDir, metaFile);
|
||||
let metaRaw: unknown;
|
||||
try {
|
||||
metaRaw = JSON.parse(await readFile(metaPath, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const meta = metaRaw as LegacyChapterMeta;
|
||||
const index = Number(meta.index);
|
||||
if (!Number.isFinite(index) || !Number.isInteger(index) || index < 0) continue;
|
||||
|
||||
const format = meta.format === 'mp3' ? 'mp3' : 'm4b';
|
||||
const sourceAudio = path.join(intermediateDir, `${index}-chapter.${format}`);
|
||||
if (!existsSync(sourceAudio)) {
|
||||
await unlink(metaPath).catch(() => { });
|
||||
continue;
|
||||
}
|
||||
|
||||
const titleTag = encodeChapterTitleTag(index, meta.title ?? `Chapter ${index + 1}`);
|
||||
const taggedTemp = path.join(intermediateDir, `${index}.tagged.tmp.${format}`);
|
||||
|
||||
try {
|
||||
await rewriteAudioTitleTag(sourceAudio, taggedTemp, format, titleTag);
|
||||
} catch {
|
||||
await transcodeWithTitleTag(sourceAudio, taggedTemp, format, titleTag);
|
||||
}
|
||||
|
||||
const finalName = encodeChapterFileName(index, meta.title ?? `Chapter ${index + 1}`, format);
|
||||
const finalPath = path.join(intermediateDir, finalName);
|
||||
await unlink(finalPath).catch(() => { });
|
||||
await rename(taggedTemp, finalPath);
|
||||
|
||||
await unlink(sourceAudio).catch(() => { });
|
||||
await unlink(metaPath).catch(() => { });
|
||||
migratedIndices.add(index);
|
||||
}
|
||||
|
||||
// Migrate any remaining legacy chapter files without metadata.
|
||||
files = await readdir(intermediateDir).catch(() => []);
|
||||
for (const file of files) {
|
||||
const match = /^(\d+)-chapter\.(mp3|m4b)$/i.exec(file);
|
||||
if (!match) continue;
|
||||
const index = Number(match[1]);
|
||||
if (!Number.isInteger(index) || index < 0) continue;
|
||||
if (migratedIndices.has(index)) continue;
|
||||
|
||||
const format = match[2].toLowerCase() as 'mp3' | 'm4b';
|
||||
const sourceAudio = path.join(intermediateDir, file);
|
||||
const titleTag = encodeChapterTitleTag(index, `Chapter ${index + 1}`);
|
||||
const taggedTemp = path.join(intermediateDir, `${index}.tagged.tmp.${format}`);
|
||||
|
||||
try {
|
||||
await rewriteAudioTitleTag(sourceAudio, taggedTemp, format, titleTag);
|
||||
} catch {
|
||||
await transcodeWithTitleTag(sourceAudio, taggedTemp, format, titleTag);
|
||||
}
|
||||
|
||||
const finalName = encodeChapterFileName(index, `Chapter ${index + 1}`, format);
|
||||
const finalPath = path.join(intermediateDir, finalName);
|
||||
await unlink(finalPath).catch(() => { });
|
||||
await rename(taggedTemp, finalPath);
|
||||
|
||||
await unlink(sourceAudio).catch(() => { });
|
||||
}
|
||||
|
||||
// Rename any sha-named chapter files from previous runs into the index__title scheme.
|
||||
files = await readdir(intermediateDir).catch(() => []);
|
||||
const shaCandidates = files.filter((file) => /^[a-f0-9]{64}\.(mp3|m4b)$/i.test(file));
|
||||
for (const file of shaCandidates) {
|
||||
const sourceAudio = path.join(intermediateDir, file);
|
||||
const format = file.toLowerCase().endsWith('.mp3') ? 'mp3' : 'm4b';
|
||||
|
||||
let decoded: { index: number; title: string } | null = null;
|
||||
try {
|
||||
const probe = await ffprobeAudio(sourceAudio);
|
||||
decoded = probe.titleTag ? decodeChapterTitleTag(probe.titleTag) : null;
|
||||
} catch {
|
||||
decoded = null;
|
||||
}
|
||||
if (!decoded) continue;
|
||||
|
||||
const finalName = encodeChapterFileName(decoded.index, decoded.title, format);
|
||||
const finalPath = path.join(intermediateDir, finalName);
|
||||
await unlink(finalPath).catch(() => { });
|
||||
await rename(sourceAudio, finalPath).catch(() => { });
|
||||
}
|
||||
|
||||
// Remove any leftover input temp files.
|
||||
files = await readdir(intermediateDir).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (/^\d+-input\.mp3$/i.test(file)) {
|
||||
await unlink(path.join(intermediateDir, file)).catch(() => { });
|
||||
}
|
||||
if (file.endsWith('.meta.json') && file !== 'audiobook.meta.json') {
|
||||
await unlink(path.join(intermediateDir, file)).catch(() => { });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeAudiobookChapterLayout(): Promise<void> {
|
||||
let entries: Array<import('fs').Dirent> = [];
|
||||
try {
|
||||
entries = await readdir(AUDIOBOOKS_V1_DIR, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.name.endsWith('-audiobook')) continue;
|
||||
const dir = path.join(AUDIOBOOKS_V1_DIR, entry.name);
|
||||
try {
|
||||
await normalizeAudiobookDirectoryChapterLayout(dir);
|
||||
} catch (error) {
|
||||
console.error('Error migrating audiobook chapter layout:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
47
src/lib/server/ffmpeg-bin.ts
Normal file
47
src/lib/server/ffmpeg-bin.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { existsSync } from 'fs';
|
||||
import ffmpegStatic from 'ffmpeg-static';
|
||||
import ffprobeStatic from 'ffprobe-static';
|
||||
|
||||
function normalizePath(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string {
|
||||
if (envValue) {
|
||||
if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) {
|
||||
throw new Error(`${envVarName} points to a missing binary: ${envValue}`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
if (!bundledValue) {
|
||||
throw new Error(
|
||||
`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`,
|
||||
);
|
||||
}
|
||||
|
||||
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) {
|
||||
throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`);
|
||||
}
|
||||
return bundledValue;
|
||||
}
|
||||
|
||||
export function getFFmpegPath(): string {
|
||||
return resolveBinary(
|
||||
normalizePath(process.env.FFMPEG_BIN),
|
||||
normalizePath(ffmpegStatic),
|
||||
'FFMPEG_BIN',
|
||||
'ffmpeg-static',
|
||||
);
|
||||
}
|
||||
|
||||
export function getFFprobePath(): string {
|
||||
return resolveBinary(
|
||||
normalizePath(process.env.FFPROBE_BIN),
|
||||
normalizePath(ffprobeStatic?.path),
|
||||
'FFPROBE_BIN',
|
||||
'ffprobe-static',
|
||||
);
|
||||
}
|
||||
|
|
@ -1,11 +1,106 @@
|
|||
import { ListObjectsV2Command } from '@aws-sdk/client-s3';
|
||||
import { and, eq, inArray, like } from 'drizzle-orm';
|
||||
import { db } from '../src/db';
|
||||
import { audiobooks, audiobookChapters, documents } from '../src/db/schema';
|
||||
import { deleteDocumentPrefix } from '../src/lib/server/documents-blobstore';
|
||||
import { getS3Config, isS3Configured } from '../src/lib/server/s3';
|
||||
import { deleteAudiobookPrefix } from '../src/lib/server/audiobooks-blobstore';
|
||||
import { getS3Client, getS3Config, isS3Configured } from '../src/lib/server/s3';
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
if (items.length === 0) return [];
|
||||
const groups: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) {
|
||||
groups.push(items.slice(i, i + size));
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
async function listKeysByPrefix(prefix: string): Promise<string[]> {
|
||||
const config = getS3Config();
|
||||
const client = getS3Client();
|
||||
let continuationToken: string | undefined;
|
||||
const keys: string[] = [];
|
||||
|
||||
do {
|
||||
const response = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: config.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
}),
|
||||
);
|
||||
|
||||
for (const entry of response.Contents ?? []) {
|
||||
if (entry.Key) keys.push(entry.Key);
|
||||
}
|
||||
continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
function parseAudiobookScopeFromKey(
|
||||
key: string,
|
||||
audiobooksNsRootPrefix: string,
|
||||
): { userId: string; bookId: string } | null {
|
||||
if (!key.startsWith(audiobooksNsRootPrefix)) return null;
|
||||
const rel = key.slice(audiobooksNsRootPrefix.length);
|
||||
const parts = rel.split('/');
|
||||
// ns/<namespace>/users/<userId>/<bookId>-audiobook/<file>
|
||||
if (parts.length < 5) return null;
|
||||
if (parts[1] !== 'users') return null;
|
||||
const encodedUserId = parts[2];
|
||||
const dirName = parts[3];
|
||||
if (!encodedUserId || !dirName.endsWith('-audiobook')) return null;
|
||||
const bookId = dirName.slice(0, -'-audiobook'.length);
|
||||
if (!bookId) return null;
|
||||
|
||||
let userId: string;
|
||||
try {
|
||||
userId = decodeURIComponent(encodedUserId);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return { userId, bookId };
|
||||
}
|
||||
|
||||
export default async function globalTeardown(): Promise<void> {
|
||||
// Always clear namespaced no-auth SQL rows from prior runs.
|
||||
await db.delete(audiobookChapters).where(like(audiobookChapters.userId, 'unclaimed::%'));
|
||||
await db.delete(audiobooks).where(like(audiobooks.userId, 'unclaimed::%'));
|
||||
await db.delete(documents).where(like(documents.userId, 'unclaimed::%'));
|
||||
|
||||
if (!isS3Configured()) return;
|
||||
|
||||
const config = getS3Config();
|
||||
const nsRootPrefix = `${config.prefix}/documents_v1/ns/`;
|
||||
await deleteDocumentPrefix(nsRootPrefix);
|
||||
}
|
||||
const docsNsRootPrefix = `${config.prefix}/documents_v1/ns/`;
|
||||
const audiobooksNsRootPrefix = `${config.prefix}/audiobooks_v1/ns/`;
|
||||
|
||||
// Remove SQL audiobook rows for namespaced objects (covers auth claim flows too).
|
||||
const audiobookKeys = await listKeysByPrefix(audiobooksNsRootPrefix);
|
||||
const byUser = new Map<string, Set<string>>();
|
||||
for (const key of audiobookKeys) {
|
||||
const scope = parseAudiobookScopeFromKey(key, audiobooksNsRootPrefix);
|
||||
if (!scope) continue;
|
||||
let set = byUser.get(scope.userId);
|
||||
if (!set) {
|
||||
set = new Set<string>();
|
||||
byUser.set(scope.userId, set);
|
||||
}
|
||||
set.add(scope.bookId);
|
||||
}
|
||||
|
||||
for (const [userId, bookIds] of byUser) {
|
||||
for (const ids of chunk(Array.from(bookIds), 200)) {
|
||||
await db
|
||||
.delete(audiobookChapters)
|
||||
.where(and(eq(audiobookChapters.userId, userId), inArray(audiobookChapters.bookId, ids)));
|
||||
await db
|
||||
.delete(audiobooks)
|
||||
.where(and(eq(audiobooks.userId, userId), inArray(audiobooks.id, ids)));
|
||||
}
|
||||
}
|
||||
|
||||
await deleteDocumentPrefix(docsNsRootPrefix);
|
||||
await deleteAudiobookPrefix(audiobooksNsRootPrefix);
|
||||
}
|
||||
|
|
|
|||
52
tests/unit/audiobooks-blobstore.spec.ts
Normal file
52
tests/unit/audiobooks-blobstore.spec.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import { test, expect } from '@playwright/test';
|
||||
import {
|
||||
audiobookKey,
|
||||
audiobookPrefix,
|
||||
isMissingBlobError,
|
||||
isPreconditionFailed,
|
||||
} from '../../src/lib/server/audiobooks-blobstore';
|
||||
|
||||
function configureS3Env() {
|
||||
process.env.S3_BUCKET = process.env.S3_BUCKET || 'test-bucket';
|
||||
process.env.S3_REGION = process.env.S3_REGION || 'us-east-1';
|
||||
process.env.S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID || 'test-access';
|
||||
process.env.S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY || 'test-secret';
|
||||
process.env.S3_PREFIX = 'openreader-test';
|
||||
}
|
||||
|
||||
test.describe('audiobooks-blobstore', () => {
|
||||
test.beforeAll(() => {
|
||||
configureS3Env();
|
||||
});
|
||||
|
||||
test('builds audiobook prefix with namespace', () => {
|
||||
const prefix = audiobookPrefix('book-123', 'user-abc', 'worker1');
|
||||
expect(prefix).toBe('openreader-test/audiobooks_v1/ns/worker1/users/user-abc/book-123-audiobook/');
|
||||
});
|
||||
|
||||
test('builds audiobook prefix without namespace', () => {
|
||||
const prefix = audiobookPrefix('book-123', 'unclaimed', null);
|
||||
expect(prefix).toBe('openreader-test/audiobooks_v1/users/unclaimed/book-123-audiobook/');
|
||||
});
|
||||
|
||||
test('builds key for chapter file', () => {
|
||||
const key = audiobookKey('book-123', 'user-abc', '0001__Chapter%201.mp3', null);
|
||||
expect(key).toBe('openreader-test/audiobooks_v1/users/user-abc/book-123-audiobook/0001__Chapter%201.mp3');
|
||||
});
|
||||
|
||||
test('rejects invalid file names in keys', () => {
|
||||
expect(() => audiobookKey('book-123', 'user-abc', '../bad.mp3', null)).toThrow(/Invalid audiobook file name/);
|
||||
});
|
||||
|
||||
test('detects missing blob errors', () => {
|
||||
expect(isMissingBlobError({ name: 'NoSuchKey' })).toBeTruthy();
|
||||
expect(isMissingBlobError({ $metadata: { httpStatusCode: 404 } })).toBeTruthy();
|
||||
expect(isMissingBlobError({ name: 'AccessDenied' })).toBeFalsy();
|
||||
});
|
||||
|
||||
test('detects precondition failed errors', () => {
|
||||
expect(isPreconditionFailed({ name: 'PreconditionFailed' })).toBeTruthy();
|
||||
expect(isPreconditionFailed({ $metadata: { httpStatusCode: 412 } })).toBeTruthy();
|
||||
expect(isPreconditionFailed({ $metadata: { httpStatusCode: 409 } })).toBeFalsy();
|
||||
});
|
||||
});
|
||||
11
vercel.json
Normal file
11
vercel.json
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"$schema": "https://openapi.vercel.sh/vercel.json",
|
||||
"functions": {
|
||||
"app/api/audiobook/route.ts": {
|
||||
"memory": 3009
|
||||
},
|
||||
"app/api/whisper/route.ts": {
|
||||
"memory": 3009
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue