From 2c23dc904371c4cf66dfcdd874943f72f5aff2c9 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 3 Feb 2026 12:17:30 -0700 Subject: [PATCH] refactor(db): remove conditional database usage from API layer Ensure all metadata operations consistently use the database layer, removing filesystem-only fallback paths and conditional DB checks. - Simplified migration scripts to always run on startup - Updated document/audiobook APIs to always query DB - Added ensureDbIndexed() calls across all routes - Extracted test namespace utilities to dedicated module - Removed migration-manager.ts (functionality consolidated) - Updated rate limiter to assume DB is always available BREAKING CHANGE: Database is now required in all configurations. When auth is disabled, SQLite is used by default at /app/docstore/sqlite3.db. --- template.env => .env.example | 6 +- README.md | 11 +- drizzle.config.pg.ts | 6 +- drizzle/scripts/generate.mjs | 16 +- .../{migrate-if-auth.mjs => migrate.mjs} | 9 - package.json | 4 +- src/app/api/audiobook/chapter/route.ts | 99 +++---- src/app/api/audiobook/route.ts | 132 +++++---- src/app/api/audiobook/status/route.ts | 75 ++---- src/app/api/documents/route.ts | 135 +++------- src/app/api/user/claim/route.ts | 27 +- src/components/auth/ClaimDataModal.tsx | 6 +- src/db/index.ts | 6 - src/lib/server/claim-data.ts | 183 +------------ src/lib/server/db-indexing.ts | 254 ++++++++++++++++++ src/lib/server/docstore.ts | 52 +--- src/lib/server/migration-manager.ts | 178 ------------ src/lib/server/rate-limiter.ts | 12 - src/lib/server/test-namespace.ts | 30 +++ tests/export.spec.ts | 123 ++++++--- 20 files changed, 596 insertions(+), 768 deletions(-) rename template.env => .env.example (84%) rename drizzle/scripts/{migrate-if-auth.mjs => migrate.mjs} (72%) create mode 100644 src/lib/server/db-indexing.ts delete mode 100644 src/lib/server/migration-manager.ts create mode 100644 src/lib/server/test-namespace.ts diff --git a/template.env b/.env.example similarity index 84% rename from template.env rename to .env.example index 6e50229..f0aa30c 100644 --- a/template.env +++ b/.env.example @@ -9,12 +9,14 @@ API_KEY=api_key_optional # Path to your local whisper.cpp CLI binary for STT timestamp generation WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli +# Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables. +# Defaults to SQLite at docstore/sqlite3.db when not set. +POSTGRES_URL= + # Auth (recommended for contributors and public instances) # (Optional) Auth is only enabled when **both** BETTER_AUTH_SECRET and BETTER_AUTH_URL are set BETTER_AUTH_URL=http://localhost:3003 # Your externally facing URL for this app BETTER_AUTH_SECRET=some_random_secret_key # Generate with `openssl rand -base64 32` -# (Optional) Backend DB used only when authentication is enabled, defaults to SQLite when not set -POSTGRES_URL= # (Optional) Sign in w/ GitHub Configuration GITHUB_CLIENT_ID= GITHUB_CLIENT_SECRET= diff --git a/README.md b/README.md index 3753019..040c20c 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,8 @@ OpenReader WebUI is an open source text to speech document reader web app built > - `API_BASE` should point to your TTS API server's base URL (if running Kokoro-FastAPI locally in Docker, use `http://host.docker.internal:8880/v1`). > - `BETTER_AUTH_URL` should be your externally-facing URL for this app (for example `https://reader.example.com` or `http://localhost:3003`). > - To enable auth, set **both** `BETTER_AUTH_URL` and `BETTER_AUTH_SECRET` generated with `openssl rand -base64 32`. - > - If you set `POSTGRES_URL`, the container will attempt to run migrations against it. Ensure the database is accessible. + > - OpenReader always uses a backend DB for server-side metadata. By default it uses SQLite at `/app/docstore/sqlite3.db` (persisted when `/app/docstore` is mounted). + > - If you set `POSTGRES_URL`, the container uses Postgres instead of SQLite and will run migrations against it on startup. Ensure the database is accessible.
Docker environment variables (Click to expand) @@ -85,7 +86,7 @@ OpenReader WebUI is an open source text to speech document reader web app built | `API_KEY` | Default TTS API key | `none` or your provider key | | `BETTER_AUTH_URL` | Enables auth when set with `BETTER_AUTH_SECRET` | External URL for this app, e.g. `http://localhost:3003` or `https://reader.example.com` | | `BETTER_AUTH_SECRET` | Enables auth when set with `BETTER_AUTH_URL` | Generate with `openssl rand -base64 32` | - | `POSTGRES_URL` | Use Postgres for auth storage instead of SQLite | If set, startup migrations target Postgres | + | `POSTGRES_URL` | Use Postgres for server DB (metadata + auth tables) instead of SQLite | If set, startup migrations target Postgres | | `GITHUB_CLIENT_ID` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_SECRET` | | `GITHUB_CLIENT_SECRET` | Optional GitHub OAuth sign-in | Requires `GITHUB_CLIENT_ID` | @@ -241,7 +242,7 @@ Optionally required for different features: 3. Configure the environment: ```bash - cp template.env .env + cp .env.example .env # Edit .env with your configuration settings ``` @@ -258,10 +259,10 @@ Optionally required for different features: > > Note: The base URL for the TTS API should be accessible and relative to the Next.js server -4. Run auth DB migrations: +4. Run DB migrations: - **Production / Docker**: Migrations run automatically on startup via `pnpm start`. - - **Development**: Run explicitly: + - **Development**: When using `pnpm dev`, it needs to be run explicitly: ```bash pnpm migrate diff --git a/drizzle.config.pg.ts b/drizzle.config.pg.ts index af2aded..75b384e 100644 --- a/drizzle.config.pg.ts +++ b/drizzle.config.pg.ts @@ -3,9 +3,11 @@ import * as dotenv from 'dotenv'; dotenv.config(); -const url = process.env.POSTGRES_URL; +let url = process.env.POSTGRES_URL; if (!url) { - throw new Error('POSTGRES_URL is required for drizzle.config.pg.ts'); + //throw new Error('POSTGRES_URL is required for drizzle.config.pg.ts'); + console.warn('[drizzle.config.pg.ts] POSTGRES_URL is not set; using a placeholder URL.'); + url = 'postgresql://placeholder:placeholder@localhost:5432/placeholder'; } export default { diff --git a/drizzle/scripts/generate.mjs b/drizzle/scripts/generate.mjs index 448e8e6..e112795 100644 --- a/drizzle/scripts/generate.mjs +++ b/drizzle/scripts/generate.mjs @@ -20,18 +20,10 @@ function loadEnvFiles() { loadEnvFiles(); -const authEnabled = Boolean(process.env.BETTER_AUTH_SECRET && process.env.BETTER_AUTH_URL); - -if (!authEnabled) { - // When auth is disabled, the app must not touch sqlite/postgres at all. - console.log('[generate] Skipping (auth disabled). Missing BETTER_AUTH_SECRET and/or BETTER_AUTH_URL.'); - process.exit(0); -} - -if (!process.env.POSTGRES_URL) { - console.error('[generate] POSTGRES_URL is required to generate postgres migrations.'); - process.exit(1); -} +// if (!process.env.POSTGRES_URL) { +// console.error('[generate] POSTGRES_URL is required to generate postgres migrations.'); +// process.exit(1); +// } const extraArgs = process.argv.slice(2); diff --git a/drizzle/scripts/migrate-if-auth.mjs b/drizzle/scripts/migrate.mjs similarity index 72% rename from drizzle/scripts/migrate-if-auth.mjs rename to drizzle/scripts/migrate.mjs index c190068..bcc4154 100644 --- a/drizzle/scripts/migrate-if-auth.mjs +++ b/drizzle/scripts/migrate.mjs @@ -20,15 +20,6 @@ function loadEnvFiles() { loadEnvFiles(); -const authEnabled = Boolean(process.env.BETTER_AUTH_SECRET && process.env.BETTER_AUTH_URL); - -if (!authEnabled) { - // When auth is disabled, the app must not touch sqlite/postgres at all. - // That includes running migrations which can create/open DB files. - console.log('[migrate] Skipping (auth disabled). Missing BETTER_AUTH_SECRET and/or BETTER_AUTH_URL.'); - process.exit(0); -} - const extraArgs = process.argv.slice(2); const hasConfigArg = extraArgs.includes('--config'); diff --git a/package.json b/package.json index aa9db01..d376aa1 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,10 @@ "scripts": { "dev": "next dev --turbopack -p 3003", "build": "next build", - "start": "node drizzle/scripts/migrate-if-auth.mjs && next start -p 3003", + "start": "node drizzle/scripts/migrate.mjs && next start -p 3003", "lint": "next lint", "test": "playwright test", - "migrate": "node drizzle/scripts/migrate-if-auth.mjs", + "migrate": "node drizzle/scripts/migrate.mjs", "generate": "node drizzle/scripts/generate.mjs" }, "dependencies": { diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 609d352..2c2dfbb 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -1,13 +1,15 @@ import { NextRequest, NextResponse } from 'next/server'; import { createReadStream, existsSync } from 'fs'; import { readdir, unlink } from 'fs/promises'; -import { join, resolve } from 'path'; -import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, UNCLAIMED_USER_ID, isAudiobooksV1Ready } from '@/lib/server/docstore'; +import { join } from 'path'; +import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore'; import { findStoredChapterByIndex } from '@/lib/server/audiobook'; import { db } from '@/db'; import { audiobooks, audiobookChapters } from '@/db/schema'; -import { and, eq, or } from 'drizzle-orm'; +import { and, eq, inArray } from 'drizzle-orm'; import { requireAuthContext } from '@/lib/server/auth'; +import { ensureDbIndexed } from '@/lib/server/db-indexing'; +import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; export const dynamic = 'force-dynamic'; @@ -17,27 +19,14 @@ export const dynamic = 'force-dynamic'; * When auth is enabled, returns the user-specific directory. */ function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string { - const raw = request.headers.get('x-openreader-test-namespace')?.trim(); - - const getTestNamespacePath = (baseDir: string): string => { - if (!raw) return baseDir; - const safe = raw.replace(/[^a-zA-Z0-9._-]/g, ''); - if (!safe || safe === '.' || safe === '..' || safe.includes('..')) { - return baseDir; - } - const resolved = resolve(baseDir, safe); - if (!resolved.startsWith(resolve(baseDir) + '/')) { - return baseDir; - } - return resolved; - }; + const namespace = getOpenReaderTestNamespace(request.headers); if (!authEnabled || !userId) { - return getTestNamespacePath(AUDIOBOOKS_V1_DIR); + return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace); } const userDir = getUserAudiobookDir(userId); - return getTestNamespacePath(userDir); + return applyOpenReaderTestNamespacePath(userDir, namespace); } export async function GET(request: NextRequest) { @@ -60,6 +49,7 @@ export async function GET(request: NextRequest) { ); } + await ensureAudiobooksV1Ready(); if (!(await isAudiobooksV1Ready())) { return NextResponse.json( { error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' }, @@ -70,30 +60,23 @@ export async function GET(request: NextRequest) { const ctxOrRes = await requireAuthContext(request); if (ctxOrRes instanceof Response) return ctxOrRes; const { userId, authEnabled } = ctxOrRes; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = userId ?? unclaimedUserId; + const allowedUserIds = authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + await ensureDbIndexed(); // Verify ownership with composite PK - allow access to user's own OR unclaimed audiobooks - if (authEnabled && db && userId) { - const [existingBook] = await db.select().from(audiobooks).where( - and( - eq(audiobooks.id, bookId), - or(eq(audiobooks.userId, userId), eq(audiobooks.userId, UNCLAIMED_USER_ID)) - ) - ); - if (!existingBook) { - return NextResponse.json({ error: 'Book not found' }, { status: 404 }); - } + const [existingBook] = await db + .select({ userId: audiobooks.userId }) + .from(audiobooks) + .where(and(eq(audiobooks.id, bookId), inArray(audiobooks.userId, allowedUserIds))); + if (!existingBook) { + return NextResponse.json({ error: 'Book not found' }, { status: 404 }); } - // Get the audiobook directory - check user's directory first, then unclaimed - let intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`); - - // If not found in user's directory and auth is enabled, check unclaimed directory - if (!existsSync(intermediateDir) && authEnabled && userId) { - const unclaimedDir = join(getAudiobooksRootDir(request, UNCLAIMED_USER_ID, authEnabled), `${bookId}-audiobook`); - if (existsSync(unclaimedDir)) { - intermediateDir = unclaimedDir; - } - } + const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`); const chapter = await findStoredChapterByIndex(intermediateDir, chapterIndex, request.signal); if (!chapter || !existsSync(chapter.filePath)) { @@ -160,6 +143,7 @@ export async function DELETE(request: NextRequest) { ); } + await ensureAudiobooksV1Ready(); if (!(await isAudiobooksV1Ready())) { return NextResponse.json( { error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' }, @@ -170,27 +154,30 @@ export async function DELETE(request: NextRequest) { const ctxOrRes = await requireAuthContext(request); if (ctxOrRes instanceof Response) return ctxOrRes; const { userId, authEnabled } = ctxOrRes; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const storageUserId = userId ?? getUnclaimedUserIdForNamespace(testNamespace); + + await ensureDbIndexed(); // Verify ownership and delete from DB with composite PK - if (authEnabled && db && userId) { - const [existingBook] = await db.select().from(audiobooks).where( - and(eq(audiobooks.id, bookId), eq(audiobooks.userId, userId)) - ); - if (!existingBook) { - return NextResponse.json({ error: 'Book not found' }, { status: 404 }); - } - - // Delete from DB - await db.delete(audiobookChapters).where( - and( - eq(audiobookChapters.bookId, bookId), - eq(audiobookChapters.userId, userId), - eq(audiobookChapters.chapterIndex, chapterIndex) - ), - ); + const [existingBook] = await db + .select({ userId: audiobooks.userId }) + .from(audiobooks) + .where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId))); + if (!existingBook) { + return NextResponse.json({ error: 'Book not found' }, { status: 404 }); } - const intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`); + // Delete from DB + await db.delete(audiobookChapters).where( + and( + eq(audiobookChapters.bookId, bookId), + eq(audiobookChapters.userId, storageUserId), + eq(audiobookChapters.chapterIndex, chapterIndex), + ), + ); + + const intermediateDir = join(getAudiobooksRootDir(request, storageUserId, authEnabled), `${bookId}-audiobook`); const chapterPrefix = `${String(chapterIndex + 1).padStart(4, '0')}__`; const files = await readdir(intermediateDir).catch(() => []); for (const file of files) { diff --git a/src/app/api/audiobook/route.ts b/src/app/api/audiobook/route.ts index 5abfafe..ba57a95 100644 --- a/src/app/api/audiobook/route.ts +++ b/src/app/api/audiobook/route.ts @@ -2,17 +2,18 @@ import { NextRequest, NextResponse } from 'next/server'; import { spawn } from 'child_process'; import { readFile, writeFile, mkdir, unlink, rm, rename, readdir } from 'fs/promises'; import { existsSync, createReadStream } from 'fs'; -import { basename, join, resolve } from 'path'; +import { basename, join } from 'path'; import { randomUUID } from 'crypto'; -import { AUDIOBOOKS_V1_DIR, UNCLAIMED_USER_ID, isAudiobooksV1Ready, getUserAudiobookDir } from '@/lib/server/docstore'; +import { AUDIOBOOKS_V1_DIR, ensureAudiobooksV1Ready, isAudiobooksV1Ready, getUserAudiobookDir } from '@/lib/server/docstore'; import { encodeChapterFileName, encodeChapterTitleTag, listStoredChapters, ffprobeAudio, escapeFFMetadata } from '@/lib/server/audiobook'; import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts'; import type { AudiobookGenerationSettings } from '@/types/client'; import { db } from '@/db'; import { audiobooks, audiobookChapters } from '@/db/schema'; -import { eq, and, or } from 'drizzle-orm'; -import { isAuthEnabled } from '@/lib/server/auth-config'; +import { eq, and, inArray } from 'drizzle-orm'; import { requireAuthContext } from '@/lib/server/auth'; +import { ensureDbIndexed } from '@/lib/server/db-indexing'; +import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; export const dynamic = 'force-dynamic'; @@ -20,17 +21,8 @@ export const dynamic = 'force-dynamic'; * Apply test namespace to a directory path if present in request headers. */ function applyTestNamespace(baseDir: string, request: NextRequest): string { - const raw = request.headers.get('x-openreader-test-namespace')?.trim(); - if (!raw) return baseDir; - const safe = raw.replace(/[^a-zA-Z0-9._-]/g, ''); - if (!safe || safe === '.' || safe === '..' || safe.includes('..')) { - return baseDir; - } - const resolved = resolve(baseDir, safe); - if (!resolved.startsWith(resolve(baseDir) + '/')) { - return baseDir; - } - return resolved; + const namespace = getOpenReaderTestNamespace(request.headers); + return applyOpenReaderTestNamespacePath(baseDir, namespace); } /** @@ -183,6 +175,7 @@ export async function POST(request: NextRequest) { const data: ConversionRequest = await request.json(); const requestedFormat = data.format || 'm4b'; + await ensureAudiobooksV1Ready(); if (!(await isAudiobooksV1Ready())) { return NextResponse.json( { error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' }, @@ -193,6 +186,8 @@ export async function POST(request: NextRequest) { const ctxOrRes = await requireAuthContext(request); if (ctxOrRes instanceof Response) return ctxOrRes; const userId = ctxOrRes.userId; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const storageUserId = userId ?? getUnclaimedUserIdForNamespace(testNamespace); // Generate or use existing book ID const bookId = data.bookId || randomUUID(); @@ -202,19 +197,14 @@ export async function POST(request: NextRequest) { } // DB Check / Insert Audiobook - if (isAuthEnabled() && db && userId) { - const [existingBook] = await db.select().from(audiobooks).where( - and(eq(audiobooks.id, bookId), eq(audiobooks.userId, userId)) - ); - - if (!existingBook) { - await db.insert(audiobooks).values({ - id: bookId, - userId, - title: data.chapterTitle || 'Untitled Audiobook', - }); - } - } + await db + .insert(audiobooks) + .values({ + id: bookId, + userId: storageUserId, + title: data.chapterTitle || 'Untitled Audiobook', + }) + .onConflictDoNothing(); const intermediateDir = join(getAudiobooksRootDir(request, userId, ctxOrRes.authEnabled), `${bookId}-audiobook`); @@ -363,21 +353,22 @@ export async function POST(request: NextRequest) { await unlink(inputPath).catch(console.error); // Insert Chapter Record (Denormalized) - if (isAuthEnabled() && db && userId) { - await db.insert(audiobookChapters).values({ + await db + .insert(audiobookChapters) + .values({ id: `${bookId}-${chapterIndex}`, bookId, - userId, + userId: storageUserId, chapterIndex, title: data.chapterTitle, duration, format, - filePath: finalChapterName - }).onConflictDoUpdate({ + filePath: finalChapterName, + }) + .onConflictDoUpdate({ target: [audiobookChapters.id, audiobookChapters.userId], - set: { title: data.chapterTitle, duration, format, filePath: finalChapterName } + set: { title: data.chapterTitle, duration, format, filePath: finalChapterName }, }); - } return NextResponse.json({ index: chapterIndex, @@ -414,6 +405,7 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Invalid bookId parameter' }, { status: 400 }); } + await ensureAudiobooksV1Ready(); if (!(await isAudiobooksV1Ready())) { return NextResponse.json( { error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' }, @@ -424,30 +416,26 @@ export async function GET(request: NextRequest) { const ctxOrRes = await requireAuthContext(request); if (ctxOrRes instanceof Response) return ctxOrRes; const { userId, authEnabled } = ctxOrRes; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = userId ?? unclaimedUserId; + const allowedUserIds = authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + await ensureDbIndexed(); // Check if audiobook exists for user OR is unclaimed (similar to documents) - if (authEnabled && db && userId) { - const [existingBook] = await db.select().from(audiobooks).where( - and( - eq(audiobooks.id, bookId), - or(eq(audiobooks.userId, userId), eq(audiobooks.userId, UNCLAIMED_USER_ID)) - ) - ); - if (!existingBook) { - return NextResponse.json({ error: 'Book not found' }, { status: 404 }); - } + const [existingBook] = await db + .select({ userId: audiobooks.userId }) + .from(audiobooks) + .where(and(eq(audiobooks.id, bookId), inArray(audiobooks.userId, allowedUserIds))); + if (!existingBook) { + return NextResponse.json({ error: 'Book not found' }, { status: 404 }); } - // Get the audiobook directory - check user's directory first, then unclaimed - let intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`); - - // If not found in user's directory and auth is enabled, check unclaimed directory - if (!existsSync(intermediateDir) && authEnabled && userId) { - const unclaimedDir = join(getAudiobooksRootDir(request, UNCLAIMED_USER_ID, authEnabled), `${bookId}-audiobook`); - if (existsSync(unclaimedDir)) { - intermediateDir = unclaimedDir; - } - } + const intermediateDir = join( + getAudiobooksRootDir(request, existingBook.userId, authEnabled), + `${bookId}-audiobook`, + ); if (!existsSync(intermediateDir)) { return NextResponse.json({ error: 'Book not found' }, { status: 404 }); @@ -634,6 +622,7 @@ export async function DELETE(request: NextRequest) { return NextResponse.json({ error: 'Invalid bookId parameter' }, { status: 400 }); } + await ensureAudiobooksV1Ready(); if (!(await isAudiobooksV1Ready())) { return NextResponse.json( { error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' }, @@ -644,27 +633,28 @@ export async function DELETE(request: NextRequest) { const ctxOrRes = await requireAuthContext(request); if (ctxOrRes instanceof Response) return ctxOrRes; const { userId, authEnabled } = ctxOrRes; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const storageUserId = userId ?? getUnclaimedUserIdForNamespace(testNamespace); + + await ensureDbIndexed(); // Delete from DB - with composite PK, we delete by both id and userId - if (authEnabled && db && userId) { - const [existingBook] = await db.select().from(audiobooks).where( - and(eq(audiobooks.id, bookId), eq(audiobooks.userId, userId)) - ); + const [existingBook] = await db + .select({ userId: audiobooks.userId }) + .from(audiobooks) + .where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId))); - if (!existingBook) { - return NextResponse.json({ error: 'Book not found' }, { status: 404 }); - } - - // Delete chapters first (no foreign key constraint with composite PK) - await db.delete(audiobookChapters).where( - and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, userId)) - ); - - await db.delete(audiobooks).where( - and(eq(audiobooks.id, bookId), eq(audiobooks.userId, userId)) - ); + if (!existingBook) { + return NextResponse.json({ error: 'Book not found' }, { status: 404 }); } + // Delete chapters first (no foreign key constraint with composite PK) + await db + .delete(audiobookChapters) + .where(and(eq(audiobookChapters.bookId, bookId), eq(audiobookChapters.userId, storageUserId))); + + await db.delete(audiobooks).where(and(eq(audiobooks.id, bookId), eq(audiobooks.userId, storageUserId))); + const intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`); // If directory doesn't exist, consider it already reset diff --git a/src/app/api/audiobook/status/route.ts b/src/app/api/audiobook/status/route.ts index 3b9f91d..733f6db 100644 --- a/src/app/api/audiobook/status/route.ts +++ b/src/app/api/audiobook/status/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { existsSync } from 'fs'; -import { join, resolve } from 'path'; -import { AUDIOBOOKS_V1_DIR, UNCLAIMED_USER_ID, getUserAudiobookDir, isAudiobooksV1Ready } from '@/lib/server/docstore'; +import { join } from 'path'; +import { AUDIOBOOKS_V1_DIR, getUserAudiobookDir, ensureAudiobooksV1Ready, isAudiobooksV1Ready } from '@/lib/server/docstore'; import { listStoredChapters } from '@/lib/server/audiobook'; import type { AudiobookGenerationSettings } from '@/types/client'; import type { TTSAudiobookFormat, TTSAudiobookChapter } from '@/types/tts'; @@ -9,7 +9,9 @@ import { readFile } from 'fs/promises'; import { requireAuthContext } from '@/lib/server/auth'; import { db } from '@/db'; import { audiobooks } from '@/db/schema'; -import { eq, and, or } from 'drizzle-orm'; +import { eq, and, inArray } from 'drizzle-orm'; +import { ensureDbIndexed } from '@/lib/server/db-indexing'; +import { applyOpenReaderTestNamespacePath, getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace'; export const dynamic = 'force-dynamic'; @@ -19,27 +21,14 @@ export const dynamic = 'force-dynamic'; * When auth is enabled, returns the user-specific directory. */ function getAudiobooksRootDir(request: NextRequest, userId: string | null, authEnabled: boolean): string { - const raw = request.headers.get('x-openreader-test-namespace')?.trim(); - - const applyTestNamespace = (baseDir: string): string => { - if (!raw) return baseDir; - const safe = raw.replace(/[^a-zA-Z0-9._-]/g, ''); - if (!safe || safe === '.' || safe === '..' || safe.includes('..')) { - return baseDir; - } - const resolved = resolve(baseDir, safe); - if (!resolved.startsWith(resolve(baseDir) + '/')) { - return baseDir; - } - return resolved; - }; + const namespace = getOpenReaderTestNamespace(request.headers); if (!authEnabled || !userId) { - return applyTestNamespace(AUDIOBOOKS_V1_DIR); + return applyOpenReaderTestNamespacePath(AUDIOBOOKS_V1_DIR, namespace); } const userDir = getUserAudiobookDir(userId); - return applyTestNamespace(userDir); + return applyOpenReaderTestNamespacePath(userDir, namespace); } const SAFE_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; @@ -55,6 +44,7 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 }); } + await ensureAudiobooksV1Ready(); if (!(await isAudiobooksV1Ready())) { return NextResponse.json( { error: 'Audiobooks storage is not migrated; run /api/migrations/v1 first.' }, @@ -66,37 +56,30 @@ export async function GET(request: NextRequest) { if (ctxOrRes instanceof Response) return ctxOrRes; const { userId, authEnabled } = ctxOrRes; + const testNamespace = getOpenReaderTestNamespace(request.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = userId ?? unclaimedUserId; + const allowedUserIds = authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + await ensureDbIndexed(); // Check if audiobook exists for user OR is unclaimed (similar to documents) - if (authEnabled && db && userId) { - const [existingBook] = await db.select().from(audiobooks).where( - and( - eq(audiobooks.id, bookId), - or(eq(audiobooks.userId, userId), eq(audiobooks.userId, UNCLAIMED_USER_ID)) - ) - ); - if (!existingBook) { - // Book doesn't exist for this user or unclaimed - return empty state - return NextResponse.json({ - chapters: [], - exists: false, - hasComplete: false, - bookId: null, - settings: null, - }); - } + const [existingBook] = await db + .select({ userId: audiobooks.userId }) + .from(audiobooks) + .where(and(eq(audiobooks.id, bookId), inArray(audiobooks.userId, allowedUserIds))); + if (!existingBook) { + // Book doesn't exist for this user or unclaimed - return empty state + return NextResponse.json({ + chapters: [], + exists: false, + hasComplete: false, + bookId: null, + settings: null, + }); } - // Get the audiobook directory - check user's directory first, then unclaimed - let intermediateDir = join(getAudiobooksRootDir(request, userId, authEnabled), `${bookId}-audiobook`); - - // If not found in user's directory and auth is enabled, check unclaimed directory - if (!existsSync(intermediateDir) && authEnabled && userId) { - const unclaimedDir = join(getAudiobooksRootDir(request, UNCLAIMED_USER_ID, authEnabled), `${bookId}-audiobook`); - if (existsSync(unclaimedDir)) { - intermediateDir = unclaimedDir; - } - } + const intermediateDir = join(getAudiobooksRootDir(request, existingBook.userId, authEnabled), `${bookId}-audiobook`); if (!existsSync(intermediateDir)) { return NextResponse.json({ diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 41e9af9..2ee2722 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -2,13 +2,13 @@ import { createHash } from 'crypto'; import { readFile, stat, unlink, utimes, writeFile } from 'fs/promises'; import { NextRequest, NextResponse } from 'next/server'; import path from 'path'; -import { DOCUMENTS_V1_DIR, isDocumentsV1Ready, scanDocumentsFS } from '@/lib/server/docstore'; +import { DOCUMENTS_V1_DIR, UNCLAIMED_USER_ID, ensureDocumentsV1Ready, isDocumentsV1Ready } from '@/lib/server/docstore'; import type { BaseDocument, DocumentType, SyncedDocument } from '@/types/documents'; import { db } from '@/db'; import { documents } from '@/db/schema'; -import { eq, or, and, inArray, count } from 'drizzle-orm'; -import { isAuthEnabled } from '@/lib/server/auth-config'; +import { eq, and, inArray, count } from 'drizzle-orm'; import { requireAuthContext } from '@/lib/server/auth'; +import { ensureDbIndexed } from '@/lib/server/db-indexing'; export const dynamic = 'force-dynamic'; @@ -36,6 +36,7 @@ function toDocumentTypeFromName(name: string): DocumentType { export async function POST(req: NextRequest) { try { + await ensureDocumentsV1Ready(); if (!(await isDocumentsV1Ready())) { return NextResponse.json( { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, @@ -45,6 +46,7 @@ export async function POST(req: NextRequest) { const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + const storageUserId = ctxOrRes.userId ?? UNCLAIMED_USER_ID; const data = await req.json(); const documentsData = data.documents as SyncedDocument[]; @@ -73,28 +75,18 @@ export async function POST(req: NextRequest) { // DB Upsert // With composite PK (id, userId), we check if THIS user already has this document - if (isAuthEnabled() && db) { - const userId = ctxOrRes.userId; - if (!userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - } - - const [existing] = await db.select().from(documents).where( - and(eq(documents.id, id), eq(documents.userId, userId)) - ); - - if (!existing) { - await db.insert(documents).values({ - id, - userId, - name: safeName, - type: doc.type, - size: content.length, - lastModified: doc.lastModified, - filePath: targetFileName - }); - } - } + await db + .insert(documents) + .values({ + id, + userId: storageUserId, + name: safeName, + type: doc.type, + size: content.length, + lastModified: doc.lastModified, + filePath: targetFileName, + }) + .onConflictDoNothing(); stored.push({ oldId: doc.id, id, name: safeName }); } @@ -108,6 +100,7 @@ export async function POST(req: NextRequest) { export async function GET(req: NextRequest) { try { + await ensureDocumentsV1Ready(); if (!(await isDocumentsV1Ready())) { return NextResponse.json( { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, @@ -117,6 +110,10 @@ export async function GET(req: NextRequest) { const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + const storageUserId = ctxOrRes.userId ?? UNCLAIMED_USER_ID; + const allowedUserIds = ctxOrRes.authEnabled ? [storageUserId, UNCLAIMED_USER_ID] : [UNCLAIMED_USER_ID]; + + await ensureDbIndexed(); const url = new URL(req.url); const list = url.searchParams.get('list') === 'true'; @@ -130,30 +127,15 @@ export async function GET(req: NextRequest) { const targetIds = idsParam ? idsParam.split(',').filter(Boolean) : null; - // Query database (or filesystem) for documents the user is allowed to access + // Query database for documents the user is allowed to access let allowedDocs: { id: string; name: string; type: string; size: number; lastModified: number; filePath: string }[] = []; - if (!isAuthEnabled()) { - const fsDocs = await scanDocumentsFS(); - allowedDocs = fsDocs.map(d => ({ ...d })); - if (targetIds) { - allowedDocs = allowedDocs.filter(d => targetIds!.includes(d.id)); - } - } else { - const userId = ctxOrRes.userId; - if (!userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - } - - if (!db) throw new Error("DB not initialized"); - const rows = await db.select().from(documents).where( - and( - or(eq(documents.userId, userId), eq(documents.userId, 'unclaimed')), - targetIds ? inArray(documents.id, targetIds) : undefined - ) - ); - allowedDocs = rows as unknown as { id: string; name: string; type: string; size: number; lastModified: number; filePath: string }[]; - } + const conditions = [ + inArray(documents.userId, allowedUserIds), + ...(targetIds ? [inArray(documents.id, targetIds)] : []), + ]; + const rows = await db.select().from(documents).where(and(...conditions)); + allowedDocs = rows as unknown as { id: string; name: string; type: string; size: number; lastModified: number; filePath: string }[]; const results: (BaseDocument | SyncedDocument)[] = []; @@ -200,6 +182,7 @@ export async function GET(req: NextRequest) { export async function DELETE(req: NextRequest) { try { + await ensureDocumentsV1Ready(); if (!(await isDocumentsV1Ready())) { return NextResponse.json( { error: 'Documents storage is not migrated; run /api/migrations/v1 first.' }, @@ -210,6 +193,9 @@ export async function DELETE(req: NextRequest) { // Auth check - require session const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; + const storageUserId = ctxOrRes.userId ?? UNCLAIMED_USER_ID; + + await ensureDbIndexed(); const url = new URL(req.url); const idsParam = url.searchParams.get('ids'); @@ -220,20 +206,9 @@ export async function DELETE(req: NextRequest) { if (idsParam) { targetIds = idsParam.split(',').filter(Boolean); } else { - if (!isAuthEnabled()) { - const fsDocs = await scanDocumentsFS(); - targetIds = fsDocs.map((d) => d.id); - } else { - const userId = ctxOrRes.userId; - if (!userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - } - - // Existing behavior was "nuke everything"; keep it scoped to "my" docs. - if (!db) throw new Error("DB not initialized"); - const userDocs = await db.select({ id: documents.id }).from(documents).where(eq(documents.userId, userId)); - targetIds = userDocs.map((d: { id: string | null }) => d.id!).filter(Boolean) as string[]; - } + // Existing behavior was "nuke everything"; keep it scoped to "my" docs. + const userDocs = await db.select({ id: documents.id }).from(documents).where(eq(documents.userId, storageUserId)); + targetIds = userDocs.map((d: { id: string | null }) => d.id!).filter(Boolean) as string[]; } if (targetIds.length === 0) { @@ -242,32 +217,12 @@ export async function DELETE(req: NextRequest) { const deletedRows: { id: string; filePath: string }[] = []; - if (!isAuthEnabled()) { - // FS cleanup only - // Since we don't track ownership, we just delete the files requested. - // This implies in no-auth mode, any user can delete any file if they know the ID. - const fsDocs = await scanDocumentsFS(); - for (const doc of fsDocs) { - if (targetIds.includes(doc.id)) { - deletedRows.push({ id: doc.id, filePath: doc.filePath }); - } - } - } else { - const userId = ctxOrRes.userId; - if (!userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - } - - if (!db) throw new Error("DB not initialized"); - const rows = await db.delete(documents) - .where(and( - eq(documents.userId, userId), - inArray(documents.id, targetIds) - )) - .returning({ id: documents.id, filePath: documents.filePath }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - rows.forEach((r: any) => deletedRows.push({ id: r.id!, filePath: r.filePath })); - } + const rows = await db + .delete(documents) + .where(and(eq(documents.userId, storageUserId), inArray(documents.id, targetIds))) + .returning({ id: documents.id, filePath: documents.filePath }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + rows.forEach((r: any) => deletedRows.push({ id: r.id!, filePath: r.filePath })); // If driver doesn't support returning (e.g. older SQLite without properly configured returning), we might fallback. // But Drizzle usually handles this. @@ -279,10 +234,8 @@ export async function DELETE(req: NextRequest) { // Chech reference count for this ID // If 0 remaining, delete file let refCount = 0; - if (isAuthEnabled() && db) { - const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, row.id!)); - refCount = ref?.count ?? 0; - } + const [ref] = await db.select({ count: count() }).from(documents).where(eq(documents.id, row.id!)); + refCount = Number(ref?.count ?? 0); if (refCount === 0) { const filePath = path.join(SYNC_DIR, row.filePath); diff --git a/src/app/api/user/claim/route.ts b/src/app/api/user/claim/route.ts index 3be0c30..1011282 100644 --- a/src/app/api/user/claim/route.ts +++ b/src/app/api/user/claim/route.ts @@ -1,6 +1,23 @@ import { NextRequest, NextResponse } from 'next/server'; -import { claimAnonymousData, scanAndPopulateDB } from '@/lib/server/claim-data'; +import { claimAnonymousData } from '@/lib/server/claim-data'; import { auth } from '@/lib/server/auth'; +import { ensureDbIndexed, getUnclaimedCounts } from '@/lib/server/db-indexing'; + +export async function GET(req: NextRequest) { + try { + const session = await auth?.api.getSession({ headers: req.headers }); + if (!session || !session.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + await ensureDbIndexed(); + const counts = await getUnclaimedCounts(); + return NextResponse.json({ success: true, ...counts }); + } catch (error) { + console.error('Error checking claimable data:', error); + return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 }); + } +} export async function POST(req: NextRequest) { try { @@ -10,14 +27,8 @@ export async function POST(req: NextRequest) { } const userId = session.user.id; - const { action } = await req.json(); // "claim" or "scan" or "start_fresh" (optional logic) - if (action === 'scan') { - const counts = await scanAndPopulateDB(); - return NextResponse.json({ success: true, message: 'Scanned file system', ...counts }); - } - - // Default action: Claim + await ensureDbIndexed(); const result = await claimAnonymousData(userId); return NextResponse.json({ diff --git a/src/components/auth/ClaimDataModal.tsx b/src/components/auth/ClaimDataModal.tsx index 0539ca6..84fa66e 100644 --- a/src/components/auth/ClaimDataModal.tsx +++ b/src/components/auth/ClaimDataModal.tsx @@ -25,9 +25,7 @@ export default function ClaimDataModal() { try { const res = await fetch('/api/user/claim', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ action: 'scan' }) + method: 'GET', }); if (res.ok) { const data = await res.json(); @@ -52,8 +50,6 @@ export default function ClaimDataModal() { try { const res = await fetch('/api/user/claim', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ action: 'claim' }) }); if (res.ok) { const data = await res.json(); diff --git a/src/db/index.ts b/src/db/index.ts index 1ca9d1d..d405189 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -5,7 +5,6 @@ import Database from 'better-sqlite3'; import path from 'path'; import fs from 'fs'; import * as schema from './schema'; -import { isAuthEnabled } from '@/lib/server/auth-config'; // Singleton logic not strictly needed if Next.js handles module caching, but good for safety // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -14,11 +13,6 @@ let dbInstance: any = null; export function getDrizzleDB() { if (dbInstance) return dbInstance; - // If auth is NOT enabled, we do not want to connect to any database - if (!isAuthEnabled()) { - return null; - } - if (process.env.POSTGRES_URL) { const pool = new Pool({ connectionString: process.env.POSTGRES_URL, diff --git a/src/lib/server/claim-data.ts b/src/lib/server/claim-data.ts index 2620c48..6a8782d 100644 --- a/src/lib/server/claim-data.ts +++ b/src/lib/server/claim-data.ts @@ -1,191 +1,18 @@ import { db } from '@/db'; import { documents, audiobooks, audiobookChapters } from '@/db/schema'; -import { eq, and, count } from 'drizzle-orm'; +import { eq, and } from 'drizzle-orm'; import fs from 'fs/promises'; -import { existsSync } from 'fs'; -import path from 'path'; -import { - DOCUMENTS_V1_DIR, - AUDIOBOOKS_V1_DIR, +import { UNCLAIMED_USER_ID, - getUnclaimedAudiobookDir, getUserAudiobookDir, moveAudiobookToUser, - listUserAudiobookIds + listUserAudiobookIds, } from './docstore'; -import { listStoredChapters } from './audiobook'; import { isAuthEnabled } from '@/lib/server/auth-config'; -// Helper to check if a document is already indexed -async function isDocumentIndexed(id: string) { - if (!isAuthEnabled() || !db) return false; - const result = await db.select({ id: documents.id }).from(documents).where(eq(documents.id, id)); - return result.length > 0; -} - -// Helper to check if an audiobook is indexed for a specific user (composite PK) -async function isAudiobookIndexedForUser(id: string, userId: string) { - if (!isAuthEnabled() || !db) return false; - const result = await db.select({ id: audiobooks.id }).from(audiobooks).where( - and(eq(audiobooks.id, id), eq(audiobooks.userId, userId)) - ); - return result.length > 0; -} - -/** - * Returns count of unclaimed documents and audiobooks in the DB (userId IS 'unclaimed') - */ -export async function getUnclaimedCounts() { - if (!isAuthEnabled() || !db) return { documents: 0, audiobooks: 0 }; - const [docCount] = await db.select({ count: count() }).from(documents).where(eq(documents.userId, UNCLAIMED_USER_ID)); - const [bookCount] = await db.select({ count: count() }).from(audiobooks).where(eq(audiobooks.userId, UNCLAIMED_USER_ID)); - - return { - documents: docCount?.count ?? 0, - audiobooks: bookCount?.count ?? 0 - }; -} - -/** - * Migrate legacy audiobooks from AUDIOBOOKS_V1_DIR to the unclaimed user folder. - * This handles the case where audiobooks were created before the per-user storage refactor. - */ -async function migrateLegacyAudiobooksToUnclaimed(): Promise { - if (!existsSync(AUDIOBOOKS_V1_DIR)) return 0; - - const unclaimedDir = getUnclaimedAudiobookDir(); - await fs.mkdir(unclaimedDir, { recursive: true }); - - const entries = await fs.readdir(AUDIOBOOKS_V1_DIR, { withFileTypes: true }); - let migrated = 0; - - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if (!entry.name.endsWith('-audiobook')) continue; - - const sourceDir = path.join(AUDIOBOOKS_V1_DIR, entry.name); - const targetDir = path.join(unclaimedDir, entry.name); - - // Skip if already exists in unclaimed - if (existsSync(targetDir)) continue; - - try { - await fs.rename(sourceDir, targetDir); - migrated++; - console.log(`Migrated legacy audiobook to unclaimed: ${entry.name}`); - } catch (err) { - console.error(`Error migrating legacy audiobook ${entry.name}:`, err); - } - } - - return migrated; -} - -export async function scanAndPopulateDB() { - if (!isAuthEnabled() || !db) { - console.log('Skipping DB population (Auth/DB disabled)'); - return { documents: 0, audiobooks: 0 }; - } - - console.log('Scanning file system for un-indexed content...'); - - // 0. Migrate legacy audiobooks from AUDIOBOOKS_V1_DIR to unclaimed folder - await migrateLegacyAudiobooksToUnclaimed(); - - // 1. Scan Documents (unchanged - documents use shared storage) - if (existsSync(DOCUMENTS_V1_DIR)) { - const files = await fs.readdir(DOCUMENTS_V1_DIR); - for (const file of files) { - const match = /^([a-f0-9]{64})__(.+)$/i.exec(file); - if (!match) continue; - - const id = match[1]; - const encodedName = match[2]; - if (await isDocumentIndexed(id)) continue; - - let name: string; - try { - name = decodeURIComponent(encodedName); - } catch { - continue; - } - - const filePath = path.join(DOCUMENTS_V1_DIR, file); - const stats = await fs.stat(filePath); - - const ext = path.extname(name).toLowerCase().replace('.', ''); - - await db.insert(documents).values({ - id, - userId: UNCLAIMED_USER_ID, - name, - type: ext, - size: stats.size, - lastModified: Math.floor(stats.mtimeMs), - filePath: file, - }); - console.log(`Indexed document: ${name} (${id})`); - } - } - - // 2. Scan Audiobooks from unclaimed folder - const unclaimedDir = getUnclaimedAudiobookDir(); - if (existsSync(unclaimedDir)) { - const entries = await fs.readdir(unclaimedDir, { withFileTypes: true }); - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if (!entry.name.endsWith('-audiobook')) continue; - - const bookId = entry.name.replace('-audiobook', ''); - if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue; - - const dirPath = path.join(unclaimedDir, entry.name); - - let title = `Unknown Title`; - - try { - const metaPath = path.join(dirPath, 'audiobook.meta.json'); - const metaContent = await fs.readFile(metaPath, 'utf8'); - JSON.parse(metaContent); // validating json - } catch { } - - const chapters = await listStoredChapters(dirPath); - const totalDuration = chapters.reduce((acc, c) => acc + (c.durationSec || 0), 0); - - if (chapters.length > 0) { - title = chapters[0].title || title; - } - - await db.insert(audiobooks).values({ - id: bookId, - userId: UNCLAIMED_USER_ID, - title: title, - duration: totalDuration, - }); - console.log(`Indexed audiobook: ${bookId}`); - - for (const chapter of chapters) { - await db.insert(audiobookChapters).values({ - id: `${bookId}-${chapter.index}`, - bookId: bookId, - userId: UNCLAIMED_USER_ID, - chapterIndex: chapter.index, - title: chapter.title, - duration: chapter.durationSec || 0, - filePath: chapter.filePath, - format: chapter.format - }); - } - } - } - - // Return current unclaimed counts (includes newly indexed + previously unclaimed) - return getUnclaimedCounts(); -} - export async function claimAnonymousData(userId: string) { - if (!isAuthEnabled() || !db || !userId) return { documents: 0, audiobooks: 0 }; + if (!isAuthEnabled() || !userId) return { documents: 0, audiobooks: 0 }; // Get list of unclaimed audiobook IDs before updating DB const unclaimedBookIds = await listUserAudiobookIds(UNCLAIMED_USER_ID); @@ -262,7 +89,7 @@ export async function claimAnonymousData(userId: string) { * @returns number of audiobooks transferred */ export async function transferUserAudiobooks(fromUserId: string, toUserId: string): Promise { - if (!isAuthEnabled() || !db || !fromUserId || !toUserId) return 0; + if (!isAuthEnabled() || !fromUserId || !toUserId) return 0; const bookIds = await listUserAudiobookIds(fromUserId); let transferred = 0; diff --git a/src/lib/server/db-indexing.ts b/src/lib/server/db-indexing.ts new file mode 100644 index 0000000..9f1c479 --- /dev/null +++ b/src/lib/server/db-indexing.ts @@ -0,0 +1,254 @@ +import fs from 'fs/promises'; +import { existsSync } from 'fs'; +import path from 'path'; +import { isAuthEnabled } from '@/lib/server/auth-config'; +import { db } from '@/db'; +import { audiobookChapters, audiobooks, documents } from '@/db/schema'; +import { and, count, eq } from 'drizzle-orm'; +import { listStoredChapters } from '@/lib/server/audiobook'; +import { AUDIOBOOKS_V1_DIR, DOCUMENTS_V1_DIR, UNCLAIMED_USER_ID, getUnclaimedAudiobookDir } from '@/lib/server/docstore'; + +const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); +const MIGRATIONS_DIR = path.join(DOCSTORE_DIR, '.migrations'); +const STATE_PATH = path.join(MIGRATIONS_DIR, 'db-index.json'); + +type DbIndexState = { + indexedAt: number; + mode: 'auth' | 'noauth'; +}; + +let inflight: Promise | null = null; +let memoryIndexedMode: DbIndexState['mode'] | null = null; + +async function readState(): Promise { + try { + const raw = await fs.readFile(STATE_PATH, 'utf8'); + return JSON.parse(raw) as DbIndexState; + } catch { + return null; + } +} + +async function writeState(): Promise { + await fs.mkdir(MIGRATIONS_DIR, { recursive: true }); + const state: DbIndexState = { + indexedAt: Date.now(), + mode: isAuthEnabled() ? 'auth' : 'noauth', + }; + await fs.writeFile(STATE_PATH, JSON.stringify(state, null, 2)); +} + +async function hasUnclaimedDbRows(): Promise { + const [docCount] = await db + .select({ count: count() }) + .from(documents) + .where(eq(documents.userId, UNCLAIMED_USER_ID)); + const [bookCount] = await db + .select({ count: count() }) + .from(audiobooks) + .where(eq(audiobooks.userId, UNCLAIMED_USER_ID)); + + return Number(docCount?.count ?? 0) > 0 || Number(bookCount?.count ?? 0) > 0; +} + +async function hasFilesystemContent(mode: DbIndexState['mode']): Promise { + // Documents are always stored under documents_v1. + try { + const entries = await fs.readdir(DOCUMENTS_V1_DIR); + if (entries.some((name) => /^[a-f0-9]{64}__.+$/i.test(name))) return true; + } catch { + // ignore + } + + // Audiobooks differ by auth-mode layout. + const audiobookDir = mode === 'auth' ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR; + try { + const entries = await fs.readdir(audiobookDir, { withFileTypes: true }); + if (entries.some((e) => e.isDirectory() && e.name.endsWith('-audiobook'))) return true; + } catch { + // ignore + } + + return false; +} + +async function isDocumentIndexed(id: string): Promise { + const result = await db.select({ id: documents.id }).from(documents).where(eq(documents.id, id)); + return result.length > 0; +} + +async function isAudiobookIndexedForUser(id: string, userId: string): Promise { + const result = await db + .select({ id: audiobooks.id }) + .from(audiobooks) + .where(and(eq(audiobooks.id, id), eq(audiobooks.userId, userId))); + return result.length > 0; +} + +async function migrateLegacyAudiobooksToUnclaimed(): Promise { + if (!existsSync(AUDIOBOOKS_V1_DIR)) return 0; + + const unclaimedDir = getUnclaimedAudiobookDir(); + await fs.mkdir(unclaimedDir, { recursive: true }); + + const entries = await fs.readdir(AUDIOBOOKS_V1_DIR, { withFileTypes: true }); + let migrated = 0; + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (!entry.name.endsWith('-audiobook')) continue; + + const sourceDir = path.join(AUDIOBOOKS_V1_DIR, entry.name); + const targetDir = path.join(unclaimedDir, entry.name); + + if (existsSync(targetDir)) continue; + + try { + await fs.rename(sourceDir, targetDir); + migrated++; + console.log(`Migrated legacy audiobook to unclaimed: ${entry.name}`); + } catch (err) { + console.error(`Error migrating legacy audiobook ${entry.name}:`, err); + } + } + + return migrated; +} + +export async function getUnclaimedCounts(): Promise<{ documents: number; audiobooks: number }> { + const [docCount] = await db.select({ count: count() }).from(documents).where(eq(documents.userId, UNCLAIMED_USER_ID)); + const [bookCount] = await db.select({ count: count() }).from(audiobooks).where(eq(audiobooks.userId, UNCLAIMED_USER_ID)); + + return { + documents: Number(docCount?.count ?? 0), + audiobooks: Number(bookCount?.count ?? 0), + }; +} + +async function scanAndPopulateDb(): Promise<{ documents: number; audiobooks: number }> { + const authEnabled = isAuthEnabled(); + + console.log('Scanning file system for un-indexed content...'); + + if (authEnabled) { + await migrateLegacyAudiobooksToUnclaimed(); + } + + if (existsSync(DOCUMENTS_V1_DIR)) { + const files = await fs.readdir(DOCUMENTS_V1_DIR); + for (const file of files) { + const match = /^([a-f0-9]{64})__(.+)$/i.exec(file); + if (!match) continue; + + const id = match[1]; + const encodedName = match[2]; + if (await isDocumentIndexed(id)) continue; + + let name: string; + try { + name = decodeURIComponent(encodedName); + } catch { + continue; + } + + const filePath = path.join(DOCUMENTS_V1_DIR, file); + const stats = await fs.stat(filePath); + const ext = path.extname(name).toLowerCase().replace('.', ''); + + await db.insert(documents).values({ + id, + userId: UNCLAIMED_USER_ID, + name, + type: ext, + size: stats.size, + lastModified: Math.floor(stats.mtimeMs), + filePath: file, + }); + console.log(`Indexed document: ${name} (${id})`); + } + } + + const audiobookScanDir = authEnabled ? getUnclaimedAudiobookDir() : AUDIOBOOKS_V1_DIR; + if (existsSync(audiobookScanDir)) { + const entries = await fs.readdir(audiobookScanDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (!entry.name.endsWith('-audiobook')) continue; + + const bookId = entry.name.replace('-audiobook', ''); + if (await isAudiobookIndexedForUser(bookId, UNCLAIMED_USER_ID)) continue; + + const dirPath = path.join(audiobookScanDir, entry.name); + + let title = 'Unknown Title'; + try { + const metaPath = path.join(dirPath, 'audiobook.meta.json'); + const metaContent = await fs.readFile(metaPath, 'utf8'); + JSON.parse(metaContent); + } catch { + // ignore + } + + const chapters = await listStoredChapters(dirPath); + const totalDuration = chapters.reduce((acc, c) => acc + (c.durationSec || 0), 0); + if (chapters.length > 0) title = chapters[0].title || title; + + await db.insert(audiobooks).values({ + id: bookId, + userId: UNCLAIMED_USER_ID, + title, + duration: totalDuration, + }); + console.log(`Indexed audiobook: ${bookId}`); + + for (const chapter of chapters) { + await db.insert(audiobookChapters).values({ + id: `${bookId}-${chapter.index}`, + bookId, + userId: UNCLAIMED_USER_ID, + chapterIndex: chapter.index, + title: chapter.title, + duration: chapter.durationSec || 0, + filePath: chapter.filePath, + format: chapter.format, + }); + } + } + } + + return getUnclaimedCounts(); +} + +/** + * Ensure DB has rows for existing filesystem content (documents/audiobooks). + * + * This is intentionally safe to call from routes: + * - Runs at most once per process (memoryIndexed) + * - Uses a persisted state file under docstore/.migrations to avoid rescanning every boot + */ +export async function ensureDbIndexed(): Promise { + const mode: DbIndexState['mode'] = isAuthEnabled() ? 'auth' : 'noauth'; + if (memoryIndexedMode === mode) return; + + inflight ??= (async () => { + const hasState = existsSync(STATE_PATH) ? await readState() : null; + if (hasState && hasState.mode === mode) { + // If the DB was reset but the state file survived, don't get stuck "indexed" forever. + // Only skip if the DB has rows OR there is no content on disk to index. + const [dbHasRows, fsHasRows] = await Promise.all([hasUnclaimedDbRows(), hasFilesystemContent(mode)]); + if (dbHasRows || !fsHasRows) { + memoryIndexedMode = mode; + return; + } + } + + await fs.mkdir(DOCSTORE_DIR, { recursive: true }); + await scanAndPopulateDb(); + await writeState(); + memoryIndexedMode = mode; + })().finally(() => { + inflight = null; + }); + + await inflight; +} diff --git a/src/lib/server/docstore.ts b/src/lib/server/docstore.ts index 07ebc4b..8ade6a3 100644 --- a/src/lib/server/docstore.ts +++ b/src/lib/server/docstore.ts @@ -8,13 +8,13 @@ import { decodeChapterTitleTag, encodeChapterFileName, encodeChapterTitleTag, ff export const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); export const DOCUMENTS_V1_DIR = path.join(DOCSTORE_DIR, 'documents_v1'); export const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1'); -export const AUDIOBOOKS_USERS_DIR = path.join(DOCSTORE_DIR, 'audiobooks_users'); +export const AUDIOBOOKS_USERS_DIR = path.join(DOCSTORE_DIR, 'audiobooks_users_v1'); export const UNCLAIMED_USER_ID = 'unclaimed'; /** * Get the audiobook directory for a specific user when auth is enabled. - * Returns path like: docstore/audiobooks_users/{userId} + * Returns path like: docstore/audiobooks_users_v1/{userId} */ export function getUserAudiobookDir(userId: string): string { // Sanitize userId to prevent path traversal @@ -27,7 +27,7 @@ export function getUserAudiobookDir(userId: string): string { /** * Get the unclaimed audiobooks directory for pre-auth content. - * Returns path like: docstore/audiobooks_users/unclaimed + * Returns path like: docstore/audiobooks_users_v1/unclaimed */ export function getUnclaimedAudiobookDir(): string { return path.join(AUDIOBOOKS_USERS_DIR, UNCLAIMED_USER_ID); @@ -36,7 +36,7 @@ export function getUnclaimedAudiobookDir(): string { /** * Get the full path to a specific audiobook directory. * - When auth is disabled: docstore/audiobooks_v1/{bookId}-audiobook - * - When auth is enabled: docstore/audiobooks_users/{userId}/{bookId}-audiobook + * - When auth is enabled: docstore/audiobooks_users_v1/{userId}/{bookId}-audiobook */ export function getAudiobookPath(bookId: string, userId: string | null, authEnabled: boolean): string { if (!authEnabled || !userId) { @@ -219,50 +219,6 @@ export type FSDocument = { filePath: string; }; -export async function scanDocumentsFS(): Promise { - if (!existsSync(DOCUMENTS_V1_DIR)) return []; - - const results: FSDocument[] = []; - let files: string[] = []; - try { - files = await readdir(DOCUMENTS_V1_DIR); - } catch { - return []; - } - - for (const file of files) { - // Expected format: id__filename or id.ext (legacy fallback?) - // Actually current format is id__encodedName - const match = /^([a-f0-9]{64})__(.+)$/i.exec(file); - if (!match) continue; - - const id = match[1]; - const encodedName = match[2]; - const name = decodeURIComponent(encodedName); - const ext = path.extname(name).toLowerCase().replace('.', ''); - - // Validate file exists and get stats - try { - const filePath = path.join(DOCUMENTS_V1_DIR, file); - const stats = await stat(filePath); - if (!stats.isFile()) continue; - - results.push({ - id, - name, - type: ext, - size: stats.size, - lastModified: Math.floor(stats.mtimeMs), - filePath: file, - }); - } catch { - continue; - } - } - - return results; -} - export async function ensureDocumentsV1Ready(): Promise { await mkdir(DOCSTORE_DIR, { recursive: true }); await mkdir(DOCUMENTS_V1_DIR, { recursive: true }); diff --git a/src/lib/server/migration-manager.ts b/src/lib/server/migration-manager.ts deleted file mode 100644 index 4810833..0000000 --- a/src/lib/server/migration-manager.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { db } from '@/db'; -import { documents, audiobooks, audiobookChapters } from '@/db/schema'; -import { eq, isNull, count } from 'drizzle-orm'; -import fs from 'fs/promises'; -import { existsSync } from 'fs'; -import path from 'path'; -import { DOCUMENTS_V1_DIR, AUDIOBOOKS_V1_DIR } from './docstore'; -import { listStoredChapters } from './audiobook'; - -import { isAuthEnabled } from '@/lib/server/auth-config'; - -// Helper to check if a file is already indexed -async function isDocumentIndexed(id: string) { - if (!isAuthEnabled() || !db) return false; // If no DB, assume not indexed or handle differently? - // Actually if no DB, we don't index into DB. So returning false is fine, - // but scanAndPopulateDB should probably return early if no DB. - - const result = await db.select({ id: documents.id }).from(documents).where(eq(documents.id, id)); - return result.length > 0; -} - -async function isAudiobookIndexed(id: string) { - if (!isAuthEnabled() || !db) return false; - const result = await db.select({ id: audiobooks.id }).from(audiobooks).where(eq(audiobooks.id, id)); - return result.length > 0; -} - -const UNCLAIMED_ID = 'unclaimed'; - -/** - * Returns count of unclaimed documents and audiobooks in the DB (userId IS 'unclaimed') - */ -export async function getUnclaimedCounts() { - if (!isAuthEnabled() || !db) return { documents: 0, audiobooks: 0 }; - const [docCount] = await db.select({ count: count() }).from(documents).where(eq(documents.userId, UNCLAIMED_ID)); - const [bookCount] = await db.select({ count: count() }).from(audiobooks).where(eq(audiobooks.userId, UNCLAIMED_ID)); - - return { - documents: docCount?.count ?? 0, - audiobooks: bookCount?.count ?? 0 - }; -} - -export async function scanAndPopulateDB() { - if (!isAuthEnabled() || !db) { - console.log('Skipping DB population (Auth/DB disabled)'); - return { documents: 0, audiobooks: 0 }; - } - - console.log('Scanning file system for un-indexed content...'); - - // 0. Fix legacy NULL userIds - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await (db as any).update(documents).set({ userId: UNCLAIMED_ID }).where(isNull(documents.userId)); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await (db as any).update(audiobooks).set({ userId: UNCLAIMED_ID }).where(isNull(audiobooks.userId)); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await (db as any).update(audiobookChapters).set({ userId: UNCLAIMED_ID }).where(isNull(audiobookChapters.userId)); - } catch (err) { - console.error('Error migrating legacy NULL userIds:', err); - } - - // 1. Scan Documents - if (existsSync(DOCUMENTS_V1_DIR)) { - const files = await fs.readdir(DOCUMENTS_V1_DIR); - for (const file of files) { - const match = /^([a-f0-9]{64})__(.+)$/i.exec(file); - if (!match) continue; - - const id = match[1]; - const encodedName = match[2]; - if (await isDocumentIndexed(id)) continue; - - let name: string; - try { - name = decodeURIComponent(encodedName); - } catch { - continue; - } - - const filePath = path.join(DOCUMENTS_V1_DIR, file); - const stats = await fs.stat(filePath); - - const ext = path.extname(name).toLowerCase().replace('.', ''); - - await db.insert(documents).values({ - id, - userId: UNCLAIMED_ID, - name, - type: ext, - size: stats.size, - lastModified: Math.floor(stats.mtimeMs), - filePath: file, - }); - console.log(`Indexed document: ${name} (${id})`); - } - } - - // 2. Scan Audiobooks - if (existsSync(AUDIOBOOKS_V1_DIR)) { - const entries = await fs.readdir(AUDIOBOOKS_V1_DIR, { withFileTypes: true }); - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if (!entry.name.endsWith('-audiobook')) continue; - - const bookId = entry.name.replace('-audiobook', ''); - if (await isAudiobookIndexed(bookId)) continue; - - const dirPath = path.join(AUDIOBOOKS_V1_DIR, entry.name); - - let title = `Unknown Title`; - - try { - const metaPath = path.join(dirPath, 'audiobook.meta.json'); - const metaContent = await fs.readFile(metaPath, 'utf8'); - JSON.parse(metaContent); // validating json - } catch { } - - const chapters = await listStoredChapters(dirPath); - const totalDuration = chapters.reduce((acc, c) => acc + (c.durationSec || 0), 0); - - if (chapters.length > 0) { - title = chapters[0].title || title; - } - - await db.insert(audiobooks).values({ - id: bookId, - userId: UNCLAIMED_ID, - title: title, - duration: totalDuration, - }); - console.log(`Indexed audiobook: ${bookId}`); - - for (const chapter of chapters) { - await db.insert(audiobookChapters).values({ - id: `${bookId}-${chapter.index}`, - bookId: bookId, - userId: UNCLAIMED_ID, - chapterIndex: chapter.index, - title: chapter.title, - duration: chapter.durationSec || 0, - filePath: chapter.filePath, - format: chapter.format - }) - } - } - } - - // Return current unclaimed counts (includes newly indexed + previously unclaimed) - return getUnclaimedCounts(); -} - -export async function claimAnonymousData(userId: string) { - if (!isAuthEnabled() || !db || !userId) return { documents: 0, audiobooks: 0 }; - - // Update Documents - const docResult = await db.update(documents) - .set({ userId }) - .where(eq(documents.userId, UNCLAIMED_ID)) - .returning({ id: documents.id }); // If supported by driver, otherwise use run and check changes - - // Update Audiobooks - const bookResult = await db.update(audiobooks) - .set({ userId }) - .where(eq(audiobooks.userId, UNCLAIMED_ID)) - .returning({ id: audiobooks.id }); - - // Update Chapters (denormalized userId) - await db.update(audiobookChapters) - .set({ userId }) - .where(eq(audiobookChapters.userId, UNCLAIMED_ID)); // Or match by bookId join - - return { - documents: docResult.length, - audiobooks: bookResult.length - }; -} diff --git a/src/lib/server/rate-limiter.ts b/src/lib/server/rate-limiter.ts index 62c7999..2cf097a 100644 --- a/src/lib/server/rate-limiter.ts +++ b/src/lib/server/rate-limiter.ts @@ -146,8 +146,6 @@ export class RateLimiter { } try { - if (!db) throw new Error("DB not initialized"); - const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue; // Use a DB transaction to avoid partial increments across buckets and to avoid @@ -254,13 +252,6 @@ export class RateLimiter { } const bucketResults: Array<{ currentCount: number; limit: number }> = []; - if (!db) { - const effective = pickEffectiveResult([]); - return { - ...effective, - resetTime: this.getResetTime(), - }; - } for (const bucket of buckets) { const result = await safeDb().select({ charCount: userTtsChars.charCount }) @@ -289,8 +280,6 @@ export class RateLimiter { async transferAnonymousUsage(anonymousUserId: string, authenticatedUserId: string): Promise { if (!isAuthEnabled()) return; - if (!db) return; - const today = new Date().toISOString().split('T')[0]; const dateValue = today as unknown as UserTtsCharsDateValue; const updatedAt = this.getUpdatedAtValue() as unknown as UserTtsCharsUpdatedAtValue; @@ -332,7 +321,6 @@ export class RateLimiter { const cutoffDateStr = cutoffDate.toISOString().split('T')[0]; const cutoffDateValue = cutoffDateStr as unknown as UserTtsCharsDateValue; - if (!db) return; // Assuming string comparison works for YYYY-MM-DD await safeDb().delete(userTtsChars).where(lt(userTtsChars.date, cutoffDateValue)); } diff --git a/src/lib/server/test-namespace.ts b/src/lib/server/test-namespace.ts new file mode 100644 index 0000000..5fe4671 --- /dev/null +++ b/src/lib/server/test-namespace.ts @@ -0,0 +1,30 @@ +import path from 'path'; +import { UNCLAIMED_USER_ID } from '@/lib/server/docstore'; + +const TEST_NAMESPACE_HEADER = 'x-openreader-test-namespace'; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; + +export function getOpenReaderTestNamespace(headers: Headers): string | null { + const raw = headers.get(TEST_NAMESPACE_HEADER)?.trim(); + if (!raw) return null; + + const safe = raw.replace(/[^a-zA-Z0-9._-]/g, ''); + if (!safe || safe === '.' || safe === '..' || safe.includes('..')) return null; + if (!SAFE_NAMESPACE_REGEX.test(safe)) return null; + + return safe; +} + +export function applyOpenReaderTestNamespacePath(baseDir: string, namespace: string | null): string { + if (!namespace) return baseDir; + + const resolved = path.resolve(baseDir, namespace); + if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir; + return resolved; +} + +export function getUnclaimedUserIdForNamespace(namespace: string | null): string { + if (!namespace) return UNCLAIMED_USER_ID; + return `${UNCLAIMED_USER_ID}::${namespace}`; +} + diff --git a/tests/export.spec.ts b/tests/export.spec.ts index 1e297ec..c767c40 100644 --- a/tests/export.spec.ts +++ b/tests/export.spec.ts @@ -1,5 +1,7 @@ import { test, expect, Page } from '@playwright/test'; import fs from 'fs'; +import os from 'os'; +import path from 'path'; import util from 'util'; import { execFile } from 'child_process'; import { setupTest, uploadAndDisplay } from './helpers'; @@ -39,19 +41,52 @@ async function waitForChaptersHeading(page: Page) { await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 }); } -async function downloadFullAudiobook(page: Page, timeoutMs = 60_000) { +type DownloadedAudiobook = { + filePath: string; + suggestedFilename: string; + cleanup: () => Promise; +}; + +async function downloadFullAudiobook(page: Page, timeoutMs = 60_000): Promise { const fullDownloadButton = page.getByRole('button', { name: /Full Download/i }); await expect(fullDownloadButton).toBeVisible({ timeout: timeoutMs }); const [download] = await Promise.all([ page.waitForEvent('download', { timeout: timeoutMs }), fullDownloadButton.click(), ]); - const tempFilePath = `./tmp_download_${Date.now()}.mp3`; - await download.saveAs(tempFilePath); - expect(fs.existsSync(tempFilePath)).toBeTruthy(); - const stats = fs.statSync(tempFilePath); + const failure = await download.failure(); + expect(failure).toBeNull(); + + const suggestedFilename = download.suggestedFilename(); + let createdTempDir: string | null = null; + let filePath = await download.path(); + + // Some environments/browsers may not expose a stable download path; fall back to saving + // into a temp directory outside the repo (and clean up after assertions). + if (!filePath) { + createdTempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'openreader-audiobook-')); + const name = suggestedFilename || `download_${Date.now()}.mp3`; + filePath = path.join(createdTempDir, name); + await download.saveAs(filePath); + } + + expect(fs.existsSync(filePath)).toBeTruthy(); + const stats = fs.statSync(filePath); expect(stats.size).toBeGreaterThan(0); - return tempFilePath; + return { + filePath, + suggestedFilename, + cleanup: async () => { + try { + await fs.promises.unlink(filePath); + } catch { + // ignore + } + if (createdTempDir) { + await fs.promises.rm(createdTempDir, { recursive: true, force: true }); + } + }, + }; } async function getAudioDurationSeconds(filePath: string) { @@ -74,6 +109,19 @@ async function expectChaptersBackendState(page: Page, bookId: string) { return json; } +async function withDownloadedFullAudiobook( + page: Page, + fn: (args: { filePath: string; suggestedFilename: string }) => Promise, + timeoutMs = 60_000 +): Promise { + const dl = await downloadFullAudiobook(page, timeoutMs); + try { + return await fn({ filePath: dl.filePath, suggestedFilename: dl.suggestedFilename }); + } finally { + await dl.cleanup(); + } +} + /** * Poll the backend until the chapter count stops changing for `stableMs` milliseconds. * This helps avoid race conditions where in-flight TTS requests complete after cancellation. @@ -160,16 +208,16 @@ test('exports full MP3 audiobook for PDF using mocked 10s TTS sample', async ({ // Trigger full download from the FRONTEND button and capture via Playwright's download API. // The button label can be "Full Download (MP3)" or "Full Download (M4B)" depending on // the server-side detected format, so match more loosely on the accessible name. - const downloadedPath = await downloadFullAudiobook(page); - - // Use ffprobe (same toolchain as the server) to validate the combined audio duration. - // The TTS route is mocked to return a 10s sample.mp3 for each page, so with at least - // two chapters we should be close to ~20 seconds of audio. - const durationSeconds = await getAudioDurationSeconds(downloadedPath); - // Duration must be within a reasonable window around 20 seconds to allow - // for encoding variations and container overhead. - expect(durationSeconds).toBeGreaterThan(18); - expect(durationSeconds).toBeLessThan(22); + await withDownloadedFullAudiobook(page, async ({ filePath }) => { + // Use ffprobe (same toolchain as the server) to validate the combined audio duration. + // The TTS route is mocked to return a 10s sample.mp3 for each page, so with at least + // two chapters we should be close to ~20 seconds of audio. + const durationSeconds = await getAudioDurationSeconds(filePath); + // Duration must be within a reasonable window around 20 seconds to allow + // for encoding variations and container overhead. + expect(durationSeconds).toBeGreaterThan(18); + expect(durationSeconds).toBeLessThan(22); + }); // Also check the chapter metadata API for consistency const json = await expectChaptersBackendState(page, bookId); @@ -238,11 +286,11 @@ test('exports partial MP3 audiobook for EPUB using mocked 10s TTS sample', async await expect(chapterActionsButtons).toHaveCount(chapterCountAfterCancel, { timeout: 60_000 }); // The Full Download button should still be available for the partially generated audiobook - const downloadedPath = await downloadFullAudiobook(page); - - const durationSeconds = await getAudioDurationSeconds(downloadedPath); - expect(durationSeconds).toBeGreaterThan(25); - expect(durationSeconds).toBeLessThan(300); + await withDownloadedFullAudiobook(page, async ({ filePath }) => { + const durationSeconds = await getAudioDurationSeconds(filePath); + expect(durationSeconds).toBeGreaterThan(25); + expect(durationSeconds).toBeLessThan(300); + }); // Backend should still reflect the same number of chapters as when we first // observed the stabilized post-cancellation state. @@ -272,12 +320,12 @@ test('exports a single MP3 audiobook PDF page via chapters menu', async ({ page await expect(chapterActionsButtons.first()).toBeVisible({ timeout: 90_000 }); // Download via frontend button - const downloadedPath = await downloadFullAudiobook(page); - - const durationSeconds = await getAudioDurationSeconds(downloadedPath); - // For EPUB we just assert a sane non-trivial duration; at least one 10s mocked chapter. - expect(durationSeconds).toBeGreaterThan(9); - expect(durationSeconds).toBeLessThan(300); + await withDownloadedFullAudiobook(page, async ({ filePath }) => { + const durationSeconds = await getAudioDurationSeconds(filePath); + // For EPUB we just assert a sane non-trivial duration; at least one 10s mocked chapter. + expect(durationSeconds).toBeGreaterThan(9); + expect(durationSeconds).toBeLessThan(300); + }); await resetAudiobookIfPresent(page); }); @@ -372,12 +420,12 @@ test('regenerates a single MP3 audiobook PDF page and exports full audiobook', a await expect(chapterActionsButtons).toHaveCount(chapterCountBefore, { timeout: 20_000 }); // Full Download should still work and produce a valid combined audiobook - const downloadedPath = await downloadFullAudiobook(page); - - const durationSeconds = await getAudioDurationSeconds(downloadedPath); - // With two mocked 10s chapters we expect roughly 20s; allow a small window. - expect(durationSeconds).toBeGreaterThan(18); - expect(durationSeconds).toBeLessThan(22); + await withDownloadedFullAudiobook(page, async ({ filePath }) => { + const durationSeconds = await getAudioDurationSeconds(filePath); + // With two mocked 10s chapters we expect roughly 20s; allow a small window. + expect(durationSeconds).toBeGreaterThan(18); + expect(durationSeconds).toBeLessThan(22); + }); // Backend should still report the same number of chapters and valid durations const json = await expectChaptersBackendState(page, bookId); @@ -448,10 +496,11 @@ test('resumes audiobook when a chapter is missing and full download succeeds (PD // UI should also stop showing a missing placeholder after resume completes. await expect(page.getByText(/Missing •/)).toHaveCount(0, { timeout: 120_000 }); - const downloadedPath = await downloadFullAudiobook(page); - const durationSeconds = await getAudioDurationSeconds(downloadedPath); - expect(durationSeconds).toBeGreaterThan(18); - expect(durationSeconds).toBeLessThan(22); + await withDownloadedFullAudiobook(page, async ({ filePath }) => { + const durationSeconds = await getAudioDurationSeconds(filePath); + expect(durationSeconds).toBeGreaterThan(18); + expect(durationSeconds).toBeLessThan(22); + }); const jsonAfterResume = await expectChaptersBackendState(page, bookId); expect(jsonAfterResume.exists).toBe(true);