From e0cd9bc27013a758038c6c7ded6ac528a18e34cb Mon Sep 17 00:00:00 2001 From: Richard R Date: Wed, 3 Jun 2026 14:35:27 -0600 Subject: [PATCH] feat(user): remove legacy local file deletion and add cleanup for claimed FS sources Eliminate the unused --delete-local option and related code from the filesystem migration script to prevent accidental local data removal. Introduce a cleanupClaimedLegacyFsSources utility for removing legacy filesystem sources after claim, and integrate it into the user claim flow. Add unit tests to verify the cleanup logic. --- scripts/migrate-fs-v2.mjs | 34 +--- src/lib/server/user/claim-data.ts | 36 +++++ .../server/user/legacy-fs-claim-cleanup.ts | 149 ++++++++++++++++++ .../legacy-fs-claim-cleanup.vitest.spec.ts | 104 ++++++++++++ 4 files changed, 290 insertions(+), 33 deletions(-) create mode 100644 src/lib/server/user/legacy-fs-claim-cleanup.ts create mode 100644 tests/unit/legacy-fs-claim-cleanup.vitest.spec.ts diff --git a/scripts/migrate-fs-v2.mjs b/scripts/migrate-fs-v2.mjs index 5771e43..e2f0c7d 100644 --- a/scripts/migrate-fs-v2.mjs +++ b/scripts/migrate-fs-v2.mjs @@ -44,7 +44,6 @@ function parseBool(value, fallback = false) { function parseArgs(argv) { const args = { dryRun: false, - deleteLocal: false, namespace: null, }; @@ -59,15 +58,6 @@ function parseArgs(argv) { args.dryRun = parseBool(raw.slice('--dry-run='.length), true); continue; } - if (raw === '--delete-local' && argv[i + 1]) { - args.deleteLocal = parseBool(argv[i + 1], true); - i += 1; - continue; - } - if (raw.startsWith('--delete-local=')) { - args.deleteLocal = parseBool(raw.slice('--delete-local='.length), true); - continue; - } if (raw === '--namespace' && argv[i + 1]) { args.namespace = sanitizeNamespace(argv[i + 1]); i += 1; @@ -801,7 +791,7 @@ async function migrateDatabaseRows(database, userId, docCandidates, audiobookBoo async function main() { loadEnvFiles(); - const { dryRun, deleteLocal, namespace } = parseArgs(process.argv.slice(2)); + const { dryRun, namespace } = parseArgs(process.argv.slice(2)); const unclaimedUserId = getUnclaimedUserIdForNamespace(namespace); const docsDir = applyNamespacePath(DOCUMENTS_V1_DIR, namespace); const audiobooksDir = applyNamespacePath(AUDIOBOOKS_V1_DIR, namespace); @@ -815,8 +805,6 @@ async function main() { let uploaded = 0; let alreadyPresent = 0; - let deletedLocal = 0; - for (const candidate of documentCandidates) { if (!dryRun) { try { @@ -837,17 +825,10 @@ async function main() { } } - if (deleteLocal && !dryRun) { - for (const localPath of candidate.localPaths) { - const removed = await fsp.unlink(localPath).then(() => true).catch(() => false); - if (removed) deletedLocal += 1; - } - } } let audiobookUploaded = 0; let audiobookAlreadyPresent = 0; - let audiobookDeletedLocal = 0; for (const book of audiobookBooks) { for (const upload of book.uploads) { @@ -871,16 +852,6 @@ async function main() { } } - if (deleteLocal && !dryRun) { - const removed = await fsp.unlink(upload.sourcePath).then(() => true).catch(() => false); - if (removed) audiobookDeletedLocal += 1; - } - } - - if (deleteLocal && !dryRun) { - for (const dirPath of book.sourceDirs) { - await fsp.rm(dirPath, { recursive: true, force: true }).catch(() => { }); - } } } @@ -897,7 +868,6 @@ async function main() { const result = { success: true, dryRun, - deleteLocal, docsDir, audiobooksDir, namespace, @@ -905,14 +875,12 @@ async function main() { uploaded, alreadyPresent, skippedInvalid, - deletedLocal, dbRowsUpdated: dbStats.dbRowsUpdated, dbRowsSeeded: dbStats.dbRowsSeeded, audiobookBooksScanned: audiobookStats.booksScanned, audiobookFilesScanned: audiobookStats.filesScanned, audiobookUploaded, audiobookAlreadyPresent, - audiobookDeletedLocal, audiobookSkippedTransient: audiobookStats.skippedTransient, audiobookDbBooksSeeded: dbStats.audiobookDbBooksSeeded, audiobookDbChaptersSeeded: dbStats.audiobookDbChaptersSeeded, diff --git a/src/lib/server/user/claim-data.ts b/src/lib/server/user/claim-data.ts index 3a9f88c..24673ec 100644 --- a/src/lib/server/user/claim-data.ts +++ b/src/lib/server/user/claim-data.ts @@ -2,6 +2,7 @@ import { db } from '@/db'; import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress } from '@/db/schema'; import { eq, and, inArray } from 'drizzle-orm'; import { UNCLAIMED_USER_ID } from '../storage/docstore-legacy'; +import { cleanupClaimedLegacyFsSources } from './legacy-fs-claim-cleanup'; import { deleteAudiobookObject, getAudiobookObjectBuffer, @@ -9,6 +10,8 @@ import { putAudiobookObject, } from '../audiobooks/blobstore'; import { isS3Configured } from '../storage/s3'; +import { logDegraded } from '../errors/logging'; +import { hashForLog, serverLogger } from '../logger'; type AudiobookRow = { id: string; @@ -91,6 +94,17 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string return { documents: 0, audiobooks: 0, preferences: 0, progress: 0 }; } + const [claimableDocumentRows, claimableAudiobookRows] = await Promise.all([ + db + .select({ id: documents.id }) + .from(documents) + .where(eq(documents.userId, unclaimedUserId)) as Promise>, + db + .select({ id: audiobooks.id }) + .from(audiobooks) + .where(eq(audiobooks.userId, unclaimedUserId)) as Promise>, + ]); + const [documentsClaimed, audiobooksClaimed, preferencesClaimed, progressClaimed] = await Promise.all([ transferUserDocuments(unclaimedUserId, userId), transferUserAudiobooks(unclaimedUserId, userId, namespace), @@ -98,6 +112,28 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string transferUserProgress(unclaimedUserId, userId), ]); + if (claimableDocumentRows.length > 0 || claimableAudiobookRows.length > 0) { + await cleanupClaimedLegacyFsSources({ + documentIds: claimableDocumentRows.map((row) => row.id), + audiobookIds: claimableAudiobookRows.map((row) => row.id), + namespace, + }).catch((error) => { + logDegraded(serverLogger, { + event: 'user.claim.legacy_fs_cleanup.failed', + msg: 'Failed to remove claimed legacy filesystem sources', + step: 'cleanup_claimed_legacy_fs_sources', + context: { + claimedUserIdHash: hashForLog(userId), + unclaimedUserIdHash: hashForLog(unclaimedUserId), + documentCount: claimableDocumentRows.length, + audiobookCount: claimableAudiobookRows.length, + namespace, + }, + error, + }); + }); + } + return { documents: documentsClaimed, audiobooks: audiobooksClaimed, diff --git a/src/lib/server/user/legacy-fs-claim-cleanup.ts b/src/lib/server/user/legacy-fs-claim-cleanup.ts new file mode 100644 index 0000000..d11987a --- /dev/null +++ b/src/lib/server/user/legacy-fs-claim-cleanup.ts @@ -0,0 +1,149 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { + AUDIOBOOKS_V1_DIR, + DOCSTORE_DIR, + DOCUMENTS_V1_DIR, +} from '@/lib/server/storage/docstore-legacy'; + +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; + +function applyNamespacePath(baseDir: string, namespace: string | null): string { + if (!namespace) return baseDir; + const safeNamespace = namespace.trim(); + if (!SAFE_NAMESPACE_REGEX.test(safeNamespace)) return baseDir; + const resolved = path.resolve(baseDir, safeNamespace); + if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir; + return resolved; +} + +function extractIdFromFileName(fileName: string): string | null { + const match = /^([a-f0-9]{64})__/i.exec(fileName); + if (!match) return null; + const id = match[1].toLowerCase(); + return DOCUMENT_ID_REGEX.test(id) ? id : null; +} + +function isLegacyDocumentMetadata(value: unknown): value is { + id: string; + type: string; +} { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + return typeof record.id === 'string' && typeof record.type === 'string'; +} + +async function collectLegacyDocumentPaths(input: { + documentIds: Set; + namespace: string | null; +}): Promise> { + const matches = new Set(); + if (input.documentIds.size === 0) return matches; + + const docsDir = applyNamespacePath(DOCUMENTS_V1_DIR, input.namespace); + const docstoreDir = applyNamespacePath(DOCSTORE_DIR, input.namespace); + + if (fs.existsSync(docsDir)) { + const entries = await fsp.readdir(docsDir, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (!entry.isFile()) continue; + const fullPath = path.join(docsDir, entry.name); + let id = extractIdFromFileName(entry.name); + if (!id) { + const bytes = await fsp.readFile(fullPath).catch(() => null); + if (!bytes) continue; + id = createHash('sha256').update(bytes).digest('hex'); + } + if (id && input.documentIds.has(id)) { + matches.add(fullPath); + } + } + } + + if (fs.existsSync(docstoreDir)) { + const entries = await fsp.readdir(docstoreDir, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue; + const metadataPath = path.join(docstoreDir, entry.name); + const parsed = await fsp.readFile(metadataPath, 'utf8') + .then((raw) => JSON.parse(raw) as unknown) + .catch(() => null); + if (!isLegacyDocumentMetadata(parsed)) continue; + + const contentPath = path.join(docstoreDir, `${parsed.id}.${parsed.type}`); + const bytes = await fsp.readFile(contentPath).catch(() => null); + if (!bytes) continue; + + const id = createHash('sha256').update(bytes).digest('hex'); + if (!input.documentIds.has(id)) continue; + + matches.add(metadataPath); + matches.add(contentPath); + } + } + + return matches; +} + +async function collectLegacyAudiobookDirs(input: { + audiobookIds: Set; + namespace: string | null; +}): Promise> { + const matches = new Set(); + if (input.audiobookIds.size === 0) return matches; + + const roots = [ + applyNamespacePath(AUDIOBOOKS_V1_DIR, input.namespace), + applyNamespacePath(DOCSTORE_DIR, input.namespace), + ]; + + for (const root of roots) { + if (!fs.existsSync(root)) continue; + const entries = await fsp.readdir(root, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.endsWith('-audiobook')) continue; + const bookId = entry.name.slice(0, -'-audiobook'.length); + if (input.audiobookIds.has(bookId)) { + matches.add(path.join(root, entry.name)); + } + } + } + + return matches; +} + +export async function cleanupClaimedLegacyFsSources(input: { + documentIds: string[]; + audiobookIds: string[]; + namespace?: string | null; +}): Promise<{ deletedDocumentPaths: number; deletedAudiobookDirs: number }> { + const documentIds = new Set(input.documentIds.map((id) => id.trim().toLowerCase()).filter(Boolean)); + const audiobookIds = new Set(input.audiobookIds.map((id) => id.trim()).filter(Boolean)); + const namespace = input.namespace ?? null; + + const [documentPaths, audiobookDirs] = await Promise.all([ + collectLegacyDocumentPaths({ documentIds, namespace }), + collectLegacyAudiobookDirs({ audiobookIds, namespace }), + ]); + + let deletedDocumentPaths = 0; + for (const filePath of documentPaths) { + const removed = await fsp.unlink(filePath).then(() => true).catch(() => false); + if (removed) deletedDocumentPaths += 1; + } + + let deletedAudiobookDirs = 0; + for (const dirPath of audiobookDirs) { + const existed = fs.existsSync(dirPath); + await fsp.rm(dirPath, { recursive: true, force: true }).catch(() => {}); + if (existed && !fs.existsSync(dirPath)) deletedAudiobookDirs += 1; + } + + return { + deletedDocumentPaths, + deletedAudiobookDirs, + }; +} diff --git a/tests/unit/legacy-fs-claim-cleanup.vitest.spec.ts b/tests/unit/legacy-fs-claim-cleanup.vitest.spec.ts new file mode 100644 index 0000000..c5555e0 --- /dev/null +++ b/tests/unit/legacy-fs-claim-cleanup.vitest.spec.ts @@ -0,0 +1,104 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +const hoisted = vi.hoisted(() => ({ + roots: { + docstoreDir: '', + documentsDir: '', + audiobooksDir: '', + }, +})); + +vi.mock('@/lib/server/storage/docstore-legacy', () => ({ + get DOCSTORE_DIR() { + return hoisted.roots.docstoreDir; + }, + get DOCUMENTS_V1_DIR() { + return hoisted.roots.documentsDir; + }, + get AUDIOBOOKS_V1_DIR() { + return hoisted.roots.audiobooksDir; + }, +})); + +describe('cleanupClaimedLegacyFsSources', () => { + let tempRoot = ''; + + beforeEach(async () => { + tempRoot = await fsp.mkdtemp(path.join(os.tmpdir(), 'openreader-legacy-cleanup-')); + hoisted.roots = { + docstoreDir: path.join(tempRoot, 'docstore'), + documentsDir: path.join(tempRoot, 'documents_v1'), + audiobooksDir: path.join(tempRoot, 'audiobooks_v1'), + }; + await fsp.mkdir(hoisted.roots.docstoreDir, { recursive: true }); + await fsp.mkdir(hoisted.roots.documentsDir, { recursive: true }); + await fsp.mkdir(hoisted.roots.audiobooksDir, { recursive: true }); + }); + + afterEach(async () => { + await fsp.rm(tempRoot, { recursive: true, force: true }); + }); + + test('removes only claimed legacy document and audiobook sources', async () => { + const namespace = 'ns1'; + const docsNsDir = path.join(hoisted.roots.documentsDir, namespace); + const docstoreNsDir = path.join(hoisted.roots.docstoreDir, namespace); + const audiobooksNsDir = path.join(hoisted.roots.audiobooksDir, namespace); + await fsp.mkdir(docsNsDir, { recursive: true }); + await fsp.mkdir(docstoreNsDir, { recursive: true }); + await fsp.mkdir(audiobooksNsDir, { recursive: true }); + + const doc1Bytes = Buffer.from('claimed-doc-1'); + const doc1Id = createHash('sha256').update(doc1Bytes).digest('hex'); + const doc1Path = path.join(docsNsDir, 'legacy-random-name.pdf'); + await fsp.writeFile(doc1Path, doc1Bytes); + + const doc2Bytes = Buffer.from('claimed-doc-2'); + const doc2Id = createHash('sha256').update(doc2Bytes).digest('hex'); + const doc2MetaPath = path.join(docstoreNsDir, 'legacy-doc-2.json'); + const doc2ContentPath = path.join(docstoreNsDir, 'legacy-doc-2.pdf'); + await fsp.writeFile(doc2MetaPath, JSON.stringify({ + id: 'legacy-doc-2', + name: 'legacy-doc-2.pdf', + size: doc2Bytes.length, + lastModified: Date.now(), + type: 'pdf', + })); + await fsp.writeFile(doc2ContentPath, doc2Bytes); + + const keepBytes = Buffer.from('keep-doc'); + const keepId = createHash('sha256').update(keepBytes).digest('hex'); + const keepPath = path.join(docsNsDir, `${keepId}__keep.pdf`); + await fsp.writeFile(keepPath, keepBytes); + + const claimedBookDir = path.join(audiobooksNsDir, 'book-1-audiobook'); + const keepBookDir = path.join(docstoreNsDir, 'book-2-audiobook'); + await fsp.mkdir(claimedBookDir, { recursive: true }); + await fsp.mkdir(keepBookDir, { recursive: true }); + await fsp.writeFile(path.join(claimedBookDir, '0001__Chapter%201.mp3'), 'audio'); + await fsp.writeFile(path.join(keepBookDir, '0001__Chapter%201.mp3'), 'audio'); + + const { cleanupClaimedLegacyFsSources } = await import('../../src/lib/server/user/legacy-fs-claim-cleanup'); + const result = await cleanupClaimedLegacyFsSources({ + documentIds: [doc1Id, doc2Id], + audiobookIds: ['book-1'], + namespace, + }); + + expect(result).toEqual({ + deletedDocumentPaths: 3, + deletedAudiobookDirs: 1, + }); + expect(fs.existsSync(doc1Path)).toBe(false); + expect(fs.existsSync(doc2MetaPath)).toBe(false); + expect(fs.existsSync(doc2ContentPath)).toBe(false); + expect(fs.existsSync(keepPath)).toBe(true); + expect(fs.existsSync(claimedBookDir)).toBe(false); + expect(fs.existsSync(keepBookDir)).toBe(true); + }); +});