feat(user): remove legacy local file deletion and add cleanup for claimed FS sources
Eliminate the unused --delete-local option and related code from the filesystem migration script to prevent accidental local data removal. Introduce a cleanupClaimedLegacyFsSources utility for removing legacy filesystem sources after claim, and integrate it into the user claim flow. Add unit tests to verify the cleanup logic.
This commit is contained in:
Richard R
parent
985c7aaeba
commit
e0cd9bc270
4 changed files with 290 additions and 33 deletions
|
|
@ -44,7 +44,6 @@ function parseBool(value, fallback = false) {
|
|||
function parseArgs(argv) {
|
||||
const args = {
|
||||
dryRun: false,
|
||||
deleteLocal: false,
|
||||
namespace: null,
|
||||
};
|
||||
|
||||
|
|
@ -59,15 +58,6 @@ function parseArgs(argv) {
|
|||
args.dryRun = parseBool(raw.slice('--dry-run='.length), true);
|
||||
continue;
|
||||
}
|
||||
if (raw === '--delete-local' && argv[i + 1]) {
|
||||
args.deleteLocal = parseBool(argv[i + 1], true);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (raw.startsWith('--delete-local=')) {
|
||||
args.deleteLocal = parseBool(raw.slice('--delete-local='.length), true);
|
||||
continue;
|
||||
}
|
||||
if (raw === '--namespace' && argv[i + 1]) {
|
||||
args.namespace = sanitizeNamespace(argv[i + 1]);
|
||||
i += 1;
|
||||
|
|
@ -801,7 +791,7 @@ async function migrateDatabaseRows(database, userId, docCandidates, audiobookBoo
|
|||
|
||||
async function main() {
|
||||
loadEnvFiles();
|
||||
const { dryRun, deleteLocal, namespace } = parseArgs(process.argv.slice(2));
|
||||
const { dryRun, namespace } = parseArgs(process.argv.slice(2));
|
||||
const unclaimedUserId = getUnclaimedUserIdForNamespace(namespace);
|
||||
const docsDir = applyNamespacePath(DOCUMENTS_V1_DIR, namespace);
|
||||
const audiobooksDir = applyNamespacePath(AUDIOBOOKS_V1_DIR, namespace);
|
||||
|
|
@ -815,8 +805,6 @@ async function main() {
|
|||
|
||||
let uploaded = 0;
|
||||
let alreadyPresent = 0;
|
||||
let deletedLocal = 0;
|
||||
|
||||
for (const candidate of documentCandidates) {
|
||||
if (!dryRun) {
|
||||
try {
|
||||
|
|
@ -837,17 +825,10 @@ async function main() {
|
|||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
for (const localPath of candidate.localPaths) {
|
||||
const removed = await fsp.unlink(localPath).then(() => true).catch(() => false);
|
||||
if (removed) deletedLocal += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let audiobookUploaded = 0;
|
||||
let audiobookAlreadyPresent = 0;
|
||||
let audiobookDeletedLocal = 0;
|
||||
|
||||
for (const book of audiobookBooks) {
|
||||
for (const upload of book.uploads) {
|
||||
|
|
@ -871,16 +852,6 @@ async function main() {
|
|||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
const removed = await fsp.unlink(upload.sourcePath).then(() => true).catch(() => false);
|
||||
if (removed) audiobookDeletedLocal += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteLocal && !dryRun) {
|
||||
for (const dirPath of book.sourceDirs) {
|
||||
await fsp.rm(dirPath, { recursive: true, force: true }).catch(() => { });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -897,7 +868,6 @@ async function main() {
|
|||
const result = {
|
||||
success: true,
|
||||
dryRun,
|
||||
deleteLocal,
|
||||
docsDir,
|
||||
audiobooksDir,
|
||||
namespace,
|
||||
|
|
@ -905,14 +875,12 @@ async function main() {
|
|||
uploaded,
|
||||
alreadyPresent,
|
||||
skippedInvalid,
|
||||
deletedLocal,
|
||||
dbRowsUpdated: dbStats.dbRowsUpdated,
|
||||
dbRowsSeeded: dbStats.dbRowsSeeded,
|
||||
audiobookBooksScanned: audiobookStats.booksScanned,
|
||||
audiobookFilesScanned: audiobookStats.filesScanned,
|
||||
audiobookUploaded,
|
||||
audiobookAlreadyPresent,
|
||||
audiobookDeletedLocal,
|
||||
audiobookSkippedTransient: audiobookStats.skippedTransient,
|
||||
audiobookDbBooksSeeded: dbStats.audiobookDbBooksSeeded,
|
||||
audiobookDbChaptersSeeded: dbStats.audiobookDbChaptersSeeded,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { db } from '@/db';
|
|||
import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress } from '@/db/schema';
|
||||
import { eq, and, inArray } from 'drizzle-orm';
|
||||
import { UNCLAIMED_USER_ID } from '../storage/docstore-legacy';
|
||||
import { cleanupClaimedLegacyFsSources } from './legacy-fs-claim-cleanup';
|
||||
import {
|
||||
deleteAudiobookObject,
|
||||
getAudiobookObjectBuffer,
|
||||
|
|
@ -9,6 +10,8 @@ import {
|
|||
putAudiobookObject,
|
||||
} from '../audiobooks/blobstore';
|
||||
import { isS3Configured } from '../storage/s3';
|
||||
import { logDegraded } from '../errors/logging';
|
||||
import { hashForLog, serverLogger } from '../logger';
|
||||
|
||||
type AudiobookRow = {
|
||||
id: string;
|
||||
|
|
@ -91,6 +94,17 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string
|
|||
return { documents: 0, audiobooks: 0, preferences: 0, progress: 0 };
|
||||
}
|
||||
|
||||
const [claimableDocumentRows, claimableAudiobookRows] = await Promise.all([
|
||||
db
|
||||
.select({ id: documents.id })
|
||||
.from(documents)
|
||||
.where(eq(documents.userId, unclaimedUserId)) as Promise<Array<{ id: string }>>,
|
||||
db
|
||||
.select({ id: audiobooks.id })
|
||||
.from(audiobooks)
|
||||
.where(eq(audiobooks.userId, unclaimedUserId)) as Promise<Array<{ id: string }>>,
|
||||
]);
|
||||
|
||||
const [documentsClaimed, audiobooksClaimed, preferencesClaimed, progressClaimed] = await Promise.all([
|
||||
transferUserDocuments(unclaimedUserId, userId),
|
||||
transferUserAudiobooks(unclaimedUserId, userId, namespace),
|
||||
|
|
@ -98,6 +112,28 @@ export async function claimAnonymousData(userId: string, unclaimedUserId: string
|
|||
transferUserProgress(unclaimedUserId, userId),
|
||||
]);
|
||||
|
||||
if (claimableDocumentRows.length > 0 || claimableAudiobookRows.length > 0) {
|
||||
await cleanupClaimedLegacyFsSources({
|
||||
documentIds: claimableDocumentRows.map((row) => row.id),
|
||||
audiobookIds: claimableAudiobookRows.map((row) => row.id),
|
||||
namespace,
|
||||
}).catch((error) => {
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.claim.legacy_fs_cleanup.failed',
|
||||
msg: 'Failed to remove claimed legacy filesystem sources',
|
||||
step: 'cleanup_claimed_legacy_fs_sources',
|
||||
context: {
|
||||
claimedUserIdHash: hashForLog(userId),
|
||||
unclaimedUserIdHash: hashForLog(unclaimedUserId),
|
||||
documentCount: claimableDocumentRows.length,
|
||||
audiobookCount: claimableAudiobookRows.length,
|
||||
namespace,
|
||||
},
|
||||
error,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
documents: documentsClaimed,
|
||||
audiobooks: audiobooksClaimed,
|
||||
|
|
|
|||
149
src/lib/server/user/legacy-fs-claim-cleanup.ts
Normal file
149
src/lib/server/user/legacy-fs-claim-cleanup.ts
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
AUDIOBOOKS_V1_DIR,
|
||||
DOCSTORE_DIR,
|
||||
DOCUMENTS_V1_DIR,
|
||||
} from '@/lib/server/storage/docstore-legacy';
|
||||
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
||||
function applyNamespacePath(baseDir: string, namespace: string | null): string {
|
||||
if (!namespace) return baseDir;
|
||||
const safeNamespace = namespace.trim();
|
||||
if (!SAFE_NAMESPACE_REGEX.test(safeNamespace)) return baseDir;
|
||||
const resolved = path.resolve(baseDir, safeNamespace);
|
||||
if (!resolved.startsWith(path.resolve(baseDir) + path.sep)) return baseDir;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function extractIdFromFileName(fileName: string): string | null {
|
||||
const match = /^([a-f0-9]{64})__/i.exec(fileName);
|
||||
if (!match) return null;
|
||||
const id = match[1].toLowerCase();
|
||||
return DOCUMENT_ID_REGEX.test(id) ? id : null;
|
||||
}
|
||||
|
||||
function isLegacyDocumentMetadata(value: unknown): value is {
|
||||
id: string;
|
||||
type: string;
|
||||
} {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const record = value as Record<string, unknown>;
|
||||
return typeof record.id === 'string' && typeof record.type === 'string';
|
||||
}
|
||||
|
||||
async function collectLegacyDocumentPaths(input: {
|
||||
documentIds: Set<string>;
|
||||
namespace: string | null;
|
||||
}): Promise<Set<string>> {
|
||||
const matches = new Set<string>();
|
||||
if (input.documentIds.size === 0) return matches;
|
||||
|
||||
const docsDir = applyNamespacePath(DOCUMENTS_V1_DIR, input.namespace);
|
||||
const docstoreDir = applyNamespacePath(DOCSTORE_DIR, input.namespace);
|
||||
|
||||
if (fs.existsSync(docsDir)) {
|
||||
const entries = await fsp.readdir(docsDir, { withFileTypes: true }).catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
const fullPath = path.join(docsDir, entry.name);
|
||||
let id = extractIdFromFileName(entry.name);
|
||||
if (!id) {
|
||||
const bytes = await fsp.readFile(fullPath).catch(() => null);
|
||||
if (!bytes) continue;
|
||||
id = createHash('sha256').update(bytes).digest('hex');
|
||||
}
|
||||
if (id && input.documentIds.has(id)) {
|
||||
matches.add(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fs.existsSync(docstoreDir)) {
|
||||
const entries = await fsp.readdir(docstoreDir, { withFileTypes: true }).catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
|
||||
const metadataPath = path.join(docstoreDir, entry.name);
|
||||
const parsed = await fsp.readFile(metadataPath, 'utf8')
|
||||
.then((raw) => JSON.parse(raw) as unknown)
|
||||
.catch(() => null);
|
||||
if (!isLegacyDocumentMetadata(parsed)) continue;
|
||||
|
||||
const contentPath = path.join(docstoreDir, `${parsed.id}.${parsed.type}`);
|
||||
const bytes = await fsp.readFile(contentPath).catch(() => null);
|
||||
if (!bytes) continue;
|
||||
|
||||
const id = createHash('sha256').update(bytes).digest('hex');
|
||||
if (!input.documentIds.has(id)) continue;
|
||||
|
||||
matches.add(metadataPath);
|
||||
matches.add(contentPath);
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
async function collectLegacyAudiobookDirs(input: {
|
||||
audiobookIds: Set<string>;
|
||||
namespace: string | null;
|
||||
}): Promise<Set<string>> {
|
||||
const matches = new Set<string>();
|
||||
if (input.audiobookIds.size === 0) return matches;
|
||||
|
||||
const roots = [
|
||||
applyNamespacePath(AUDIOBOOKS_V1_DIR, input.namespace),
|
||||
applyNamespacePath(DOCSTORE_DIR, input.namespace),
|
||||
];
|
||||
|
||||
for (const root of roots) {
|
||||
if (!fs.existsSync(root)) continue;
|
||||
const entries = await fsp.readdir(root, { withFileTypes: true }).catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !entry.name.endsWith('-audiobook')) continue;
|
||||
const bookId = entry.name.slice(0, -'-audiobook'.length);
|
||||
if (input.audiobookIds.has(bookId)) {
|
||||
matches.add(path.join(root, entry.name));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
export async function cleanupClaimedLegacyFsSources(input: {
|
||||
documentIds: string[];
|
||||
audiobookIds: string[];
|
||||
namespace?: string | null;
|
||||
}): Promise<{ deletedDocumentPaths: number; deletedAudiobookDirs: number }> {
|
||||
const documentIds = new Set(input.documentIds.map((id) => id.trim().toLowerCase()).filter(Boolean));
|
||||
const audiobookIds = new Set(input.audiobookIds.map((id) => id.trim()).filter(Boolean));
|
||||
const namespace = input.namespace ?? null;
|
||||
|
||||
const [documentPaths, audiobookDirs] = await Promise.all([
|
||||
collectLegacyDocumentPaths({ documentIds, namespace }),
|
||||
collectLegacyAudiobookDirs({ audiobookIds, namespace }),
|
||||
]);
|
||||
|
||||
let deletedDocumentPaths = 0;
|
||||
for (const filePath of documentPaths) {
|
||||
const removed = await fsp.unlink(filePath).then(() => true).catch(() => false);
|
||||
if (removed) deletedDocumentPaths += 1;
|
||||
}
|
||||
|
||||
let deletedAudiobookDirs = 0;
|
||||
for (const dirPath of audiobookDirs) {
|
||||
const existed = fs.existsSync(dirPath);
|
||||
await fsp.rm(dirPath, { recursive: true, force: true }).catch(() => {});
|
||||
if (existed && !fs.existsSync(dirPath)) deletedAudiobookDirs += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
deletedDocumentPaths,
|
||||
deletedAudiobookDirs,
|
||||
};
|
||||
}
|
||||
104
tests/unit/legacy-fs-claim-cleanup.vitest.spec.ts
Normal file
104
tests/unit/legacy-fs-claim-cleanup.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
roots: {
|
||||
docstoreDir: '',
|
||||
documentsDir: '',
|
||||
audiobooksDir: '',
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/docstore-legacy', () => ({
|
||||
get DOCSTORE_DIR() {
|
||||
return hoisted.roots.docstoreDir;
|
||||
},
|
||||
get DOCUMENTS_V1_DIR() {
|
||||
return hoisted.roots.documentsDir;
|
||||
},
|
||||
get AUDIOBOOKS_V1_DIR() {
|
||||
return hoisted.roots.audiobooksDir;
|
||||
},
|
||||
}));
|
||||
|
||||
describe('cleanupClaimedLegacyFsSources', () => {
|
||||
let tempRoot = '';
|
||||
|
||||
beforeEach(async () => {
|
||||
tempRoot = await fsp.mkdtemp(path.join(os.tmpdir(), 'openreader-legacy-cleanup-'));
|
||||
hoisted.roots = {
|
||||
docstoreDir: path.join(tempRoot, 'docstore'),
|
||||
documentsDir: path.join(tempRoot, 'documents_v1'),
|
||||
audiobooksDir: path.join(tempRoot, 'audiobooks_v1'),
|
||||
};
|
||||
await fsp.mkdir(hoisted.roots.docstoreDir, { recursive: true });
|
||||
await fsp.mkdir(hoisted.roots.documentsDir, { recursive: true });
|
||||
await fsp.mkdir(hoisted.roots.audiobooksDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fsp.rm(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('removes only claimed legacy document and audiobook sources', async () => {
|
||||
const namespace = 'ns1';
|
||||
const docsNsDir = path.join(hoisted.roots.documentsDir, namespace);
|
||||
const docstoreNsDir = path.join(hoisted.roots.docstoreDir, namespace);
|
||||
const audiobooksNsDir = path.join(hoisted.roots.audiobooksDir, namespace);
|
||||
await fsp.mkdir(docsNsDir, { recursive: true });
|
||||
await fsp.mkdir(docstoreNsDir, { recursive: true });
|
||||
await fsp.mkdir(audiobooksNsDir, { recursive: true });
|
||||
|
||||
const doc1Bytes = Buffer.from('claimed-doc-1');
|
||||
const doc1Id = createHash('sha256').update(doc1Bytes).digest('hex');
|
||||
const doc1Path = path.join(docsNsDir, 'legacy-random-name.pdf');
|
||||
await fsp.writeFile(doc1Path, doc1Bytes);
|
||||
|
||||
const doc2Bytes = Buffer.from('claimed-doc-2');
|
||||
const doc2Id = createHash('sha256').update(doc2Bytes).digest('hex');
|
||||
const doc2MetaPath = path.join(docstoreNsDir, 'legacy-doc-2.json');
|
||||
const doc2ContentPath = path.join(docstoreNsDir, 'legacy-doc-2.pdf');
|
||||
await fsp.writeFile(doc2MetaPath, JSON.stringify({
|
||||
id: 'legacy-doc-2',
|
||||
name: 'legacy-doc-2.pdf',
|
||||
size: doc2Bytes.length,
|
||||
lastModified: Date.now(),
|
||||
type: 'pdf',
|
||||
}));
|
||||
await fsp.writeFile(doc2ContentPath, doc2Bytes);
|
||||
|
||||
const keepBytes = Buffer.from('keep-doc');
|
||||
const keepId = createHash('sha256').update(keepBytes).digest('hex');
|
||||
const keepPath = path.join(docsNsDir, `${keepId}__keep.pdf`);
|
||||
await fsp.writeFile(keepPath, keepBytes);
|
||||
|
||||
const claimedBookDir = path.join(audiobooksNsDir, 'book-1-audiobook');
|
||||
const keepBookDir = path.join(docstoreNsDir, 'book-2-audiobook');
|
||||
await fsp.mkdir(claimedBookDir, { recursive: true });
|
||||
await fsp.mkdir(keepBookDir, { recursive: true });
|
||||
await fsp.writeFile(path.join(claimedBookDir, '0001__Chapter%201.mp3'), 'audio');
|
||||
await fsp.writeFile(path.join(keepBookDir, '0001__Chapter%201.mp3'), 'audio');
|
||||
|
||||
const { cleanupClaimedLegacyFsSources } = await import('../../src/lib/server/user/legacy-fs-claim-cleanup');
|
||||
const result = await cleanupClaimedLegacyFsSources({
|
||||
documentIds: [doc1Id, doc2Id],
|
||||
audiobookIds: ['book-1'],
|
||||
namespace,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
deletedDocumentPaths: 3,
|
||||
deletedAudiobookDirs: 1,
|
||||
});
|
||||
expect(fs.existsSync(doc1Path)).toBe(false);
|
||||
expect(fs.existsSync(doc2MetaPath)).toBe(false);
|
||||
expect(fs.existsSync(doc2ContentPath)).toBe(false);
|
||||
expect(fs.existsSync(keepPath)).toBe(true);
|
||||
expect(fs.existsSync(claimedBookDir)).toBe(false);
|
||||
expect(fs.existsSync(keepBookDir)).toBe(true);
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue