refactor(user): overhaul user data cleanup and export for cascading deletes and TTS segment support
Revise user data cleanup logic to ensure proper cascading deletion of user-related database rows and S3 objects, including shared document and preview artifacts. Introduce explicit checks for last ownership before removing shared resources. Add TTS segment cache and audio cleanup to document and user deletion flows. Expand user data export to include TTS segment entries and audio, job events, document settings, and linked auth sessions. Update schema with ON DELETE CASCADE for userTtsChars and userJobEvents. Add new migration scripts and comprehensive unit tests for cleanup and export scenarios.
This commit is contained in:
parent
92df3f907c
commit
76553023e8
27 changed files with 4425 additions and 66 deletions
8
drizzle/postgres/0010_user-data-cleanup-cascades.sql
Normal file
8
drizzle/postgres/0010_user-data-cleanup-cascades.sql
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
DELETE FROM "user_job_events" WHERE NOT EXISTS (
|
||||
SELECT 1 FROM "user" WHERE "user"."id" = "user_job_events"."user_id"
|
||||
);--> statement-breakpoint
|
||||
DELETE FROM "user_tts_chars" WHERE NOT EXISTS (
|
||||
SELECT 1 FROM "user" WHERE "user"."id" = "user_tts_chars"."user_id"
|
||||
);--> statement-breakpoint
|
||||
ALTER TABLE "user_job_events" ADD CONSTRAINT "user_job_events_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "user_tts_chars" ADD CONSTRAINT "user_tts_chars_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;
|
||||
1922
drizzle/postgres/meta/0010_snapshot.json
Normal file
1922
drizzle/postgres/meta/0010_snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -71,6 +71,13 @@
|
|||
"when": 1780625663880,
|
||||
"tag": "0009_drop_pdf_parse",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 10,
|
||||
"version": "7",
|
||||
"when": 1780785284335,
|
||||
"tag": "0010_user-data-cleanup-cascades",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
35
drizzle/sqlite/0010_user-data-cleanup-cascades.sql
Normal file
35
drizzle/sqlite/0010_user-data-cleanup-cascades.sql
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
PRAGMA foreign_keys=OFF;--> statement-breakpoint
|
||||
DELETE FROM `user_job_events` WHERE NOT EXISTS (
|
||||
SELECT 1 FROM `user` WHERE `user`.`id` = `user_job_events`.`user_id`
|
||||
);--> statement-breakpoint
|
||||
DELETE FROM `user_tts_chars` WHERE NOT EXISTS (
|
||||
SELECT 1 FROM `user` WHERE `user`.`id` = `user_tts_chars`.`user_id`
|
||||
);--> statement-breakpoint
|
||||
CREATE TABLE `__new_user_job_events` (
|
||||
`user_id` text NOT NULL,
|
||||
`action` text NOT NULL,
|
||||
`op_id` text NOT NULL,
|
||||
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
|
||||
PRIMARY KEY(`user_id`, `action`, `op_id`),
|
||||
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
INSERT INTO `__new_user_job_events`("user_id", "action", "op_id", "created_at") SELECT "user_id", "action", "op_id", "created_at" FROM `user_job_events`;--> statement-breakpoint
|
||||
DROP TABLE `user_job_events`;--> statement-breakpoint
|
||||
ALTER TABLE `__new_user_job_events` RENAME TO `user_job_events`;--> statement-breakpoint
|
||||
PRAGMA foreign_keys=ON;--> statement-breakpoint
|
||||
CREATE INDEX `idx_user_job_events_user_action_created` ON `user_job_events` (`user_id`,`action`,`created_at`);--> statement-breakpoint
|
||||
CREATE TABLE `__new_user_tts_chars` (
|
||||
`user_id` text NOT NULL,
|
||||
`date` text NOT NULL,
|
||||
`char_count` integer DEFAULT 0,
|
||||
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
|
||||
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
|
||||
PRIMARY KEY(`user_id`, `date`),
|
||||
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
INSERT INTO `__new_user_tts_chars`("user_id", "date", "char_count", "created_at", "updated_at") SELECT "user_id", "date", "char_count", "created_at", "updated_at" FROM `user_tts_chars`;--> statement-breakpoint
|
||||
DROP TABLE `user_tts_chars`;--> statement-breakpoint
|
||||
ALTER TABLE `__new_user_tts_chars` RENAME TO `user_tts_chars`;--> statement-breakpoint
|
||||
CREATE INDEX `idx_user_tts_chars_date` ON `user_tts_chars` (`date`);
|
||||
1765
drizzle/sqlite/meta/0010_snapshot.json
Normal file
1765
drizzle/sqlite/meta/0010_snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -71,6 +71,13 @@
|
|||
"when": 1780625663601,
|
||||
"tag": "0009_drop_pdf_parse",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 10,
|
||||
"version": "6",
|
||||
"when": 1780785284041,
|
||||
"tag": "0010_user-data-cleanup-cascades",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@ import { NextResponse } from 'next/server';
|
|||
import { auth } from '@/lib/server/auth/auth';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { deleteUserStorageData } from '@/lib/server/user/data-cleanup';
|
||||
import { errorToLog, hashForLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
|
||||
export async function DELETE() {
|
||||
|
|
@ -18,21 +18,12 @@ export async function DELETE() {
|
|||
}
|
||||
|
||||
try {
|
||||
// Best-effort cleanup for test namespaced storage using request context.
|
||||
// The Better Auth beforeDelete hook still runs and handles non-namespaced data.
|
||||
// Clean test-namespaced storage using request context. The Better Auth
|
||||
// beforeDelete hook handles non-namespaced storage and blocks deletion if
|
||||
// either cleanup cannot complete.
|
||||
const testNamespace = getOpenReaderTestNamespace(reqHeaders);
|
||||
if (testNamespace) {
|
||||
try {
|
||||
await deleteUserStorageData(session.user.id, testNamespace);
|
||||
} catch (error) {
|
||||
serverLogger.warn({
|
||||
event: 'account.delete.storage_cleanup_failed',
|
||||
degraded: true,
|
||||
step: 'namespaced_storage_cleanup',
|
||||
userIdHash: hashForLog(session.user.id),
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to clean up namespaced user storage before deletion');
|
||||
}
|
||||
await deleteUserStorageData(session.user.id, testNamespace);
|
||||
}
|
||||
|
||||
// Use Better Auth's built-in deleteUser to handle cascading cleanup
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'
|
|||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -80,9 +81,14 @@ export async function GET(req: NextRequest) {
|
|||
});
|
||||
} catch (error) {
|
||||
if (isMissingBlobError(error)) {
|
||||
await deleteDocumentTtsSegmentCache({
|
||||
userId: doc.userId,
|
||||
documentId: id,
|
||||
namespace: testNamespace,
|
||||
});
|
||||
await db
|
||||
.delete(documents)
|
||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
|
||||
.where(and(eq(documents.id, id), eq(documents.userId, doc.userId)));
|
||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||
}
|
||||
throw error;
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import {
|
|||
import { extractRawTextSnippet } from '@/lib/server/documents/text-snippets';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
|
|
@ -108,9 +109,14 @@ export async function GET(req: NextRequest) {
|
|||
);
|
||||
} catch (error) {
|
||||
if (isMissingDocumentBlobError(error)) {
|
||||
await deleteDocumentTtsSegmentCache({
|
||||
userId: doc.userId,
|
||||
documentId: id,
|
||||
namespace: testNamespace,
|
||||
});
|
||||
await db
|
||||
.delete(documents)
|
||||
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
|
||||
.where(and(eq(documents.id, id), eq(documents.userId, doc.userId)));
|
||||
return NextResponse.json({ error: 'Not found' }, { status: 404 });
|
||||
}
|
||||
throw error;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
|
@ -136,6 +137,25 @@ export async function DELETE(req: NextRequest) {
|
|||
return NextResponse.json({ success: true, deleted: 0 });
|
||||
}
|
||||
|
||||
const ownedRows = (await db
|
||||
.select({ id: documents.id, userId: documents.userId })
|
||||
.from(documents)
|
||||
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))) as Array<{
|
||||
id: string;
|
||||
userId: string;
|
||||
}>;
|
||||
|
||||
// TTS audio is user-scoped even when the underlying document blob is
|
||||
// shared. Remove it before deleting ownership metadata so failures remain
|
||||
// retryable and cannot create untraceable S3 objects.
|
||||
for (const row of ownedRows) {
|
||||
await deleteDocumentTtsSegmentCache({
|
||||
userId: row.userId,
|
||||
documentId: row.id,
|
||||
namespace: testNamespace,
|
||||
});
|
||||
}
|
||||
|
||||
const deletedRows = (await db
|
||||
.delete(documents)
|
||||
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))
|
||||
|
|
|
|||
|
|
@ -2,12 +2,26 @@ import { NextRequest, NextResponse } from 'next/server';
|
|||
import { PassThrough, Readable } from 'stream';
|
||||
import { auth } from '@/lib/server/auth/auth';
|
||||
import { db } from '@/db';
|
||||
import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress, userTtsChars } from '@/db/schema';
|
||||
import {
|
||||
documents,
|
||||
audiobooks,
|
||||
audiobookChapters,
|
||||
documentSettings,
|
||||
ttsSegmentEntries,
|
||||
ttsSegmentVariants,
|
||||
userDocumentProgress,
|
||||
userJobEvents,
|
||||
userPreferences,
|
||||
userTtsChars,
|
||||
} from '@/db/schema';
|
||||
import * as authSchemaSqlite from '@/db/schema_auth_sqlite';
|
||||
import * as authSchemaPostgres from '@/db/schema_auth_postgres';
|
||||
import { and, desc, eq, inArray } from 'drizzle-orm';
|
||||
import archiver from 'archiver';
|
||||
import { appendUserExportArchive } from '@/lib/server/user/data-export';
|
||||
import { getDocumentBlobStream } from '@/lib/server/documents/blobstore';
|
||||
import { getAudiobookObjectStream, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore';
|
||||
import { getTtsSegmentAudioObjectStream } from '@/lib/server/tts/segments-blobstore';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { nowTimestampMs } from '@/lib/shared/timestamps';
|
||||
|
|
@ -17,13 +31,6 @@ import { errorResponse } from '@/lib/server/errors/next-response';
|
|||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Export storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!auth) {
|
||||
return errorResponse(new Error('Auth not initialized'), {
|
||||
apiErrorMessage: 'Auth not initialized',
|
||||
|
|
@ -41,11 +48,21 @@ export async function GET(req: NextRequest) {
|
|||
|
||||
const userId = session.user.id;
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
const storageEnabled = isS3Configured();
|
||||
const requireStorage = () => {
|
||||
if (!storageEnabled) {
|
||||
throw new Error('Storage is not configured; file content could not be exported');
|
||||
}
|
||||
};
|
||||
|
||||
const [
|
||||
prefs,
|
||||
progress,
|
||||
ttsUsage,
|
||||
jobEvents,
|
||||
perDocumentSettings,
|
||||
segmentEntries,
|
||||
segmentVariants,
|
||||
userDocs,
|
||||
userAudiobooks,
|
||||
] = await Promise.all([
|
||||
|
|
@ -60,6 +77,26 @@ export async function GET(req: NextRequest) {
|
|||
.from(userTtsChars)
|
||||
.where(eq(userTtsChars.userId, userId))
|
||||
.orderBy(desc(userTtsChars.date)),
|
||||
db
|
||||
.select()
|
||||
.from(userJobEvents)
|
||||
.where(eq(userJobEvents.userId, userId))
|
||||
.orderBy(desc(userJobEvents.createdAt)),
|
||||
db
|
||||
.select()
|
||||
.from(documentSettings)
|
||||
.where(eq(documentSettings.userId, userId))
|
||||
.orderBy(desc(documentSettings.updatedAt)),
|
||||
db
|
||||
.select()
|
||||
.from(ttsSegmentEntries)
|
||||
.where(eq(ttsSegmentEntries.userId, userId))
|
||||
.orderBy(desc(ttsSegmentEntries.updatedAt)),
|
||||
db
|
||||
.select()
|
||||
.from(ttsSegmentVariants)
|
||||
.where(eq(ttsSegmentVariants.userId, userId))
|
||||
.orderBy(desc(ttsSegmentVariants.updatedAt)),
|
||||
db
|
||||
.select()
|
||||
.from(documents)
|
||||
|
|
@ -72,6 +109,36 @@ export async function GET(req: NextRequest) {
|
|||
.orderBy(desc(audiobooks.createdAt)),
|
||||
]);
|
||||
|
||||
const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite;
|
||||
// Auth exports intentionally select metadata only. Credential and session
|
||||
// secrets must never be written into the archive.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const database = db as any;
|
||||
const [authSessions, linkedAccounts] = await Promise.all([
|
||||
database
|
||||
.select({
|
||||
id: authSchema.session.id,
|
||||
expiresAt: authSchema.session.expiresAt,
|
||||
createdAt: authSchema.session.createdAt,
|
||||
updatedAt: authSchema.session.updatedAt,
|
||||
ipAddress: authSchema.session.ipAddress,
|
||||
userAgent: authSchema.session.userAgent,
|
||||
})
|
||||
.from(authSchema.session)
|
||||
.where(eq(authSchema.session.userId, userId)),
|
||||
database
|
||||
.select({
|
||||
id: authSchema.account.id,
|
||||
accountId: authSchema.account.accountId,
|
||||
providerId: authSchema.account.providerId,
|
||||
scope: authSchema.account.scope,
|
||||
createdAt: authSchema.account.createdAt,
|
||||
updatedAt: authSchema.account.updatedAt,
|
||||
})
|
||||
.from(authSchema.account)
|
||||
.where(eq(authSchema.account.userId, userId)),
|
||||
]);
|
||||
|
||||
const archive = archiver('zip', {
|
||||
zlib: { level: 0 },
|
||||
forceZip64: true,
|
||||
|
|
@ -114,7 +181,6 @@ export async function GET(req: NextRequest) {
|
|||
const exportedAtMs = nowTimestampMs();
|
||||
const profileData = {
|
||||
user: session.user,
|
||||
session: session.session,
|
||||
exportedAtMs,
|
||||
};
|
||||
|
||||
|
|
@ -126,13 +192,31 @@ export async function GET(req: NextRequest) {
|
|||
preferences: prefs[0] ?? null,
|
||||
readingHistory: progress,
|
||||
ttsUsage,
|
||||
jobEvents,
|
||||
documentSettings: perDocumentSettings,
|
||||
authSessions,
|
||||
linkedAccounts,
|
||||
documents: userDocs,
|
||||
audiobooks: userAudiobooks,
|
||||
audiobookChapters: allChapters,
|
||||
getDocumentBlobStream: async (documentId: string) => getDocumentBlobStream(documentId, testNamespace),
|
||||
listAudiobookObjects: async (bookId: string, ownerId: string) => listAudiobookObjects(bookId, ownerId, testNamespace),
|
||||
getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) =>
|
||||
getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace),
|
||||
ttsSegmentEntries: segmentEntries,
|
||||
ttsSegmentVariants: segmentVariants,
|
||||
getDocumentBlobStream: async (documentId: string) => {
|
||||
requireStorage();
|
||||
return getDocumentBlobStream(documentId, testNamespace);
|
||||
},
|
||||
listAudiobookObjects: async (bookId: string, ownerId: string) => {
|
||||
requireStorage();
|
||||
return listAudiobookObjects(bookId, ownerId, testNamespace);
|
||||
},
|
||||
getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) => {
|
||||
requireStorage();
|
||||
return getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace);
|
||||
},
|
||||
getTtsSegmentAudioStream: async (audioKey: string) => {
|
||||
requireStorage();
|
||||
return (await getTtsSegmentAudioObjectStream(audioKey)).stream;
|
||||
},
|
||||
});
|
||||
|
||||
if (!req.signal.aborted) {
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ export const audiobookChapters = pgTable('audiobook_chapters', {
|
|||
// defined here. Only application-specific tables belong in this file.
|
||||
|
||||
export const userTtsChars = pgTable("user_tts_chars", {
|
||||
userId: text('user_id').notNull(),
|
||||
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
|
||||
date: date('date').notNull(),
|
||||
charCount: bigint('char_count', { mode: 'number' }).default(0),
|
||||
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
|
||||
|
|
@ -71,7 +71,7 @@ export const userTtsChars = pgTable("user_tts_chars", {
|
|||
// worker bounds each op by a hard cap, "ops created in the last hard-cap
|
||||
// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically.
|
||||
export const userJobEvents = pgTable('user_job_events', {
|
||||
userId: text('user_id').notNull(),
|
||||
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
|
||||
action: text('action').notNull(),
|
||||
opId: text('op_id').notNull(),
|
||||
createdAt: bigint('created_at', { mode: 'number' }).notNull().default(PG_NOW_MS),
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ export const audiobookChapters = sqliteTable('audiobook_chapters', {
|
|||
// defined here. Only application-specific tables belong in this file.
|
||||
|
||||
export const userTtsChars = sqliteTable("user_tts_chars", {
|
||||
userId: text('user_id').notNull(),
|
||||
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
|
||||
date: text('date').notNull(), // SQLite doesn't have native DATE type, text YYYY-MM-DD is standard
|
||||
charCount: integer('char_count').default(0),
|
||||
createdAt: integer('created_at').default(SQLITE_NOW_MS),
|
||||
|
|
@ -71,7 +71,7 @@ export const userTtsChars = sqliteTable("user_tts_chars", {
|
|||
// worker bounds each op by a hard cap, "ops created in the last hard-cap
|
||||
// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically.
|
||||
export const userJobEvents = sqliteTable('user_job_events', {
|
||||
userId: text('user_id').notNull(),
|
||||
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
|
||||
action: text('action').notNull(),
|
||||
opId: text('op_id').notNull(),
|
||||
createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS),
|
||||
|
|
|
|||
|
|
@ -255,7 +255,11 @@ export async function deleteAudiobookPrefix(prefix: string): Promise<number> {
|
|||
},
|
||||
}),
|
||||
);
|
||||
deleted += deleteRes.Deleted?.length ?? 0;
|
||||
const errors = deleteRes.Errors ?? [];
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Failed deleting ${errors.length} audiobook storage objects`);
|
||||
}
|
||||
deleted += keys.length;
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
|
|
|
|||
|
|
@ -96,8 +96,9 @@ const createAuth = () => betterAuth({
|
|||
context: { userIdHash: hashForLog(user.id) },
|
||||
error,
|
||||
});
|
||||
// Don't throw – allow the user deletion to proceed even if S3 cleanup fails.
|
||||
// Orphaned blobs are preferable to a blocked account deletion.
|
||||
// Without a durable cleanup queue, proceeding would permanently
|
||||
// orphan user-scoped storage and non-cascading database rows.
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -507,10 +507,10 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
|
|||
const parsedPrefix = `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/`;
|
||||
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })).catch(() => undefined);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined);
|
||||
await deleteDocumentPrefix(parsedPrefix).catch(() => undefined);
|
||||
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey }));
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey }));
|
||||
await deleteDocumentPrefix(parsedPrefix);
|
||||
await deleteDocumentPrefix(`${key}/`);
|
||||
}
|
||||
|
||||
export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> {
|
||||
|
|
@ -564,7 +564,11 @@ export async function deleteDocumentPrefix(prefix: string): Promise<number> {
|
|||
},
|
||||
}),
|
||||
);
|
||||
deleted += deleteRes.Deleted?.length ?? 0;
|
||||
const errors = deleteRes.Errors ?? [];
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Failed deleting ${errors.length} document storage objects`);
|
||||
}
|
||||
deleted += keys.length;
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import {
|
||||
enqueueDocumentPreview,
|
||||
} from '@/lib/server/documents/previews';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
|
||||
import type { BaseDocument, DocumentType } from '@/types/documents';
|
||||
|
||||
type RegisterUploadedDocumentInput = {
|
||||
|
|
@ -17,6 +19,23 @@ type RegisterUploadedDocumentInput = {
|
|||
};
|
||||
|
||||
export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> {
|
||||
const [existing] = await db
|
||||
.select({ lastModified: documents.lastModified })
|
||||
.from(documents)
|
||||
.where(and(
|
||||
eq(documents.id, input.documentId),
|
||||
eq(documents.userId, input.userId),
|
||||
))
|
||||
.limit(1);
|
||||
|
||||
if (existing && Number(existing.lastModified) !== input.lastModified) {
|
||||
await deleteDocumentTtsSegmentCache({
|
||||
userId: input.userId,
|
||||
documentId: input.documentId,
|
||||
namespace: input.namespace,
|
||||
});
|
||||
}
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
|
|
|
|||
|
|
@ -244,7 +244,12 @@ export async function deleteTtsSegmentPrefix(prefix: string): Promise<number> {
|
|||
},
|
||||
}),
|
||||
);
|
||||
deleted += deleteRes.Deleted?.length ?? 0;
|
||||
const errors = deleteRes.Errors ?? [];
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Failed deleting ${errors.length} TTS segment audio objects`);
|
||||
}
|
||||
// Quiet=true commonly omits Deleted entries on successful requests.
|
||||
deleted += keys.length;
|
||||
}
|
||||
|
||||
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import { and, eq } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema';
|
||||
import { deleteTtsSegmentAudioObjects } from '@/lib/server/tts/segments-blobstore';
|
||||
import {
|
||||
deleteTtsSegmentAudioObjects,
|
||||
deleteTtsSegmentPrefix,
|
||||
} from '@/lib/server/tts/segments-blobstore';
|
||||
import { buildTtsSegmentDocumentPrefix } from '@/lib/server/tts/segments';
|
||||
import { getS3Config } from '@/lib/server/storage/s3';
|
||||
import type { ReaderType } from '@/types/user-state';
|
||||
import { serverLogger } from '@/lib/server/logger';
|
||||
import { logDegraded } from '@/lib/server/errors/logging';
|
||||
|
|
@ -50,8 +55,6 @@ export async function clearTtsSegmentCache(
|
|||
)
|
||||
.where(and(...conditions))) as Array<{ segmentId: string; audioKey: string | null }>;
|
||||
|
||||
await db.delete(ttsSegmentEntries).where(and(...conditions));
|
||||
|
||||
const audioKeys = rows
|
||||
.map((row) => row.audioKey)
|
||||
.filter((key): key is string => Boolean(key));
|
||||
|
|
@ -80,10 +83,38 @@ export async function clearTtsSegmentCache(
|
|||
}
|
||||
}
|
||||
|
||||
// Keep metadata when storage cleanup is incomplete so a later retry still
|
||||
// knows which objects must be removed.
|
||||
if (!warning) {
|
||||
await db.delete(ttsSegmentEntries).where(and(...conditions));
|
||||
}
|
||||
|
||||
return {
|
||||
deletedSegments: rows.length,
|
||||
deletedSegments: warning ? 0 : rows.length,
|
||||
requestedAudioObjects: uniqueAudioKeys.length,
|
||||
deletedAudioObjects,
|
||||
...(warning ? { warning } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function deleteDocumentTtsSegmentCache(input: {
|
||||
userId: string;
|
||||
documentId: string;
|
||||
namespace: string | null;
|
||||
}): Promise<void> {
|
||||
const storagePrefix = getS3Config().prefix;
|
||||
for (const storageVersion of ['v1', 'v2'] as const) {
|
||||
await deleteTtsSegmentPrefix(buildTtsSegmentDocumentPrefix({
|
||||
storagePrefix,
|
||||
namespace: input.namespace,
|
||||
userId: input.userId,
|
||||
documentId: input.documentId,
|
||||
storageVersion,
|
||||
}));
|
||||
}
|
||||
|
||||
await db.delete(ttsSegmentEntries).where(and(
|
||||
eq(ttsSegmentEntries.userId, input.userId),
|
||||
eq(ttsSegmentEntries.documentId, input.documentId),
|
||||
));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -226,6 +226,17 @@ export function buildTtsSegmentAudioKey(input: {
|
|||
return `${input.storagePrefix}/tts_segments_v2/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`;
|
||||
}
|
||||
|
||||
export function buildTtsSegmentDocumentPrefix(input: {
|
||||
storagePrefix: string;
|
||||
namespace: string | null;
|
||||
userId: string;
|
||||
documentId: string;
|
||||
storageVersion?: 'v1' | 'v2';
|
||||
}): string {
|
||||
const nsSegment = input.namespace ? `ns/${input.namespace}/` : '';
|
||||
return `${input.storagePrefix}/tts_segments_${input.storageVersion ?? 'v2'}/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/`;
|
||||
}
|
||||
|
||||
export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise<number> {
|
||||
let workDir: string | null = null;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -5,10 +5,16 @@
|
|||
*/
|
||||
|
||||
import { db } from '@/db';
|
||||
import { documents, audiobooks } from '@/db/schema';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { documents, audiobooks, userJobEvents, userTtsChars } from '@/db/schema';
|
||||
import * as authSchemaSqlite from '@/db/schema_auth_sqlite';
|
||||
import * as authSchemaPostgres from '@/db/schema_auth_postgres';
|
||||
import { and, eq, ne } from 'drizzle-orm';
|
||||
import { getS3Config, isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { deleteDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import {
|
||||
deleteDocumentBlob,
|
||||
deleteDocumentPrefix,
|
||||
tempDocumentUploadPrefix,
|
||||
} from '@/lib/server/documents/blobstore';
|
||||
import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore';
|
||||
import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews';
|
||||
import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore';
|
||||
|
|
@ -34,9 +40,11 @@ export async function deleteUserStorageData(
|
|||
namespace: string | null,
|
||||
): Promise<void> {
|
||||
const s3Enabled = isS3Configured();
|
||||
const failures: unknown[] = [];
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const database = db as any;
|
||||
const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite;
|
||||
|
||||
// --- Documents & previews ---
|
||||
const userDocs: DocumentRow[] = await database
|
||||
|
|
@ -46,11 +54,22 @@ export async function deleteUserStorageData(
|
|||
|
||||
let docsDeleted = 0;
|
||||
for (const doc of userDocs) {
|
||||
if (s3Enabled) {
|
||||
const otherOwners = await database
|
||||
.select({ id: documents.id })
|
||||
.from(documents)
|
||||
.where(and(
|
||||
eq(documents.id, doc.id),
|
||||
ne(documents.userId, userId),
|
||||
))
|
||||
.limit(1);
|
||||
const isLastOwner = otherOwners.length === 0;
|
||||
|
||||
if (s3Enabled && isLastOwner) {
|
||||
try {
|
||||
await deleteDocumentBlob(doc.id, namespace);
|
||||
docsDeleted++;
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.document_blob_delete.failed',
|
||||
msg: 'Failed to delete document blob',
|
||||
|
|
@ -66,6 +85,7 @@ export async function deleteUserStorageData(
|
|||
try {
|
||||
await deleteDocumentPreviewArtifacts(doc.id, namespace);
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.document_preview_delete.failed',
|
||||
msg: 'Failed to delete preview artifacts',
|
||||
|
|
@ -79,20 +99,24 @@ export async function deleteUserStorageData(
|
|||
}
|
||||
}
|
||||
|
||||
// Always clean up DB rows — documentPreviews has no FK cascade on user.
|
||||
try {
|
||||
await deleteDocumentPreviewRows(doc.id, namespace);
|
||||
} catch (error) {
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.document_preview_rows_delete.failed',
|
||||
msg: 'Failed to delete preview rows',
|
||||
step: 'delete_document_preview_rows',
|
||||
context: {
|
||||
documentId: doc.id,
|
||||
userIdHash: hashForLog(userId),
|
||||
},
|
||||
error,
|
||||
});
|
||||
// Preview rows/artifacts are shared by document id, so only the final
|
||||
// owner may remove them.
|
||||
if (isLastOwner) {
|
||||
try {
|
||||
await deleteDocumentPreviewRows(doc.id, namespace);
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.document_preview_rows_delete.failed',
|
||||
msg: 'Failed to delete preview rows',
|
||||
step: 'delete_document_preview_rows',
|
||||
context: {
|
||||
documentId: doc.id,
|
||||
userIdHash: hashForLog(userId),
|
||||
},
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -111,6 +135,7 @@ export async function deleteUserStorageData(
|
|||
await deleteAudiobookPrefix(prefix);
|
||||
booksDeleted++;
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.audiobook_blobs_delete.failed',
|
||||
msg: 'Failed to delete audiobook blobs',
|
||||
|
|
@ -127,6 +152,19 @@ export async function deleteUserStorageData(
|
|||
// --- TTS segments ---
|
||||
let segmentsDeleted = 0;
|
||||
if (s3Enabled) {
|
||||
try {
|
||||
await deleteDocumentPrefix(tempDocumentUploadPrefix(userId, namespace));
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.temp_document_uploads_delete.failed',
|
||||
msg: 'Failed to delete temporary document uploads',
|
||||
step: 'delete_temp_document_upload_prefix',
|
||||
context: { userIdHash: hashForLog(userId) },
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const cfg = getS3Config();
|
||||
const nsSegment = namespace ? `ns/${namespace}/` : '';
|
||||
|
|
@ -135,6 +173,7 @@ export async function deleteUserStorageData(
|
|||
segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV1);
|
||||
segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV2);
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.tts_segments_delete.failed',
|
||||
msg: 'Failed to delete TTS segment blobs',
|
||||
|
|
@ -145,6 +184,34 @@ export async function deleteUserStorageData(
|
|||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
throw new AggregateError(failures, `User storage cleanup failed in ${failures.length} operation(s)`);
|
||||
}
|
||||
|
||||
// Namespaced cleanup is an additional storage pass made by the account
|
||||
// route. Database rows are global and must only be removed during the
|
||||
// canonical non-namespaced beforeDelete pass.
|
||||
if (namespace === null) {
|
||||
// Delete explicitly for compatibility with pre-cascade installations and
|
||||
// to remove auth verification tokens, which cannot carry a user FK.
|
||||
for (const { table, userColumn, step } of [
|
||||
{ table: userTtsChars, userColumn: userTtsChars.userId, step: 'delete_user_tts_usage_rows' },
|
||||
{ table: userJobEvents, userColumn: userJobEvents.userId, step: 'delete_user_job_event_rows' },
|
||||
{ table: authSchema.verification, userColumn: authSchema.verification.value, step: 'delete_user_verification_rows' },
|
||||
]) {
|
||||
await database.delete(table).where(eq(userColumn, userId)).catch((error: unknown) => {
|
||||
failures.push(error);
|
||||
logDegraded(serverLogger, {
|
||||
event: 'user.data_cleanup.db_rows_delete.failed',
|
||||
msg: 'Failed to delete non-cascading user database rows',
|
||||
step,
|
||||
context: { userIdHash: hashForLog(userId) },
|
||||
error,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) {
|
||||
serverLogger.info({
|
||||
event: 'user.data_cleanup.completed',
|
||||
|
|
@ -156,4 +223,8 @@ export async function deleteUserStorageData(
|
|||
segmentsDeleted,
|
||||
}, 'Completed user storage cleanup');
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
throw new AggregateError(failures, `User database cleanup failed in ${failures.length} operation(s)`);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ export type ExportBlobBody =
|
|||
| ArrayBufferView
|
||||
| { transformToByteArray: () => Promise<Uint8Array> };
|
||||
|
||||
type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list';
|
||||
type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list' | 'tts_segment';
|
||||
|
||||
type ExportIssue = {
|
||||
scope: ExportIssueScope;
|
||||
|
|
@ -41,6 +41,20 @@ type ExportAudiobookObject = {
|
|||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
type ExportTtsSegmentEntry = {
|
||||
segmentEntryId: string;
|
||||
documentId: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
type ExportTtsSegmentVariant = {
|
||||
segmentId: string;
|
||||
segmentEntryId: string;
|
||||
audioKey?: string | null;
|
||||
audioFormat?: string | null;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
export type AppendUserExportArchiveInput = {
|
||||
archive: Archiver;
|
||||
userId: string;
|
||||
|
|
@ -49,12 +63,19 @@ export type AppendUserExportArchiveInput = {
|
|||
preferences: unknown | null;
|
||||
readingHistory: unknown[];
|
||||
ttsUsage: unknown[];
|
||||
jobEvents: unknown[];
|
||||
documentSettings: unknown[];
|
||||
authSessions: unknown[];
|
||||
linkedAccounts: unknown[];
|
||||
documents: ExportDocument[];
|
||||
audiobooks: ExportAudiobook[];
|
||||
audiobookChapters: ExportAudiobookChapter[];
|
||||
ttsSegmentEntries: ExportTtsSegmentEntry[];
|
||||
ttsSegmentVariants: ExportTtsSegmentVariant[];
|
||||
getDocumentBlobStream: (documentId: string) => Promise<ExportBlobBody>;
|
||||
listAudiobookObjects: (bookId: string, userId: string) => Promise<ExportAudiobookObject[]>;
|
||||
getAudiobookObjectStream: (bookId: string, userId: string, fileName: string) => Promise<ExportBlobBody>;
|
||||
getTtsSegmentAudioStream: (audioKey: string) => Promise<ExportBlobBody>;
|
||||
};
|
||||
|
||||
function isNodeReadableStream(value: unknown): value is Readable {
|
||||
|
|
@ -124,17 +145,25 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
|
|||
preferences,
|
||||
readingHistory,
|
||||
ttsUsage,
|
||||
jobEvents,
|
||||
documentSettings,
|
||||
authSessions,
|
||||
linkedAccounts,
|
||||
documents,
|
||||
audiobooks,
|
||||
audiobookChapters,
|
||||
ttsSegmentEntries,
|
||||
ttsSegmentVariants,
|
||||
getDocumentBlobStream,
|
||||
listAudiobookObjects,
|
||||
getAudiobookObjectStream,
|
||||
getTtsSegmentAudioStream,
|
||||
} = input;
|
||||
|
||||
const issues: ExportIssue[] = [];
|
||||
let documentFilesExported = 0;
|
||||
let audiobookFilesExported = 0;
|
||||
let ttsSegmentFilesExported = 0;
|
||||
|
||||
appendJson(archive, 'profile.json', profileData);
|
||||
if (preferences) {
|
||||
|
|
@ -142,7 +171,13 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
|
|||
}
|
||||
appendJson(archive, 'reading_history.json', readingHistory);
|
||||
appendJson(archive, 'tts_usage.json', ttsUsage);
|
||||
appendJson(archive, 'job_events.json', jobEvents);
|
||||
appendJson(archive, 'document_settings.json', documentSettings);
|
||||
appendJson(archive, 'auth_sessions.json', authSessions);
|
||||
appendJson(archive, 'linked_accounts.json', linkedAccounts);
|
||||
appendJson(archive, 'library_documents.json', documents);
|
||||
appendJson(archive, 'tts_segment_entries.json', ttsSegmentEntries);
|
||||
appendJson(archive, 'tts_segment_variants.json', ttsSegmentVariants);
|
||||
|
||||
const chaptersByBookId = new Map<string, ExportAudiobookChapter[]>();
|
||||
for (const chapter of audiobookChapters) {
|
||||
|
|
@ -215,8 +250,40 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
|
|||
}
|
||||
}
|
||||
|
||||
const documentIdByEntryId = new Map(
|
||||
ttsSegmentEntries.map((entry) => [entry.segmentEntryId, entry.documentId]),
|
||||
);
|
||||
const exportedAudioKeys = new Set<string>();
|
||||
for (const variant of ttsSegmentVariants) {
|
||||
const audioKey = typeof variant.audioKey === 'string' ? variant.audioKey : '';
|
||||
if (!audioKey || exportedAudioKeys.has(audioKey)) continue;
|
||||
exportedAudioKeys.add(audioKey);
|
||||
|
||||
const documentId = toSafePathSegment(
|
||||
documentIdByEntryId.get(variant.segmentEntryId) ?? 'unknown-document',
|
||||
'unknown-document',
|
||||
);
|
||||
const segmentId = toSafePathSegment(variant.segmentId, 'segment');
|
||||
const format = toSafePathSegment(variant.audioFormat || 'mp3', 'mp3');
|
||||
const entryName = `files/tts_segments/${documentId}/${segmentId}.${format}`;
|
||||
|
||||
try {
|
||||
const body = await getTtsSegmentAudioStream(audioKey);
|
||||
const stream = await bodyToNodeReadable(body);
|
||||
archive.append(stream, { name: entryName });
|
||||
ttsSegmentFilesExported += 1;
|
||||
} catch (error) {
|
||||
issues.push({
|
||||
scope: 'tts_segment',
|
||||
id: variant.segmentId,
|
||||
fileName: entryName,
|
||||
message: normalizeErrorMessage(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const manifest = {
|
||||
formatVersion: 2,
|
||||
formatVersion: 3,
|
||||
exportedAtMs,
|
||||
userId,
|
||||
scope: 'owned',
|
||||
|
|
@ -224,14 +291,26 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
|
|||
documentsMetadata: documents.length,
|
||||
audiobooksMetadata: audiobooks.length,
|
||||
audiobookChaptersMetadata: audiobookChapters.length,
|
||||
documentSettingsMetadata: documentSettings.length,
|
||||
ttsSegmentEntriesMetadata: ttsSegmentEntries.length,
|
||||
ttsSegmentVariantsMetadata: ttsSegmentVariants.length,
|
||||
authSessionsMetadata: authSessions.length,
|
||||
linkedAccountsMetadata: linkedAccounts.length,
|
||||
jobEventsMetadata: jobEvents.length,
|
||||
documentFiles: documentFilesExported,
|
||||
audiobookFiles: audiobookFilesExported,
|
||||
ttsSegmentFiles: ttsSegmentFilesExported,
|
||||
issues: issues.length,
|
||||
},
|
||||
includes: {
|
||||
metadata: true,
|
||||
documentFiles: true,
|
||||
audiobookFiles: true,
|
||||
ttsSegmentFiles: true,
|
||||
credentialSecrets: false,
|
||||
temporaryUploads: false,
|
||||
derivedDocumentPreviews: false,
|
||||
derivedParsedDocuments: false,
|
||||
filesystemSources: false,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
63
tests/unit/data-export.vitest.spec.ts
Normal file
63
tests/unit/data-export.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import type { Archiver } from 'archiver';
|
||||
import { appendUserExportArchive } from '../../src/lib/server/user/data-export';
|
||||
|
||||
describe('user data export archive', () => {
|
||||
test('includes user metadata and cached TTS audio without credential secrets', async () => {
|
||||
const entries = new Map<string, unknown>();
|
||||
const archive = {
|
||||
append(value: unknown, options: { name: string }) {
|
||||
entries.set(options.name, value);
|
||||
return archive;
|
||||
},
|
||||
} as unknown as Archiver;
|
||||
|
||||
await appendUserExportArchive({
|
||||
archive,
|
||||
userId: 'user-1',
|
||||
exportedAtMs: 123,
|
||||
profileData: { user: { id: 'user-1', email: 'person@example.com' }, exportedAtMs: 123 },
|
||||
preferences: { dataJson: '{}' },
|
||||
readingHistory: [{ documentId: 'doc-1' }],
|
||||
ttsUsage: [{ charCount: 10 }],
|
||||
jobEvents: [{ action: 'pdf-layout' }],
|
||||
documentSettings: [{ documentId: 'doc-1', dataJson: '{}' }],
|
||||
authSessions: [{ id: 'session-1', ipAddress: null }],
|
||||
linkedAccounts: [{ id: 'account-1', providerId: 'credential' }],
|
||||
documents: [],
|
||||
audiobooks: [],
|
||||
audiobookChapters: [],
|
||||
ttsSegmentEntries: [{ segmentEntryId: 'entry-1', documentId: 'doc-1' }],
|
||||
ttsSegmentVariants: [{
|
||||
segmentId: 'segment-1',
|
||||
segmentEntryId: 'entry-1',
|
||||
audioKey: 'private/audio/key',
|
||||
audioFormat: 'mp3',
|
||||
}],
|
||||
getDocumentBlobStream: async () => new Uint8Array(),
|
||||
listAudiobookObjects: async () => [],
|
||||
getAudiobookObjectStream: async () => new Uint8Array(),
|
||||
getTtsSegmentAudioStream: async () => new Uint8Array([1, 2, 3]),
|
||||
});
|
||||
|
||||
expect(entries.has('document_settings.json')).toBe(true);
|
||||
expect(entries.has('auth_sessions.json')).toBe(true);
|
||||
expect(entries.has('linked_accounts.json')).toBe(true);
|
||||
expect(entries.has('tts_segment_entries.json')).toBe(true);
|
||||
expect(entries.has('tts_segment_variants.json')).toBe(true);
|
||||
expect(entries.has('files/tts_segments/doc-1/segment-1.mp3')).toBe(true);
|
||||
|
||||
const manifest = JSON.parse(String(entries.get('export_manifest.json')));
|
||||
expect(manifest).toMatchObject({
|
||||
formatVersion: 3,
|
||||
counts: {
|
||||
ttsSegmentEntriesMetadata: 1,
|
||||
ttsSegmentVariantsMetadata: 1,
|
||||
ttsSegmentFiles: 1,
|
||||
},
|
||||
includes: {
|
||||
credentialSecrets: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
50
tests/unit/storage-prefix-cleanup.vitest.spec.ts
Normal file
50
tests/unit/storage-prefix-cleanup.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
send: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/s3', () => ({
|
||||
getS3Config: () => ({ bucket: 'test-bucket', prefix: 'openreader-test' }),
|
||||
getS3Client: () => ({ send: mocks.send }),
|
||||
getS3ProxyClient: () => ({ send: mocks.send }),
|
||||
}));
|
||||
|
||||
import { deleteAudiobookPrefix } from '../../src/lib/server/audiobooks/blobstore';
|
||||
import { deleteDocumentPrefix } from '../../src/lib/server/documents/blobstore';
|
||||
|
||||
describe('storage prefix cleanup', () => {
|
||||
beforeEach(() => {
|
||||
mocks.send.mockReset();
|
||||
});
|
||||
|
||||
test.each([
|
||||
['document', deleteDocumentPrefix],
|
||||
['audiobook', deleteAudiobookPrefix],
|
||||
])('%s cleanup counts successful quiet deletes', async (_name, removePrefix) => {
|
||||
mocks.send
|
||||
.mockResolvedValueOnce({
|
||||
Contents: [{ Key: 'prefix/a' }, { Key: 'prefix/b' }],
|
||||
IsTruncated: false,
|
||||
})
|
||||
.mockResolvedValueOnce({});
|
||||
|
||||
await expect(removePrefix('prefix/')).resolves.toBe(2);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['document', deleteDocumentPrefix],
|
||||
['audiobook', deleteAudiobookPrefix],
|
||||
])('%s cleanup fails on per-object storage errors', async (_name, removePrefix) => {
|
||||
mocks.send
|
||||
.mockResolvedValueOnce({
|
||||
Contents: [{ Key: 'prefix/a' }],
|
||||
IsTruncated: false,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
Errors: [{ Key: 'prefix/a', Code: 'AccessDenied' }],
|
||||
});
|
||||
|
||||
await expect(removePrefix('prefix/')).rejects.toThrow('Failed deleting 1');
|
||||
});
|
||||
});
|
||||
45
tests/unit/tts-segments-blobstore.vitest.spec.ts
Normal file
45
tests/unit/tts-segments-blobstore.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
send: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/s3', () => ({
|
||||
getS3Config: () => ({ bucket: 'test-bucket' }),
|
||||
getS3Client: () => ({ send: mocks.send }),
|
||||
getS3ProxyClient: () => ({ send: mocks.send }),
|
||||
}));
|
||||
|
||||
import { deleteTtsSegmentPrefix } from '../../src/lib/server/tts/segments-blobstore';
|
||||
|
||||
describe('TTS segment blob cleanup', () => {
|
||||
beforeEach(() => {
|
||||
mocks.send.mockReset();
|
||||
});
|
||||
|
||||
test('counts successful quiet deletes by requested object count', async () => {
|
||||
mocks.send
|
||||
.mockResolvedValueOnce({
|
||||
Contents: [{ Key: 'prefix/a.mp3' }, { Key: 'prefix/b.mp3' }],
|
||||
IsTruncated: false,
|
||||
})
|
||||
.mockResolvedValueOnce({});
|
||||
|
||||
await expect(deleteTtsSegmentPrefix('prefix/')).resolves.toBe(2);
|
||||
});
|
||||
|
||||
test('fails cleanup when storage reports per-object deletion errors', async () => {
|
||||
mocks.send
|
||||
.mockResolvedValueOnce({
|
||||
Contents: [{ Key: 'prefix/a.mp3' }],
|
||||
IsTruncated: false,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
Errors: [{ Key: 'prefix/a.mp3', Code: 'AccessDenied' }],
|
||||
});
|
||||
|
||||
await expect(deleteTtsSegmentPrefix('prefix/')).rejects.toThrow(
|
||||
'Failed deleting 1 TTS segment audio objects',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import {
|
||||
buildTtsSegmentDocumentPrefix,
|
||||
buildProportionalAlignment,
|
||||
buildTtsSegmentEntryId,
|
||||
buildTtsSegmentId,
|
||||
|
|
@ -12,6 +13,15 @@ import {
|
|||
} from '../../src/lib/server/tts/segments';
|
||||
|
||||
describe('tts segment helpers', () => {
|
||||
test('builds a user/document-scoped audio prefix across every version and variant', () => {
|
||||
expect(buildTtsSegmentDocumentPrefix({
|
||||
storagePrefix: 'openreader',
|
||||
namespace: 'test namespace',
|
||||
userId: 'user/name',
|
||||
documentId: 'doc-id',
|
||||
})).toBe('openreader/tts_segments_v2/ns/test namespace/users/user%2Fname/docs/doc-id/');
|
||||
});
|
||||
|
||||
test('builds stable settings hash', () => {
|
||||
const a = buildTtsSegmentSettingsHash({
|
||||
providerRef: 'openai',
|
||||
|
|
|
|||
115
tests/unit/user-data-cleanup.vitest.spec.ts
Normal file
115
tests/unit/user-data-cleanup.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
selectResults: [] as unknown[][],
|
||||
deleteWhere: vi.fn(async () => undefined),
|
||||
deleteDocumentBlob: vi.fn(async () => undefined),
|
||||
deleteDocumentPrefix: vi.fn(async () => 0),
|
||||
deleteDocumentPreviewArtifacts: vi.fn(async () => 0),
|
||||
deleteDocumentPreviewRows: vi.fn(async () => undefined),
|
||||
deleteAudiobookPrefix: vi.fn(async () => 0),
|
||||
deleteTtsSegmentPrefix: vi.fn(async () => 0),
|
||||
}));
|
||||
|
||||
function resultBuilder(result: unknown[]) {
|
||||
return {
|
||||
limit: async () => result,
|
||||
then: (resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) =>
|
||||
Promise.resolve(result).then(resolve, reject),
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock('@/db', () => ({
|
||||
db: {
|
||||
select: vi.fn(() => ({
|
||||
from: vi.fn(() => ({
|
||||
where: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])),
|
||||
})),
|
||||
})),
|
||||
delete: vi.fn(() => ({
|
||||
where: mocks.deleteWhere,
|
||||
})),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/s3', () => ({
|
||||
isS3Configured: () => true,
|
||||
getS3Config: () => ({ prefix: 'openreader-test' }),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||
deleteDocumentBlob: mocks.deleteDocumentBlob,
|
||||
deleteDocumentPrefix: mocks.deleteDocumentPrefix,
|
||||
tempDocumentUploadPrefix: () => 'temp/user/',
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/previews-blobstore', () => ({
|
||||
deleteDocumentPreviewArtifacts: mocks.deleteDocumentPreviewArtifacts,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/previews', () => ({
|
||||
deleteDocumentPreviewRows: mocks.deleteDocumentPreviewRows,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/audiobooks/blobstore', () => ({
|
||||
audiobookPrefix: () => 'audiobooks/user/',
|
||||
deleteAudiobookPrefix: mocks.deleteAudiobookPrefix,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/tts/segments-blobstore', () => ({
|
||||
deleteTtsSegmentPrefix: mocks.deleteTtsSegmentPrefix,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/logger', () => ({
|
||||
hashForLog: () => 'hash',
|
||||
serverLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/errors/logging', () => ({
|
||||
logDegraded: vi.fn(),
|
||||
}));
|
||||
|
||||
import { deleteUserStorageData } from '../../src/lib/server/user/data-cleanup';
|
||||
|
||||
describe('user data cleanup', () => {
|
||||
beforeEach(() => {
|
||||
mocks.selectResults = [];
|
||||
for (const mock of Object.values(mocks)) {
|
||||
if (typeof mock === 'function' && 'mockReset' in mock) {
|
||||
mock.mockReset();
|
||||
}
|
||||
}
|
||||
mocks.deleteWhere.mockResolvedValue(undefined);
|
||||
mocks.deleteDocumentBlob.mockResolvedValue(undefined);
|
||||
mocks.deleteDocumentPrefix.mockResolvedValue(0);
|
||||
mocks.deleteDocumentPreviewArtifacts.mockResolvedValue(0);
|
||||
mocks.deleteDocumentPreviewRows.mockResolvedValue(undefined);
|
||||
mocks.deleteAudiobookPrefix.mockResolvedValue(0);
|
||||
mocks.deleteTtsSegmentPrefix.mockResolvedValue(0);
|
||||
});
|
||||
|
||||
test('keeps shared document blobs and previews', async () => {
|
||||
mocks.selectResults = [
|
||||
[{ id: 'shared-doc' }],
|
||||
[{ id: 'shared-doc' }],
|
||||
[],
|
||||
];
|
||||
|
||||
await deleteUserStorageData('user-1', null);
|
||||
|
||||
expect(mocks.deleteDocumentBlob).not.toHaveBeenCalled();
|
||||
expect(mocks.deleteDocumentPreviewArtifacts).not.toHaveBeenCalled();
|
||||
expect(mocks.deleteDocumentPreviewRows).not.toHaveBeenCalled();
|
||||
expect(mocks.deleteWhere).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
test('blocks database cleanup when storage cleanup fails', async () => {
|
||||
mocks.selectResults = [[], []];
|
||||
mocks.deleteDocumentPrefix.mockRejectedValueOnce(new Error('storage unavailable'));
|
||||
|
||||
await expect(deleteUserStorageData('user-1', null)).rejects.toThrow(
|
||||
'User storage cleanup failed',
|
||||
);
|
||||
expect(mocks.deleteWhere).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue