refactor(user): overhaul user data cleanup and export for cascading deletes and TTS segment support

Revise user data cleanup logic to ensure proper cascading deletion of user-related
database rows and S3 objects, including shared document and preview artifacts.
Introduce explicit checks for last ownership before removing shared resources.
Add TTS segment cache and audio cleanup to document and user deletion flows.
Expand user data export to include TTS segment entries and audio, job events,
document settings, and linked auth sessions. Update schema with ON DELETE CASCADE
for userTtsChars and userJobEvents. Add new migration scripts and comprehensive
unit tests for cleanup and export scenarios.
This commit is contained in:
Richard R 2026-06-06 16:37:28 -06:00
parent 92df3f907c
commit 76553023e8
27 changed files with 4425 additions and 66 deletions

View file

@ -0,0 +1,8 @@
DELETE FROM "user_job_events" WHERE NOT EXISTS (
SELECT 1 FROM "user" WHERE "user"."id" = "user_job_events"."user_id"
);--> statement-breakpoint
DELETE FROM "user_tts_chars" WHERE NOT EXISTS (
SELECT 1 FROM "user" WHERE "user"."id" = "user_tts_chars"."user_id"
);--> statement-breakpoint
ALTER TABLE "user_job_events" ADD CONSTRAINT "user_job_events_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "user_tts_chars" ADD CONSTRAINT "user_tts_chars_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;

File diff suppressed because it is too large Load diff

View file

@ -71,6 +71,13 @@
"when": 1780625663880, "when": 1780625663880,
"tag": "0009_drop_pdf_parse", "tag": "0009_drop_pdf_parse",
"breakpoints": true "breakpoints": true
},
{
"idx": 10,
"version": "7",
"when": 1780785284335,
"tag": "0010_user-data-cleanup-cascades",
"breakpoints": true
} }
] ]
} }

View file

@ -0,0 +1,35 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
DELETE FROM `user_job_events` WHERE NOT EXISTS (
SELECT 1 FROM `user` WHERE `user`.`id` = `user_job_events`.`user_id`
);--> statement-breakpoint
DELETE FROM `user_tts_chars` WHERE NOT EXISTS (
SELECT 1 FROM `user` WHERE `user`.`id` = `user_tts_chars`.`user_id`
);--> statement-breakpoint
CREATE TABLE `__new_user_job_events` (
`user_id` text NOT NULL,
`action` text NOT NULL,
`op_id` text NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
PRIMARY KEY(`user_id`, `action`, `op_id`),
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
INSERT INTO `__new_user_job_events`("user_id", "action", "op_id", "created_at") SELECT "user_id", "action", "op_id", "created_at" FROM `user_job_events`;--> statement-breakpoint
DROP TABLE `user_job_events`;--> statement-breakpoint
ALTER TABLE `__new_user_job_events` RENAME TO `user_job_events`;--> statement-breakpoint
PRAGMA foreign_keys=ON;--> statement-breakpoint
CREATE INDEX `idx_user_job_events_user_action_created` ON `user_job_events` (`user_id`,`action`,`created_at`);--> statement-breakpoint
CREATE TABLE `__new_user_tts_chars` (
`user_id` text NOT NULL,
`date` text NOT NULL,
`char_count` integer DEFAULT 0,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
PRIMARY KEY(`user_id`, `date`),
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
INSERT INTO `__new_user_tts_chars`("user_id", "date", "char_count", "created_at", "updated_at") SELECT "user_id", "date", "char_count", "created_at", "updated_at" FROM `user_tts_chars`;--> statement-breakpoint
DROP TABLE `user_tts_chars`;--> statement-breakpoint
ALTER TABLE `__new_user_tts_chars` RENAME TO `user_tts_chars`;--> statement-breakpoint
CREATE INDEX `idx_user_tts_chars_date` ON `user_tts_chars` (`date`);

File diff suppressed because it is too large Load diff

View file

@ -71,6 +71,13 @@
"when": 1780625663601, "when": 1780625663601,
"tag": "0009_drop_pdf_parse", "tag": "0009_drop_pdf_parse",
"breakpoints": true "breakpoints": true
},
{
"idx": 10,
"version": "6",
"when": 1780785284041,
"tag": "0010_user-data-cleanup-cascades",
"breakpoints": true
} }
] ]
} }

View file

@ -3,7 +3,7 @@ import { NextResponse } from 'next/server';
import { auth } from '@/lib/server/auth/auth'; import { auth } from '@/lib/server/auth/auth';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { deleteUserStorageData } from '@/lib/server/user/data-cleanup'; import { deleteUserStorageData } from '@/lib/server/user/data-cleanup';
import { errorToLog, hashForLog, serverLogger } from '@/lib/server/logger'; import { errorToLog, serverLogger } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response'; import { errorResponse } from '@/lib/server/errors/next-response';
export async function DELETE() { export async function DELETE() {
@ -18,21 +18,12 @@ export async function DELETE() {
} }
try { try {
// Best-effort cleanup for test namespaced storage using request context. // Clean test-namespaced storage using request context. The Better Auth
// The Better Auth beforeDelete hook still runs and handles non-namespaced data. // beforeDelete hook handles non-namespaced storage and blocks deletion if
// either cleanup cannot complete.
const testNamespace = getOpenReaderTestNamespace(reqHeaders); const testNamespace = getOpenReaderTestNamespace(reqHeaders);
if (testNamespace) { if (testNamespace) {
try { await deleteUserStorageData(session.user.id, testNamespace);
await deleteUserStorageData(session.user.id, testNamespace);
} catch (error) {
serverLogger.warn({
event: 'account.delete.storage_cleanup_failed',
degraded: true,
step: 'namespaced_storage_cleanup',
userIdHash: hashForLog(session.user.id),
error: errorToLog(error),
}, 'Failed to clean up namespaced user storage before deletion');
}
} }
// Use Better Auth's built-in deleteUser to handle cascading cleanup // Use Better Auth's built-in deleteUser to handle cascading cleanup

View file

@ -9,6 +9,7 @@ import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'
import { isS3Configured } from '@/lib/server/storage/s3'; import { isS3Configured } from '@/lib/server/storage/s3';
import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorToLog, serverLogger } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response'; import { errorResponse } from '@/lib/server/errors/next-response';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
@ -80,9 +81,14 @@ export async function GET(req: NextRequest) {
}); });
} catch (error) { } catch (error) {
if (isMissingBlobError(error)) { if (isMissingBlobError(error)) {
await deleteDocumentTtsSegmentCache({
userId: doc.userId,
documentId: id,
namespace: testNamespace,
});
await db await db
.delete(documents) .delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); .where(and(eq(documents.id, id), eq(documents.userId, doc.userId)));
return NextResponse.json({ error: 'Not found' }, { status: 404 }); return NextResponse.json({ error: 'Not found' }, { status: 404 });
} }
throw error; throw error;

View file

@ -22,6 +22,7 @@ import {
import { extractRawTextSnippet } from '@/lib/server/documents/text-snippets'; import { extractRawTextSnippet } from '@/lib/server/documents/text-snippets';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3'; import { isS3Configured } from '@/lib/server/storage/s3';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
@ -108,9 +109,14 @@ export async function GET(req: NextRequest) {
); );
} catch (error) { } catch (error) {
if (isMissingDocumentBlobError(error)) { if (isMissingDocumentBlobError(error)) {
await deleteDocumentTtsSegmentCache({
userId: doc.userId,
documentId: id,
namespace: testNamespace,
});
await db await db
.delete(documents) .delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds))); .where(and(eq(documents.id, id), eq(documents.userId, doc.userId)));
return NextResponse.json({ error: 'Not found' }, { status: 404 }); return NextResponse.json({ error: 'Not found' }, { status: 404 });
} }
throw error; throw error;

View file

@ -13,6 +13,7 @@ import {
import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3'; import { isS3Configured } from '@/lib/server/storage/s3';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
import type { BaseDocument, DocumentType } from '@/types/documents'; import type { BaseDocument, DocumentType } from '@/types/documents';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
@ -136,6 +137,25 @@ export async function DELETE(req: NextRequest) {
return NextResponse.json({ success: true, deleted: 0 }); return NextResponse.json({ success: true, deleted: 0 });
} }
const ownedRows = (await db
.select({ id: documents.id, userId: documents.userId })
.from(documents)
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))) as Array<{
id: string;
userId: string;
}>;
// TTS audio is user-scoped even when the underlying document blob is
// shared. Remove it before deleting ownership metadata so failures remain
// retryable and cannot create untraceable S3 objects.
for (const row of ownedRows) {
await deleteDocumentTtsSegmentCache({
userId: row.userId,
documentId: row.id,
namespace: testNamespace,
});
}
const deletedRows = (await db const deletedRows = (await db
.delete(documents) .delete(documents)
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds))) .where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))

View file

@ -2,12 +2,26 @@ import { NextRequest, NextResponse } from 'next/server';
import { PassThrough, Readable } from 'stream'; import { PassThrough, Readable } from 'stream';
import { auth } from '@/lib/server/auth/auth'; import { auth } from '@/lib/server/auth/auth';
import { db } from '@/db'; import { db } from '@/db';
import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress, userTtsChars } from '@/db/schema'; import {
documents,
audiobooks,
audiobookChapters,
documentSettings,
ttsSegmentEntries,
ttsSegmentVariants,
userDocumentProgress,
userJobEvents,
userPreferences,
userTtsChars,
} from '@/db/schema';
import * as authSchemaSqlite from '@/db/schema_auth_sqlite';
import * as authSchemaPostgres from '@/db/schema_auth_postgres';
import { and, desc, eq, inArray } from 'drizzle-orm'; import { and, desc, eq, inArray } from 'drizzle-orm';
import archiver from 'archiver'; import archiver from 'archiver';
import { appendUserExportArchive } from '@/lib/server/user/data-export'; import { appendUserExportArchive } from '@/lib/server/user/data-export';
import { getDocumentBlobStream } from '@/lib/server/documents/blobstore'; import { getDocumentBlobStream } from '@/lib/server/documents/blobstore';
import { getAudiobookObjectStream, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore'; import { getAudiobookObjectStream, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore';
import { getTtsSegmentAudioObjectStream } from '@/lib/server/tts/segments-blobstore';
import { isS3Configured } from '@/lib/server/storage/s3'; import { isS3Configured } from '@/lib/server/storage/s3';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { nowTimestampMs } from '@/lib/shared/timestamps'; import { nowTimestampMs } from '@/lib/shared/timestamps';
@ -17,13 +31,6 @@ import { errorResponse } from '@/lib/server/errors/next-response';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'Export storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
if (!auth) { if (!auth) {
return errorResponse(new Error('Auth not initialized'), { return errorResponse(new Error('Auth not initialized'), {
apiErrorMessage: 'Auth not initialized', apiErrorMessage: 'Auth not initialized',
@ -41,11 +48,21 @@ export async function GET(req: NextRequest) {
const userId = session.user.id; const userId = session.user.id;
const testNamespace = getOpenReaderTestNamespace(req.headers); const testNamespace = getOpenReaderTestNamespace(req.headers);
const storageEnabled = isS3Configured();
const requireStorage = () => {
if (!storageEnabled) {
throw new Error('Storage is not configured; file content could not be exported');
}
};
const [ const [
prefs, prefs,
progress, progress,
ttsUsage, ttsUsage,
jobEvents,
perDocumentSettings,
segmentEntries,
segmentVariants,
userDocs, userDocs,
userAudiobooks, userAudiobooks,
] = await Promise.all([ ] = await Promise.all([
@ -60,6 +77,26 @@ export async function GET(req: NextRequest) {
.from(userTtsChars) .from(userTtsChars)
.where(eq(userTtsChars.userId, userId)) .where(eq(userTtsChars.userId, userId))
.orderBy(desc(userTtsChars.date)), .orderBy(desc(userTtsChars.date)),
db
.select()
.from(userJobEvents)
.where(eq(userJobEvents.userId, userId))
.orderBy(desc(userJobEvents.createdAt)),
db
.select()
.from(documentSettings)
.where(eq(documentSettings.userId, userId))
.orderBy(desc(documentSettings.updatedAt)),
db
.select()
.from(ttsSegmentEntries)
.where(eq(ttsSegmentEntries.userId, userId))
.orderBy(desc(ttsSegmentEntries.updatedAt)),
db
.select()
.from(ttsSegmentVariants)
.where(eq(ttsSegmentVariants.userId, userId))
.orderBy(desc(ttsSegmentVariants.updatedAt)),
db db
.select() .select()
.from(documents) .from(documents)
@ -72,6 +109,36 @@ export async function GET(req: NextRequest) {
.orderBy(desc(audiobooks.createdAt)), .orderBy(desc(audiobooks.createdAt)),
]); ]);
const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite;
// Auth exports intentionally select metadata only. Credential and session
// secrets must never be written into the archive.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const database = db as any;
const [authSessions, linkedAccounts] = await Promise.all([
database
.select({
id: authSchema.session.id,
expiresAt: authSchema.session.expiresAt,
createdAt: authSchema.session.createdAt,
updatedAt: authSchema.session.updatedAt,
ipAddress: authSchema.session.ipAddress,
userAgent: authSchema.session.userAgent,
})
.from(authSchema.session)
.where(eq(authSchema.session.userId, userId)),
database
.select({
id: authSchema.account.id,
accountId: authSchema.account.accountId,
providerId: authSchema.account.providerId,
scope: authSchema.account.scope,
createdAt: authSchema.account.createdAt,
updatedAt: authSchema.account.updatedAt,
})
.from(authSchema.account)
.where(eq(authSchema.account.userId, userId)),
]);
const archive = archiver('zip', { const archive = archiver('zip', {
zlib: { level: 0 }, zlib: { level: 0 },
forceZip64: true, forceZip64: true,
@ -114,7 +181,6 @@ export async function GET(req: NextRequest) {
const exportedAtMs = nowTimestampMs(); const exportedAtMs = nowTimestampMs();
const profileData = { const profileData = {
user: session.user, user: session.user,
session: session.session,
exportedAtMs, exportedAtMs,
}; };
@ -126,13 +192,31 @@ export async function GET(req: NextRequest) {
preferences: prefs[0] ?? null, preferences: prefs[0] ?? null,
readingHistory: progress, readingHistory: progress,
ttsUsage, ttsUsage,
jobEvents,
documentSettings: perDocumentSettings,
authSessions,
linkedAccounts,
documents: userDocs, documents: userDocs,
audiobooks: userAudiobooks, audiobooks: userAudiobooks,
audiobookChapters: allChapters, audiobookChapters: allChapters,
getDocumentBlobStream: async (documentId: string) => getDocumentBlobStream(documentId, testNamespace), ttsSegmentEntries: segmentEntries,
listAudiobookObjects: async (bookId: string, ownerId: string) => listAudiobookObjects(bookId, ownerId, testNamespace), ttsSegmentVariants: segmentVariants,
getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) => getDocumentBlobStream: async (documentId: string) => {
getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace), requireStorage();
return getDocumentBlobStream(documentId, testNamespace);
},
listAudiobookObjects: async (bookId: string, ownerId: string) => {
requireStorage();
return listAudiobookObjects(bookId, ownerId, testNamespace);
},
getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) => {
requireStorage();
return getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace);
},
getTtsSegmentAudioStream: async (audioKey: string) => {
requireStorage();
return (await getTtsSegmentAudioObjectStream(audioKey)).stream;
},
}); });
if (!req.signal.aborted) { if (!req.signal.aborted) {

View file

@ -54,7 +54,7 @@ export const audiobookChapters = pgTable('audiobook_chapters', {
// defined here. Only application-specific tables belong in this file. // defined here. Only application-specific tables belong in this file.
export const userTtsChars = pgTable("user_tts_chars", { export const userTtsChars = pgTable("user_tts_chars", {
userId: text('user_id').notNull(), userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
date: date('date').notNull(), date: date('date').notNull(),
charCount: bigint('char_count', { mode: 'number' }).default(0), charCount: bigint('char_count', { mode: 'number' }).default(0),
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS), createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
@ -71,7 +71,7 @@ export const userTtsChars = pgTable("user_tts_chars", {
// worker bounds each op by a hard cap, "ops created in the last hard-cap // worker bounds each op by a hard cap, "ops created in the last hard-cap
// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically. // window" is an upper bound on in-flight ops. Old rows are pruned opportunistically.
export const userJobEvents = pgTable('user_job_events', { export const userJobEvents = pgTable('user_job_events', {
userId: text('user_id').notNull(), userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
action: text('action').notNull(), action: text('action').notNull(),
opId: text('op_id').notNull(), opId: text('op_id').notNull(),
createdAt: bigint('created_at', { mode: 'number' }).notNull().default(PG_NOW_MS), createdAt: bigint('created_at', { mode: 'number' }).notNull().default(PG_NOW_MS),

View file

@ -54,7 +54,7 @@ export const audiobookChapters = sqliteTable('audiobook_chapters', {
// defined here. Only application-specific tables belong in this file. // defined here. Only application-specific tables belong in this file.
export const userTtsChars = sqliteTable("user_tts_chars", { export const userTtsChars = sqliteTable("user_tts_chars", {
userId: text('user_id').notNull(), userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
date: text('date').notNull(), // SQLite doesn't have native DATE type, text YYYY-MM-DD is standard date: text('date').notNull(), // SQLite doesn't have native DATE type, text YYYY-MM-DD is standard
charCount: integer('char_count').default(0), charCount: integer('char_count').default(0),
createdAt: integer('created_at').default(SQLITE_NOW_MS), createdAt: integer('created_at').default(SQLITE_NOW_MS),
@ -71,7 +71,7 @@ export const userTtsChars = sqliteTable("user_tts_chars", {
// worker bounds each op by a hard cap, "ops created in the last hard-cap // worker bounds each op by a hard cap, "ops created in the last hard-cap
// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically. // window" is an upper bound on in-flight ops. Old rows are pruned opportunistically.
export const userJobEvents = sqliteTable('user_job_events', { export const userJobEvents = sqliteTable('user_job_events', {
userId: text('user_id').notNull(), userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
action: text('action').notNull(), action: text('action').notNull(),
opId: text('op_id').notNull(), opId: text('op_id').notNull(),
createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS), createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS),

View file

@ -255,7 +255,11 @@ export async function deleteAudiobookPrefix(prefix: string): Promise<number> {
}, },
}), }),
); );
deleted += deleteRes.Deleted?.length ?? 0; const errors = deleteRes.Errors ?? [];
if (errors.length > 0) {
throw new Error(`Failed deleting ${errors.length} audiobook storage objects`);
}
deleted += keys.length;
} }
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;

View file

@ -96,8 +96,9 @@ const createAuth = () => betterAuth({
context: { userIdHash: hashForLog(user.id) }, context: { userIdHash: hashForLog(user.id) },
error, error,
}); });
// Don't throw allow the user deletion to proceed even if S3 cleanup fails. // Without a durable cleanup queue, proceeding would permanently
// Orphaned blobs are preferable to a blocked account deletion. // orphan user-scoped storage and non-cascading database rows.
throw error;
} }
}, },
}, },

View file

@ -507,10 +507,10 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
const parsedPrefix = `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/`; const parsedPrefix = `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/`;
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key })); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })).catch(() => undefined); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey }));
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey }));
await deleteDocumentPrefix(parsedPrefix).catch(() => undefined); await deleteDocumentPrefix(parsedPrefix);
await deleteDocumentPrefix(`${key}/`).catch(() => undefined); await deleteDocumentPrefix(`${key}/`);
} }
export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> { export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> {
@ -564,7 +564,11 @@ export async function deleteDocumentPrefix(prefix: string): Promise<number> {
}, },
}), }),
); );
deleted += deleteRes.Deleted?.length ?? 0; const errors = deleteRes.Errors ?? [];
if (errors.length > 0) {
throw new Error(`Failed deleting ${errors.length} document storage objects`);
}
deleted += keys.length;
} }
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;

View file

@ -1,9 +1,11 @@
import { db } from '@/db'; import { db } from '@/db';
import { documents } from '@/db/schema'; import { documents } from '@/db/schema';
import { and, eq } from 'drizzle-orm';
import { import {
enqueueDocumentPreview, enqueueDocumentPreview,
} from '@/lib/server/documents/previews'; } from '@/lib/server/documents/previews';
import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorToLog, serverLogger } from '@/lib/server/logger';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
import type { BaseDocument, DocumentType } from '@/types/documents'; import type { BaseDocument, DocumentType } from '@/types/documents';
type RegisterUploadedDocumentInput = { type RegisterUploadedDocumentInput = {
@ -17,6 +19,23 @@ type RegisterUploadedDocumentInput = {
}; };
export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> { export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> {
const [existing] = await db
.select({ lastModified: documents.lastModified })
.from(documents)
.where(and(
eq(documents.id, input.documentId),
eq(documents.userId, input.userId),
))
.limit(1);
if (existing && Number(existing.lastModified) !== input.lastModified) {
await deleteDocumentTtsSegmentCache({
userId: input.userId,
documentId: input.documentId,
namespace: input.namespace,
});
}
await db await db
.insert(documents) .insert(documents)
.values({ .values({

View file

@ -244,7 +244,12 @@ export async function deleteTtsSegmentPrefix(prefix: string): Promise<number> {
}, },
}), }),
); );
deleted += deleteRes.Deleted?.length ?? 0; const errors = deleteRes.Errors ?? [];
if (errors.length > 0) {
throw new Error(`Failed deleting ${errors.length} TTS segment audio objects`);
}
// Quiet=true commonly omits Deleted entries on successful requests.
deleted += keys.length;
} }
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;

View file

@ -1,7 +1,12 @@
import { and, eq } from 'drizzle-orm'; import { and, eq } from 'drizzle-orm';
import { db } from '@/db'; import { db } from '@/db';
import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema';
import { deleteTtsSegmentAudioObjects } from '@/lib/server/tts/segments-blobstore'; import {
deleteTtsSegmentAudioObjects,
deleteTtsSegmentPrefix,
} from '@/lib/server/tts/segments-blobstore';
import { buildTtsSegmentDocumentPrefix } from '@/lib/server/tts/segments';
import { getS3Config } from '@/lib/server/storage/s3';
import type { ReaderType } from '@/types/user-state'; import type { ReaderType } from '@/types/user-state';
import { serverLogger } from '@/lib/server/logger'; import { serverLogger } from '@/lib/server/logger';
import { logDegraded } from '@/lib/server/errors/logging'; import { logDegraded } from '@/lib/server/errors/logging';
@ -50,8 +55,6 @@ export async function clearTtsSegmentCache(
) )
.where(and(...conditions))) as Array<{ segmentId: string; audioKey: string | null }>; .where(and(...conditions))) as Array<{ segmentId: string; audioKey: string | null }>;
await db.delete(ttsSegmentEntries).where(and(...conditions));
const audioKeys = rows const audioKeys = rows
.map((row) => row.audioKey) .map((row) => row.audioKey)
.filter((key): key is string => Boolean(key)); .filter((key): key is string => Boolean(key));
@ -80,10 +83,38 @@ export async function clearTtsSegmentCache(
} }
} }
// Keep metadata when storage cleanup is incomplete so a later retry still
// knows which objects must be removed.
if (!warning) {
await db.delete(ttsSegmentEntries).where(and(...conditions));
}
return { return {
deletedSegments: rows.length, deletedSegments: warning ? 0 : rows.length,
requestedAudioObjects: uniqueAudioKeys.length, requestedAudioObjects: uniqueAudioKeys.length,
deletedAudioObjects, deletedAudioObjects,
...(warning ? { warning } : {}), ...(warning ? { warning } : {}),
}; };
} }
export async function deleteDocumentTtsSegmentCache(input: {
userId: string;
documentId: string;
namespace: string | null;
}): Promise<void> {
const storagePrefix = getS3Config().prefix;
for (const storageVersion of ['v1', 'v2'] as const) {
await deleteTtsSegmentPrefix(buildTtsSegmentDocumentPrefix({
storagePrefix,
namespace: input.namespace,
userId: input.userId,
documentId: input.documentId,
storageVersion,
}));
}
await db.delete(ttsSegmentEntries).where(and(
eq(ttsSegmentEntries.userId, input.userId),
eq(ttsSegmentEntries.documentId, input.documentId),
));
}

View file

@ -226,6 +226,17 @@ export function buildTtsSegmentAudioKey(input: {
return `${input.storagePrefix}/tts_segments_v2/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`; return `${input.storagePrefix}/tts_segments_v2/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`;
} }
export function buildTtsSegmentDocumentPrefix(input: {
storagePrefix: string;
namespace: string | null;
userId: string;
documentId: string;
storageVersion?: 'v1' | 'v2';
}): string {
const nsSegment = input.namespace ? `ns/${input.namespace}/` : '';
return `${input.storagePrefix}/tts_segments_${input.storageVersion ?? 'v2'}/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/`;
}
export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise<number> { export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise<number> {
let workDir: string | null = null; let workDir: string | null = null;
try { try {

View file

@ -5,10 +5,16 @@
*/ */
import { db } from '@/db'; import { db } from '@/db';
import { documents, audiobooks } from '@/db/schema'; import { documents, audiobooks, userJobEvents, userTtsChars } from '@/db/schema';
import { eq } from 'drizzle-orm'; import * as authSchemaSqlite from '@/db/schema_auth_sqlite';
import * as authSchemaPostgres from '@/db/schema_auth_postgres';
import { and, eq, ne } from 'drizzle-orm';
import { getS3Config, isS3Configured } from '@/lib/server/storage/s3'; import { getS3Config, isS3Configured } from '@/lib/server/storage/s3';
import { deleteDocumentBlob } from '@/lib/server/documents/blobstore'; import {
deleteDocumentBlob,
deleteDocumentPrefix,
tempDocumentUploadPrefix,
} from '@/lib/server/documents/blobstore';
import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore';
import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews'; import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews';
import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore'; import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore';
@ -34,9 +40,11 @@ export async function deleteUserStorageData(
namespace: string | null, namespace: string | null,
): Promise<void> { ): Promise<void> {
const s3Enabled = isS3Configured(); const s3Enabled = isS3Configured();
const failures: unknown[] = [];
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
const database = db as any; const database = db as any;
const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite;
// --- Documents & previews --- // --- Documents & previews ---
const userDocs: DocumentRow[] = await database const userDocs: DocumentRow[] = await database
@ -46,11 +54,22 @@ export async function deleteUserStorageData(
let docsDeleted = 0; let docsDeleted = 0;
for (const doc of userDocs) { for (const doc of userDocs) {
if (s3Enabled) { const otherOwners = await database
.select({ id: documents.id })
.from(documents)
.where(and(
eq(documents.id, doc.id),
ne(documents.userId, userId),
))
.limit(1);
const isLastOwner = otherOwners.length === 0;
if (s3Enabled && isLastOwner) {
try { try {
await deleteDocumentBlob(doc.id, namespace); await deleteDocumentBlob(doc.id, namespace);
docsDeleted++; docsDeleted++;
} catch (error) { } catch (error) {
failures.push(error);
logDegraded(serverLogger, { logDegraded(serverLogger, {
event: 'user.data_cleanup.document_blob_delete.failed', event: 'user.data_cleanup.document_blob_delete.failed',
msg: 'Failed to delete document blob', msg: 'Failed to delete document blob',
@ -66,6 +85,7 @@ export async function deleteUserStorageData(
try { try {
await deleteDocumentPreviewArtifacts(doc.id, namespace); await deleteDocumentPreviewArtifacts(doc.id, namespace);
} catch (error) { } catch (error) {
failures.push(error);
logDegraded(serverLogger, { logDegraded(serverLogger, {
event: 'user.data_cleanup.document_preview_delete.failed', event: 'user.data_cleanup.document_preview_delete.failed',
msg: 'Failed to delete preview artifacts', msg: 'Failed to delete preview artifacts',
@ -79,20 +99,24 @@ export async function deleteUserStorageData(
} }
} }
// Always clean up DB rows — documentPreviews has no FK cascade on user. // Preview rows/artifacts are shared by document id, so only the final
try { // owner may remove them.
await deleteDocumentPreviewRows(doc.id, namespace); if (isLastOwner) {
} catch (error) { try {
logDegraded(serverLogger, { await deleteDocumentPreviewRows(doc.id, namespace);
event: 'user.data_cleanup.document_preview_rows_delete.failed', } catch (error) {
msg: 'Failed to delete preview rows', failures.push(error);
step: 'delete_document_preview_rows', logDegraded(serverLogger, {
context: { event: 'user.data_cleanup.document_preview_rows_delete.failed',
documentId: doc.id, msg: 'Failed to delete preview rows',
userIdHash: hashForLog(userId), step: 'delete_document_preview_rows',
}, context: {
error, documentId: doc.id,
}); userIdHash: hashForLog(userId),
},
error,
});
}
} }
} }
@ -111,6 +135,7 @@ export async function deleteUserStorageData(
await deleteAudiobookPrefix(prefix); await deleteAudiobookPrefix(prefix);
booksDeleted++; booksDeleted++;
} catch (error) { } catch (error) {
failures.push(error);
logDegraded(serverLogger, { logDegraded(serverLogger, {
event: 'user.data_cleanup.audiobook_blobs_delete.failed', event: 'user.data_cleanup.audiobook_blobs_delete.failed',
msg: 'Failed to delete audiobook blobs', msg: 'Failed to delete audiobook blobs',
@ -127,6 +152,19 @@ export async function deleteUserStorageData(
// --- TTS segments --- // --- TTS segments ---
let segmentsDeleted = 0; let segmentsDeleted = 0;
if (s3Enabled) { if (s3Enabled) {
try {
await deleteDocumentPrefix(tempDocumentUploadPrefix(userId, namespace));
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.temp_document_uploads_delete.failed',
msg: 'Failed to delete temporary document uploads',
step: 'delete_temp_document_upload_prefix',
context: { userIdHash: hashForLog(userId) },
error,
});
}
try { try {
const cfg = getS3Config(); const cfg = getS3Config();
const nsSegment = namespace ? `ns/${namespace}/` : ''; const nsSegment = namespace ? `ns/${namespace}/` : '';
@ -135,6 +173,7 @@ export async function deleteUserStorageData(
segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV1); segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV1);
segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV2); segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV2);
} catch (error) { } catch (error) {
failures.push(error);
logDegraded(serverLogger, { logDegraded(serverLogger, {
event: 'user.data_cleanup.tts_segments_delete.failed', event: 'user.data_cleanup.tts_segments_delete.failed',
msg: 'Failed to delete TTS segment blobs', msg: 'Failed to delete TTS segment blobs',
@ -145,6 +184,34 @@ export async function deleteUserStorageData(
} }
} }
if (failures.length > 0) {
throw new AggregateError(failures, `User storage cleanup failed in ${failures.length} operation(s)`);
}
// Namespaced cleanup is an additional storage pass made by the account
// route. Database rows are global and must only be removed during the
// canonical non-namespaced beforeDelete pass.
if (namespace === null) {
// Delete explicitly for compatibility with pre-cascade installations and
// to remove auth verification tokens, which cannot carry a user FK.
for (const { table, userColumn, step } of [
{ table: userTtsChars, userColumn: userTtsChars.userId, step: 'delete_user_tts_usage_rows' },
{ table: userJobEvents, userColumn: userJobEvents.userId, step: 'delete_user_job_event_rows' },
{ table: authSchema.verification, userColumn: authSchema.verification.value, step: 'delete_user_verification_rows' },
]) {
await database.delete(table).where(eq(userColumn, userId)).catch((error: unknown) => {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.db_rows_delete.failed',
msg: 'Failed to delete non-cascading user database rows',
step,
context: { userIdHash: hashForLog(userId) },
error,
});
});
}
}
if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) { if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) {
serverLogger.info({ serverLogger.info({
event: 'user.data_cleanup.completed', event: 'user.data_cleanup.completed',
@ -156,4 +223,8 @@ export async function deleteUserStorageData(
segmentsDeleted, segmentsDeleted,
}, 'Completed user storage cleanup'); }, 'Completed user storage cleanup');
} }
if (failures.length > 0) {
throw new AggregateError(failures, `User database cleanup failed in ${failures.length} operation(s)`);
}
} }

View file

@ -11,7 +11,7 @@ export type ExportBlobBody =
| ArrayBufferView | ArrayBufferView
| { transformToByteArray: () => Promise<Uint8Array> }; | { transformToByteArray: () => Promise<Uint8Array> };
type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list'; type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list' | 'tts_segment';
type ExportIssue = { type ExportIssue = {
scope: ExportIssueScope; scope: ExportIssueScope;
@ -41,6 +41,20 @@ type ExportAudiobookObject = {
[key: string]: unknown; [key: string]: unknown;
}; };
type ExportTtsSegmentEntry = {
segmentEntryId: string;
documentId: string;
[key: string]: unknown;
};
type ExportTtsSegmentVariant = {
segmentId: string;
segmentEntryId: string;
audioKey?: string | null;
audioFormat?: string | null;
[key: string]: unknown;
};
export type AppendUserExportArchiveInput = { export type AppendUserExportArchiveInput = {
archive: Archiver; archive: Archiver;
userId: string; userId: string;
@ -49,12 +63,19 @@ export type AppendUserExportArchiveInput = {
preferences: unknown | null; preferences: unknown | null;
readingHistory: unknown[]; readingHistory: unknown[];
ttsUsage: unknown[]; ttsUsage: unknown[];
jobEvents: unknown[];
documentSettings: unknown[];
authSessions: unknown[];
linkedAccounts: unknown[];
documents: ExportDocument[]; documents: ExportDocument[];
audiobooks: ExportAudiobook[]; audiobooks: ExportAudiobook[];
audiobookChapters: ExportAudiobookChapter[]; audiobookChapters: ExportAudiobookChapter[];
ttsSegmentEntries: ExportTtsSegmentEntry[];
ttsSegmentVariants: ExportTtsSegmentVariant[];
getDocumentBlobStream: (documentId: string) => Promise<ExportBlobBody>; getDocumentBlobStream: (documentId: string) => Promise<ExportBlobBody>;
listAudiobookObjects: (bookId: string, userId: string) => Promise<ExportAudiobookObject[]>; listAudiobookObjects: (bookId: string, userId: string) => Promise<ExportAudiobookObject[]>;
getAudiobookObjectStream: (bookId: string, userId: string, fileName: string) => Promise<ExportBlobBody>; getAudiobookObjectStream: (bookId: string, userId: string, fileName: string) => Promise<ExportBlobBody>;
getTtsSegmentAudioStream: (audioKey: string) => Promise<ExportBlobBody>;
}; };
function isNodeReadableStream(value: unknown): value is Readable { function isNodeReadableStream(value: unknown): value is Readable {
@ -124,17 +145,25 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
preferences, preferences,
readingHistory, readingHistory,
ttsUsage, ttsUsage,
jobEvents,
documentSettings,
authSessions,
linkedAccounts,
documents, documents,
audiobooks, audiobooks,
audiobookChapters, audiobookChapters,
ttsSegmentEntries,
ttsSegmentVariants,
getDocumentBlobStream, getDocumentBlobStream,
listAudiobookObjects, listAudiobookObjects,
getAudiobookObjectStream, getAudiobookObjectStream,
getTtsSegmentAudioStream,
} = input; } = input;
const issues: ExportIssue[] = []; const issues: ExportIssue[] = [];
let documentFilesExported = 0; let documentFilesExported = 0;
let audiobookFilesExported = 0; let audiobookFilesExported = 0;
let ttsSegmentFilesExported = 0;
appendJson(archive, 'profile.json', profileData); appendJson(archive, 'profile.json', profileData);
if (preferences) { if (preferences) {
@ -142,7 +171,13 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
} }
appendJson(archive, 'reading_history.json', readingHistory); appendJson(archive, 'reading_history.json', readingHistory);
appendJson(archive, 'tts_usage.json', ttsUsage); appendJson(archive, 'tts_usage.json', ttsUsage);
appendJson(archive, 'job_events.json', jobEvents);
appendJson(archive, 'document_settings.json', documentSettings);
appendJson(archive, 'auth_sessions.json', authSessions);
appendJson(archive, 'linked_accounts.json', linkedAccounts);
appendJson(archive, 'library_documents.json', documents); appendJson(archive, 'library_documents.json', documents);
appendJson(archive, 'tts_segment_entries.json', ttsSegmentEntries);
appendJson(archive, 'tts_segment_variants.json', ttsSegmentVariants);
const chaptersByBookId = new Map<string, ExportAudiobookChapter[]>(); const chaptersByBookId = new Map<string, ExportAudiobookChapter[]>();
for (const chapter of audiobookChapters) { for (const chapter of audiobookChapters) {
@ -215,8 +250,40 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
} }
} }
const documentIdByEntryId = new Map(
ttsSegmentEntries.map((entry) => [entry.segmentEntryId, entry.documentId]),
);
const exportedAudioKeys = new Set<string>();
for (const variant of ttsSegmentVariants) {
const audioKey = typeof variant.audioKey === 'string' ? variant.audioKey : '';
if (!audioKey || exportedAudioKeys.has(audioKey)) continue;
exportedAudioKeys.add(audioKey);
const documentId = toSafePathSegment(
documentIdByEntryId.get(variant.segmentEntryId) ?? 'unknown-document',
'unknown-document',
);
const segmentId = toSafePathSegment(variant.segmentId, 'segment');
const format = toSafePathSegment(variant.audioFormat || 'mp3', 'mp3');
const entryName = `files/tts_segments/${documentId}/${segmentId}.${format}`;
try {
const body = await getTtsSegmentAudioStream(audioKey);
const stream = await bodyToNodeReadable(body);
archive.append(stream, { name: entryName });
ttsSegmentFilesExported += 1;
} catch (error) {
issues.push({
scope: 'tts_segment',
id: variant.segmentId,
fileName: entryName,
message: normalizeErrorMessage(error),
});
}
}
const manifest = { const manifest = {
formatVersion: 2, formatVersion: 3,
exportedAtMs, exportedAtMs,
userId, userId,
scope: 'owned', scope: 'owned',
@ -224,14 +291,26 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
documentsMetadata: documents.length, documentsMetadata: documents.length,
audiobooksMetadata: audiobooks.length, audiobooksMetadata: audiobooks.length,
audiobookChaptersMetadata: audiobookChapters.length, audiobookChaptersMetadata: audiobookChapters.length,
documentSettingsMetadata: documentSettings.length,
ttsSegmentEntriesMetadata: ttsSegmentEntries.length,
ttsSegmentVariantsMetadata: ttsSegmentVariants.length,
authSessionsMetadata: authSessions.length,
linkedAccountsMetadata: linkedAccounts.length,
jobEventsMetadata: jobEvents.length,
documentFiles: documentFilesExported, documentFiles: documentFilesExported,
audiobookFiles: audiobookFilesExported, audiobookFiles: audiobookFilesExported,
ttsSegmentFiles: ttsSegmentFilesExported,
issues: issues.length, issues: issues.length,
}, },
includes: { includes: {
metadata: true, metadata: true,
documentFiles: true, documentFiles: true,
audiobookFiles: true, audiobookFiles: true,
ttsSegmentFiles: true,
credentialSecrets: false,
temporaryUploads: false,
derivedDocumentPreviews: false,
derivedParsedDocuments: false,
filesystemSources: false, filesystemSources: false,
}, },
}; };

View file

@ -0,0 +1,63 @@
import { describe, expect, test } from 'vitest';
import type { Archiver } from 'archiver';
import { appendUserExportArchive } from '../../src/lib/server/user/data-export';
describe('user data export archive', () => {
test('includes user metadata and cached TTS audio without credential secrets', async () => {
const entries = new Map<string, unknown>();
const archive = {
append(value: unknown, options: { name: string }) {
entries.set(options.name, value);
return archive;
},
} as unknown as Archiver;
await appendUserExportArchive({
archive,
userId: 'user-1',
exportedAtMs: 123,
profileData: { user: { id: 'user-1', email: 'person@example.com' }, exportedAtMs: 123 },
preferences: { dataJson: '{}' },
readingHistory: [{ documentId: 'doc-1' }],
ttsUsage: [{ charCount: 10 }],
jobEvents: [{ action: 'pdf-layout' }],
documentSettings: [{ documentId: 'doc-1', dataJson: '{}' }],
authSessions: [{ id: 'session-1', ipAddress: null }],
linkedAccounts: [{ id: 'account-1', providerId: 'credential' }],
documents: [],
audiobooks: [],
audiobookChapters: [],
ttsSegmentEntries: [{ segmentEntryId: 'entry-1', documentId: 'doc-1' }],
ttsSegmentVariants: [{
segmentId: 'segment-1',
segmentEntryId: 'entry-1',
audioKey: 'private/audio/key',
audioFormat: 'mp3',
}],
getDocumentBlobStream: async () => new Uint8Array(),
listAudiobookObjects: async () => [],
getAudiobookObjectStream: async () => new Uint8Array(),
getTtsSegmentAudioStream: async () => new Uint8Array([1, 2, 3]),
});
expect(entries.has('document_settings.json')).toBe(true);
expect(entries.has('auth_sessions.json')).toBe(true);
expect(entries.has('linked_accounts.json')).toBe(true);
expect(entries.has('tts_segment_entries.json')).toBe(true);
expect(entries.has('tts_segment_variants.json')).toBe(true);
expect(entries.has('files/tts_segments/doc-1/segment-1.mp3')).toBe(true);
const manifest = JSON.parse(String(entries.get('export_manifest.json')));
expect(manifest).toMatchObject({
formatVersion: 3,
counts: {
ttsSegmentEntriesMetadata: 1,
ttsSegmentVariantsMetadata: 1,
ttsSegmentFiles: 1,
},
includes: {
credentialSecrets: false,
},
});
});
});

View file

@ -0,0 +1,50 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
send: vi.fn(),
}));
vi.mock('@/lib/server/storage/s3', () => ({
getS3Config: () => ({ bucket: 'test-bucket', prefix: 'openreader-test' }),
getS3Client: () => ({ send: mocks.send }),
getS3ProxyClient: () => ({ send: mocks.send }),
}));
import { deleteAudiobookPrefix } from '../../src/lib/server/audiobooks/blobstore';
import { deleteDocumentPrefix } from '../../src/lib/server/documents/blobstore';
describe('storage prefix cleanup', () => {
beforeEach(() => {
mocks.send.mockReset();
});
test.each([
['document', deleteDocumentPrefix],
['audiobook', deleteAudiobookPrefix],
])('%s cleanup counts successful quiet deletes', async (_name, removePrefix) => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a' }, { Key: 'prefix/b' }],
IsTruncated: false,
})
.mockResolvedValueOnce({});
await expect(removePrefix('prefix/')).resolves.toBe(2);
});
test.each([
['document', deleteDocumentPrefix],
['audiobook', deleteAudiobookPrefix],
])('%s cleanup fails on per-object storage errors', async (_name, removePrefix) => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a' }],
IsTruncated: false,
})
.mockResolvedValueOnce({
Errors: [{ Key: 'prefix/a', Code: 'AccessDenied' }],
});
await expect(removePrefix('prefix/')).rejects.toThrow('Failed deleting 1');
});
});

View file

@ -0,0 +1,45 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
send: vi.fn(),
}));
vi.mock('@/lib/server/storage/s3', () => ({
getS3Config: () => ({ bucket: 'test-bucket' }),
getS3Client: () => ({ send: mocks.send }),
getS3ProxyClient: () => ({ send: mocks.send }),
}));
import { deleteTtsSegmentPrefix } from '../../src/lib/server/tts/segments-blobstore';
describe('TTS segment blob cleanup', () => {
beforeEach(() => {
mocks.send.mockReset();
});
test('counts successful quiet deletes by requested object count', async () => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a.mp3' }, { Key: 'prefix/b.mp3' }],
IsTruncated: false,
})
.mockResolvedValueOnce({});
await expect(deleteTtsSegmentPrefix('prefix/')).resolves.toBe(2);
});
test('fails cleanup when storage reports per-object deletion errors', async () => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a.mp3' }],
IsTruncated: false,
})
.mockResolvedValueOnce({
Errors: [{ Key: 'prefix/a.mp3', Code: 'AccessDenied' }],
});
await expect(deleteTtsSegmentPrefix('prefix/')).rejects.toThrow(
'Failed deleting 1 TTS segment audio objects',
);
});
});

View file

@ -1,5 +1,6 @@
import { describe, expect, test } from 'vitest'; import { describe, expect, test } from 'vitest';
import { import {
buildTtsSegmentDocumentPrefix,
buildProportionalAlignment, buildProportionalAlignment,
buildTtsSegmentEntryId, buildTtsSegmentEntryId,
buildTtsSegmentId, buildTtsSegmentId,
@ -12,6 +13,15 @@ import {
} from '../../src/lib/server/tts/segments'; } from '../../src/lib/server/tts/segments';
describe('tts segment helpers', () => { describe('tts segment helpers', () => {
test('builds a user/document-scoped audio prefix across every version and variant', () => {
expect(buildTtsSegmentDocumentPrefix({
storagePrefix: 'openreader',
namespace: 'test namespace',
userId: 'user/name',
documentId: 'doc-id',
})).toBe('openreader/tts_segments_v2/ns/test namespace/users/user%2Fname/docs/doc-id/');
});
test('builds stable settings hash', () => { test('builds stable settings hash', () => {
const a = buildTtsSegmentSettingsHash({ const a = buildTtsSegmentSettingsHash({
providerRef: 'openai', providerRef: 'openai',

View file

@ -0,0 +1,115 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
selectResults: [] as unknown[][],
deleteWhere: vi.fn(async () => undefined),
deleteDocumentBlob: vi.fn(async () => undefined),
deleteDocumentPrefix: vi.fn(async () => 0),
deleteDocumentPreviewArtifacts: vi.fn(async () => 0),
deleteDocumentPreviewRows: vi.fn(async () => undefined),
deleteAudiobookPrefix: vi.fn(async () => 0),
deleteTtsSegmentPrefix: vi.fn(async () => 0),
}));
function resultBuilder(result: unknown[]) {
return {
limit: async () => result,
then: (resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) =>
Promise.resolve(result).then(resolve, reject),
};
}
vi.mock('@/db', () => ({
db: {
select: vi.fn(() => ({
from: vi.fn(() => ({
where: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])),
})),
})),
delete: vi.fn(() => ({
where: mocks.deleteWhere,
})),
},
}));
vi.mock('@/lib/server/storage/s3', () => ({
isS3Configured: () => true,
getS3Config: () => ({ prefix: 'openreader-test' }),
}));
vi.mock('@/lib/server/documents/blobstore', () => ({
deleteDocumentBlob: mocks.deleteDocumentBlob,
deleteDocumentPrefix: mocks.deleteDocumentPrefix,
tempDocumentUploadPrefix: () => 'temp/user/',
}));
vi.mock('@/lib/server/documents/previews-blobstore', () => ({
deleteDocumentPreviewArtifacts: mocks.deleteDocumentPreviewArtifacts,
}));
vi.mock('@/lib/server/documents/previews', () => ({
deleteDocumentPreviewRows: mocks.deleteDocumentPreviewRows,
}));
vi.mock('@/lib/server/audiobooks/blobstore', () => ({
audiobookPrefix: () => 'audiobooks/user/',
deleteAudiobookPrefix: mocks.deleteAudiobookPrefix,
}));
vi.mock('@/lib/server/tts/segments-blobstore', () => ({
deleteTtsSegmentPrefix: mocks.deleteTtsSegmentPrefix,
}));
vi.mock('@/lib/server/logger', () => ({
hashForLog: () => 'hash',
serverLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
vi.mock('@/lib/server/errors/logging', () => ({
logDegraded: vi.fn(),
}));
import { deleteUserStorageData } from '../../src/lib/server/user/data-cleanup';
describe('user data cleanup', () => {
beforeEach(() => {
mocks.selectResults = [];
for (const mock of Object.values(mocks)) {
if (typeof mock === 'function' && 'mockReset' in mock) {
mock.mockReset();
}
}
mocks.deleteWhere.mockResolvedValue(undefined);
mocks.deleteDocumentBlob.mockResolvedValue(undefined);
mocks.deleteDocumentPrefix.mockResolvedValue(0);
mocks.deleteDocumentPreviewArtifacts.mockResolvedValue(0);
mocks.deleteDocumentPreviewRows.mockResolvedValue(undefined);
mocks.deleteAudiobookPrefix.mockResolvedValue(0);
mocks.deleteTtsSegmentPrefix.mockResolvedValue(0);
});
test('keeps shared document blobs and previews', async () => {
mocks.selectResults = [
[{ id: 'shared-doc' }],
[{ id: 'shared-doc' }],
[],
];
await deleteUserStorageData('user-1', null);
expect(mocks.deleteDocumentBlob).not.toHaveBeenCalled();
expect(mocks.deleteDocumentPreviewArtifacts).not.toHaveBeenCalled();
expect(mocks.deleteDocumentPreviewRows).not.toHaveBeenCalled();
expect(mocks.deleteWhere).toHaveBeenCalledTimes(3);
});
test('blocks database cleanup when storage cleanup fails', async () => {
mocks.selectResults = [[], []];
mocks.deleteDocumentPrefix.mockRejectedValueOnce(new Error('storage unavailable'));
await expect(deleteUserStorageData('user-1', null)).rejects.toThrow(
'User storage cleanup failed',
);
expect(mocks.deleteWhere).not.toHaveBeenCalled();
});
});