refactor(user): overhaul user data cleanup and export for cascading deletes and TTS segment support

Revise user data cleanup logic to ensure proper cascading deletion of user-related
database rows and S3 objects, including shared document and preview artifacts.
Introduce explicit checks for last ownership before removing shared resources.
Add TTS segment cache and audio cleanup to document and user deletion flows.
Expand user data export to include TTS segment entries and audio, job events,
document settings, and linked auth sessions. Update schema with ON DELETE CASCADE
for userTtsChars and userJobEvents. Add new migration scripts and comprehensive
unit tests for cleanup and export scenarios.
This commit is contained in:
Richard R 2026-06-06 16:37:28 -06:00
parent 92df3f907c
commit 76553023e8
27 changed files with 4425 additions and 66 deletions

View file

@ -0,0 +1,8 @@
DELETE FROM "user_job_events" WHERE NOT EXISTS (
SELECT 1 FROM "user" WHERE "user"."id" = "user_job_events"."user_id"
);--> statement-breakpoint
DELETE FROM "user_tts_chars" WHERE NOT EXISTS (
SELECT 1 FROM "user" WHERE "user"."id" = "user_tts_chars"."user_id"
);--> statement-breakpoint
ALTER TABLE "user_job_events" ADD CONSTRAINT "user_job_events_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "user_tts_chars" ADD CONSTRAINT "user_tts_chars_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;

File diff suppressed because it is too large Load diff

View file

@ -71,6 +71,13 @@
"when": 1780625663880,
"tag": "0009_drop_pdf_parse",
"breakpoints": true
},
{
"idx": 10,
"version": "7",
"when": 1780785284335,
"tag": "0010_user-data-cleanup-cascades",
"breakpoints": true
}
]
}

View file

@ -0,0 +1,35 @@
PRAGMA foreign_keys=OFF;--> statement-breakpoint
DELETE FROM `user_job_events` WHERE NOT EXISTS (
SELECT 1 FROM `user` WHERE `user`.`id` = `user_job_events`.`user_id`
);--> statement-breakpoint
DELETE FROM `user_tts_chars` WHERE NOT EXISTS (
SELECT 1 FROM `user` WHERE `user`.`id` = `user_tts_chars`.`user_id`
);--> statement-breakpoint
CREATE TABLE `__new_user_job_events` (
`user_id` text NOT NULL,
`action` text NOT NULL,
`op_id` text NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)) NOT NULL,
PRIMARY KEY(`user_id`, `action`, `op_id`),
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
INSERT INTO `__new_user_job_events`("user_id", "action", "op_id", "created_at") SELECT "user_id", "action", "op_id", "created_at" FROM `user_job_events`;--> statement-breakpoint
DROP TABLE `user_job_events`;--> statement-breakpoint
ALTER TABLE `__new_user_job_events` RENAME TO `user_job_events`;--> statement-breakpoint
PRAGMA foreign_keys=ON;--> statement-breakpoint
CREATE INDEX `idx_user_job_events_user_action_created` ON `user_job_events` (`user_id`,`action`,`created_at`);--> statement-breakpoint
CREATE TABLE `__new_user_tts_chars` (
`user_id` text NOT NULL,
`date` text NOT NULL,
`char_count` integer DEFAULT 0,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
PRIMARY KEY(`user_id`, `date`),
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
INSERT INTO `__new_user_tts_chars`("user_id", "date", "char_count", "created_at", "updated_at") SELECT "user_id", "date", "char_count", "created_at", "updated_at" FROM `user_tts_chars`;--> statement-breakpoint
DROP TABLE `user_tts_chars`;--> statement-breakpoint
ALTER TABLE `__new_user_tts_chars` RENAME TO `user_tts_chars`;--> statement-breakpoint
CREATE INDEX `idx_user_tts_chars_date` ON `user_tts_chars` (`date`);

File diff suppressed because it is too large Load diff

View file

@ -71,6 +71,13 @@
"when": 1780625663601,
"tag": "0009_drop_pdf_parse",
"breakpoints": true
},
{
"idx": 10,
"version": "6",
"when": 1780785284041,
"tag": "0010_user-data-cleanup-cascades",
"breakpoints": true
}
]
}

View file

@ -3,7 +3,7 @@ import { NextResponse } from 'next/server';
import { auth } from '@/lib/server/auth/auth';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { deleteUserStorageData } from '@/lib/server/user/data-cleanup';
import { errorToLog, hashForLog, serverLogger } from '@/lib/server/logger';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response';
export async function DELETE() {
@ -18,21 +18,12 @@ export async function DELETE() {
}
try {
// Best-effort cleanup for test namespaced storage using request context.
// The Better Auth beforeDelete hook still runs and handles non-namespaced data.
// Clean test-namespaced storage using request context. The Better Auth
// beforeDelete hook handles non-namespaced storage and blocks deletion if
// either cleanup cannot complete.
const testNamespace = getOpenReaderTestNamespace(reqHeaders);
if (testNamespace) {
try {
await deleteUserStorageData(session.user.id, testNamespace);
} catch (error) {
serverLogger.warn({
event: 'account.delete.storage_cleanup_failed',
degraded: true,
step: 'namespaced_storage_cleanup',
userIdHash: hashForLog(session.user.id),
error: errorToLog(error),
}, 'Failed to clean up namespaced user storage before deletion');
}
await deleteUserStorageData(session.user.id, testNamespace);
}
// Use Better Auth's built-in deleteUser to handle cascading cleanup

View file

@ -9,6 +9,7 @@ import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'
import { isS3Configured } from '@/lib/server/storage/s3';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { errorResponse } from '@/lib/server/errors/next-response';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
export const dynamic = 'force-dynamic';
@ -80,9 +81,14 @@ export async function GET(req: NextRequest) {
});
} catch (error) {
if (isMissingBlobError(error)) {
await deleteDocumentTtsSegmentCache({
userId: doc.userId,
documentId: id,
namespace: testNamespace,
});
await db
.delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
.where(and(eq(documents.id, id), eq(documents.userId, doc.userId)));
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
throw error;

View file

@ -22,6 +22,7 @@ import {
import { extractRawTextSnippet } from '@/lib/server/documents/text-snippets';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
export const dynamic = 'force-dynamic';
@ -108,9 +109,14 @@ export async function GET(req: NextRequest) {
);
} catch (error) {
if (isMissingDocumentBlobError(error)) {
await deleteDocumentTtsSegmentCache({
userId: doc.userId,
documentId: id,
namespace: testNamespace,
});
await db
.delete(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)));
.where(and(eq(documents.id, id), eq(documents.userId, doc.userId)));
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
throw error;

View file

@ -13,6 +13,7 @@ import {
import { deleteDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
import type { BaseDocument, DocumentType } from '@/types/documents';
export const dynamic = 'force-dynamic';
@ -136,6 +137,25 @@ export async function DELETE(req: NextRequest) {
return NextResponse.json({ success: true, deleted: 0 });
}
const ownedRows = (await db
.select({ id: documents.id, userId: documents.userId })
.from(documents)
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))) as Array<{
id: string;
userId: string;
}>;
// TTS audio is user-scoped even when the underlying document blob is
// shared. Remove it before deleting ownership metadata so failures remain
// retryable and cannot create untraceable S3 objects.
for (const row of ownedRows) {
await deleteDocumentTtsSegmentCache({
userId: row.userId,
documentId: row.id,
namespace: testNamespace,
});
}
const deletedRows = (await db
.delete(documents)
.where(and(inArray(documents.userId, targetUserIds), inArray(documents.id, targetIds)))

View file

@ -2,12 +2,26 @@ import { NextRequest, NextResponse } from 'next/server';
import { PassThrough, Readable } from 'stream';
import { auth } from '@/lib/server/auth/auth';
import { db } from '@/db';
import { documents, audiobooks, audiobookChapters, userPreferences, userDocumentProgress, userTtsChars } from '@/db/schema';
import {
documents,
audiobooks,
audiobookChapters,
documentSettings,
ttsSegmentEntries,
ttsSegmentVariants,
userDocumentProgress,
userJobEvents,
userPreferences,
userTtsChars,
} from '@/db/schema';
import * as authSchemaSqlite from '@/db/schema_auth_sqlite';
import * as authSchemaPostgres from '@/db/schema_auth_postgres';
import { and, desc, eq, inArray } from 'drizzle-orm';
import archiver from 'archiver';
import { appendUserExportArchive } from '@/lib/server/user/data-export';
import { getDocumentBlobStream } from '@/lib/server/documents/blobstore';
import { getAudiobookObjectStream, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore';
import { getTtsSegmentAudioObjectStream } from '@/lib/server/tts/segments-blobstore';
import { isS3Configured } from '@/lib/server/storage/s3';
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
import { nowTimestampMs } from '@/lib/shared/timestamps';
@ -17,13 +31,6 @@ import { errorResponse } from '@/lib/server/errors/next-response';
export const dynamic = 'force-dynamic';
export async function GET(req: NextRequest) {
if (!isS3Configured()) {
return NextResponse.json(
{ error: 'Export storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
if (!auth) {
return errorResponse(new Error('Auth not initialized'), {
apiErrorMessage: 'Auth not initialized',
@ -41,11 +48,21 @@ export async function GET(req: NextRequest) {
const userId = session.user.id;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const storageEnabled = isS3Configured();
const requireStorage = () => {
if (!storageEnabled) {
throw new Error('Storage is not configured; file content could not be exported');
}
};
const [
prefs,
progress,
ttsUsage,
jobEvents,
perDocumentSettings,
segmentEntries,
segmentVariants,
userDocs,
userAudiobooks,
] = await Promise.all([
@ -60,6 +77,26 @@ export async function GET(req: NextRequest) {
.from(userTtsChars)
.where(eq(userTtsChars.userId, userId))
.orderBy(desc(userTtsChars.date)),
db
.select()
.from(userJobEvents)
.where(eq(userJobEvents.userId, userId))
.orderBy(desc(userJobEvents.createdAt)),
db
.select()
.from(documentSettings)
.where(eq(documentSettings.userId, userId))
.orderBy(desc(documentSettings.updatedAt)),
db
.select()
.from(ttsSegmentEntries)
.where(eq(ttsSegmentEntries.userId, userId))
.orderBy(desc(ttsSegmentEntries.updatedAt)),
db
.select()
.from(ttsSegmentVariants)
.where(eq(ttsSegmentVariants.userId, userId))
.orderBy(desc(ttsSegmentVariants.updatedAt)),
db
.select()
.from(documents)
@ -72,6 +109,36 @@ export async function GET(req: NextRequest) {
.orderBy(desc(audiobooks.createdAt)),
]);
const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite;
// Auth exports intentionally select metadata only. Credential and session
// secrets must never be written into the archive.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const database = db as any;
const [authSessions, linkedAccounts] = await Promise.all([
database
.select({
id: authSchema.session.id,
expiresAt: authSchema.session.expiresAt,
createdAt: authSchema.session.createdAt,
updatedAt: authSchema.session.updatedAt,
ipAddress: authSchema.session.ipAddress,
userAgent: authSchema.session.userAgent,
})
.from(authSchema.session)
.where(eq(authSchema.session.userId, userId)),
database
.select({
id: authSchema.account.id,
accountId: authSchema.account.accountId,
providerId: authSchema.account.providerId,
scope: authSchema.account.scope,
createdAt: authSchema.account.createdAt,
updatedAt: authSchema.account.updatedAt,
})
.from(authSchema.account)
.where(eq(authSchema.account.userId, userId)),
]);
const archive = archiver('zip', {
zlib: { level: 0 },
forceZip64: true,
@ -114,7 +181,6 @@ export async function GET(req: NextRequest) {
const exportedAtMs = nowTimestampMs();
const profileData = {
user: session.user,
session: session.session,
exportedAtMs,
};
@ -126,13 +192,31 @@ export async function GET(req: NextRequest) {
preferences: prefs[0] ?? null,
readingHistory: progress,
ttsUsage,
jobEvents,
documentSettings: perDocumentSettings,
authSessions,
linkedAccounts,
documents: userDocs,
audiobooks: userAudiobooks,
audiobookChapters: allChapters,
getDocumentBlobStream: async (documentId: string) => getDocumentBlobStream(documentId, testNamespace),
listAudiobookObjects: async (bookId: string, ownerId: string) => listAudiobookObjects(bookId, ownerId, testNamespace),
getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) =>
getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace),
ttsSegmentEntries: segmentEntries,
ttsSegmentVariants: segmentVariants,
getDocumentBlobStream: async (documentId: string) => {
requireStorage();
return getDocumentBlobStream(documentId, testNamespace);
},
listAudiobookObjects: async (bookId: string, ownerId: string) => {
requireStorage();
return listAudiobookObjects(bookId, ownerId, testNamespace);
},
getAudiobookObjectStream: async (bookId: string, ownerId: string, fileName: string) => {
requireStorage();
return getAudiobookObjectStream(bookId, ownerId, fileName, testNamespace);
},
getTtsSegmentAudioStream: async (audioKey: string) => {
requireStorage();
return (await getTtsSegmentAudioObjectStream(audioKey)).stream;
},
});
if (!req.signal.aborted) {

View file

@ -54,7 +54,7 @@ export const audiobookChapters = pgTable('audiobook_chapters', {
// defined here. Only application-specific tables belong in this file.
export const userTtsChars = pgTable("user_tts_chars", {
userId: text('user_id').notNull(),
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
date: date('date').notNull(),
charCount: bigint('char_count', { mode: 'number' }).default(0),
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
@ -71,7 +71,7 @@ export const userTtsChars = pgTable("user_tts_chars", {
// worker bounds each op by a hard cap, "ops created in the last hard-cap
// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically.
export const userJobEvents = pgTable('user_job_events', {
userId: text('user_id').notNull(),
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
action: text('action').notNull(),
opId: text('op_id').notNull(),
createdAt: bigint('created_at', { mode: 'number' }).notNull().default(PG_NOW_MS),

View file

@ -54,7 +54,7 @@ export const audiobookChapters = sqliteTable('audiobook_chapters', {
// defined here. Only application-specific tables belong in this file.
export const userTtsChars = sqliteTable("user_tts_chars", {
userId: text('user_id').notNull(),
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
date: text('date').notNull(), // SQLite doesn't have native DATE type, text YYYY-MM-DD is standard
charCount: integer('char_count').default(0),
createdAt: integer('created_at').default(SQLITE_NOW_MS),
@ -71,7 +71,7 @@ export const userTtsChars = sqliteTable("user_tts_chars", {
// worker bounds each op by a hard cap, "ops created in the last hard-cap
// window" is an upper bound on in-flight ops. Old rows are pruned opportunistically.
export const userJobEvents = sqliteTable('user_job_events', {
userId: text('user_id').notNull(),
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
action: text('action').notNull(),
opId: text('op_id').notNull(),
createdAt: integer('created_at').notNull().default(SQLITE_NOW_MS),

View file

@ -255,7 +255,11 @@ export async function deleteAudiobookPrefix(prefix: string): Promise<number> {
},
}),
);
deleted += deleteRes.Deleted?.length ?? 0;
const errors = deleteRes.Errors ?? [];
if (errors.length > 0) {
throw new Error(`Failed deleting ${errors.length} audiobook storage objects`);
}
deleted += keys.length;
}
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;

View file

@ -96,8 +96,9 @@ const createAuth = () => betterAuth({
context: { userIdHash: hashForLog(user.id) },
error,
});
// Don't throw allow the user deletion to proceed even if S3 cleanup fails.
// Orphaned blobs are preferable to a blocked account deletion.
// Without a durable cleanup queue, proceeding would permanently
// orphan user-scoped storage and non-cascading database rows.
throw error;
}
},
},

View file

@ -507,10 +507,10 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
const parsedPrefix = `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/`;
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })).catch(() => undefined);
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined);
await deleteDocumentPrefix(parsedPrefix).catch(() => undefined);
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey }));
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey }));
await deleteDocumentPrefix(parsedPrefix);
await deleteDocumentPrefix(`${key}/`);
}
export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise<void> {
@ -564,7 +564,11 @@ export async function deleteDocumentPrefix(prefix: string): Promise<number> {
},
}),
);
deleted += deleteRes.Deleted?.length ?? 0;
const errors = deleteRes.Errors ?? [];
if (errors.length > 0) {
throw new Error(`Failed deleting ${errors.length} document storage objects`);
}
deleted += keys.length;
}
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;

View file

@ -1,9 +1,11 @@
import { db } from '@/db';
import { documents } from '@/db/schema';
import { and, eq } from 'drizzle-orm';
import {
enqueueDocumentPreview,
} from '@/lib/server/documents/previews';
import { errorToLog, serverLogger } from '@/lib/server/logger';
import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache';
import type { BaseDocument, DocumentType } from '@/types/documents';
type RegisterUploadedDocumentInput = {
@ -17,6 +19,23 @@ type RegisterUploadedDocumentInput = {
};
export async function registerUploadedDocument(input: RegisterUploadedDocumentInput): Promise<BaseDocument> {
const [existing] = await db
.select({ lastModified: documents.lastModified })
.from(documents)
.where(and(
eq(documents.id, input.documentId),
eq(documents.userId, input.userId),
))
.limit(1);
if (existing && Number(existing.lastModified) !== input.lastModified) {
await deleteDocumentTtsSegmentCache({
userId: input.userId,
documentId: input.documentId,
namespace: input.namespace,
});
}
await db
.insert(documents)
.values({

View file

@ -244,7 +244,12 @@ export async function deleteTtsSegmentPrefix(prefix: string): Promise<number> {
},
}),
);
deleted += deleteRes.Deleted?.length ?? 0;
const errors = deleteRes.Errors ?? [];
if (errors.length > 0) {
throw new Error(`Failed deleting ${errors.length} TTS segment audio objects`);
}
// Quiet=true commonly omits Deleted entries on successful requests.
deleted += keys.length;
}
continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined;

View file

@ -1,7 +1,12 @@
import { and, eq } from 'drizzle-orm';
import { db } from '@/db';
import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema';
import { deleteTtsSegmentAudioObjects } from '@/lib/server/tts/segments-blobstore';
import {
deleteTtsSegmentAudioObjects,
deleteTtsSegmentPrefix,
} from '@/lib/server/tts/segments-blobstore';
import { buildTtsSegmentDocumentPrefix } from '@/lib/server/tts/segments';
import { getS3Config } from '@/lib/server/storage/s3';
import type { ReaderType } from '@/types/user-state';
import { serverLogger } from '@/lib/server/logger';
import { logDegraded } from '@/lib/server/errors/logging';
@ -50,8 +55,6 @@ export async function clearTtsSegmentCache(
)
.where(and(...conditions))) as Array<{ segmentId: string; audioKey: string | null }>;
await db.delete(ttsSegmentEntries).where(and(...conditions));
const audioKeys = rows
.map((row) => row.audioKey)
.filter((key): key is string => Boolean(key));
@ -80,10 +83,38 @@ export async function clearTtsSegmentCache(
}
}
// Keep metadata when storage cleanup is incomplete so a later retry still
// knows which objects must be removed.
if (!warning) {
await db.delete(ttsSegmentEntries).where(and(...conditions));
}
return {
deletedSegments: rows.length,
deletedSegments: warning ? 0 : rows.length,
requestedAudioObjects: uniqueAudioKeys.length,
deletedAudioObjects,
...(warning ? { warning } : {}),
};
}
export async function deleteDocumentTtsSegmentCache(input: {
userId: string;
documentId: string;
namespace: string | null;
}): Promise<void> {
const storagePrefix = getS3Config().prefix;
for (const storageVersion of ['v1', 'v2'] as const) {
await deleteTtsSegmentPrefix(buildTtsSegmentDocumentPrefix({
storagePrefix,
namespace: input.namespace,
userId: input.userId,
documentId: input.documentId,
storageVersion,
}));
}
await db.delete(ttsSegmentEntries).where(and(
eq(ttsSegmentEntries.userId, input.userId),
eq(ttsSegmentEntries.documentId, input.documentId),
));
}

View file

@ -226,6 +226,17 @@ export function buildTtsSegmentAudioKey(input: {
return `${input.storagePrefix}/tts_segments_v2/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/${input.documentVersion}/${input.settingsHash}/${input.segmentId}.mp3`;
}
export function buildTtsSegmentDocumentPrefix(input: {
storagePrefix: string;
namespace: string | null;
userId: string;
documentId: string;
storageVersion?: 'v1' | 'v2';
}): string {
const nsSegment = input.namespace ? `ns/${input.namespace}/` : '';
return `${input.storagePrefix}/tts_segments_${input.storageVersion ?? 'v2'}/${nsSegment}users/${encodeURIComponent(input.userId)}/docs/${input.documentId}/`;
}
export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: AbortSignal): Promise<number> {
let workDir: string | null = null;
try {

View file

@ -5,10 +5,16 @@
*/
import { db } from '@/db';
import { documents, audiobooks } from '@/db/schema';
import { eq } from 'drizzle-orm';
import { documents, audiobooks, userJobEvents, userTtsChars } from '@/db/schema';
import * as authSchemaSqlite from '@/db/schema_auth_sqlite';
import * as authSchemaPostgres from '@/db/schema_auth_postgres';
import { and, eq, ne } from 'drizzle-orm';
import { getS3Config, isS3Configured } from '@/lib/server/storage/s3';
import { deleteDocumentBlob } from '@/lib/server/documents/blobstore';
import {
deleteDocumentBlob,
deleteDocumentPrefix,
tempDocumentUploadPrefix,
} from '@/lib/server/documents/blobstore';
import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore';
import { deleteDocumentPreviewRows } from '@/lib/server/documents/previews';
import { audiobookPrefix, deleteAudiobookPrefix } from '@/lib/server/audiobooks/blobstore';
@ -34,9 +40,11 @@ export async function deleteUserStorageData(
namespace: string | null,
): Promise<void> {
const s3Enabled = isS3Configured();
const failures: unknown[] = [];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const database = db as any;
const authSchema = process.env.POSTGRES_URL ? authSchemaPostgres : authSchemaSqlite;
// --- Documents & previews ---
const userDocs: DocumentRow[] = await database
@ -46,11 +54,22 @@ export async function deleteUserStorageData(
let docsDeleted = 0;
for (const doc of userDocs) {
if (s3Enabled) {
const otherOwners = await database
.select({ id: documents.id })
.from(documents)
.where(and(
eq(documents.id, doc.id),
ne(documents.userId, userId),
))
.limit(1);
const isLastOwner = otherOwners.length === 0;
if (s3Enabled && isLastOwner) {
try {
await deleteDocumentBlob(doc.id, namespace);
docsDeleted++;
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.document_blob_delete.failed',
msg: 'Failed to delete document blob',
@ -66,6 +85,7 @@ export async function deleteUserStorageData(
try {
await deleteDocumentPreviewArtifacts(doc.id, namespace);
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.document_preview_delete.failed',
msg: 'Failed to delete preview artifacts',
@ -79,20 +99,24 @@ export async function deleteUserStorageData(
}
}
// Always clean up DB rows — documentPreviews has no FK cascade on user.
try {
await deleteDocumentPreviewRows(doc.id, namespace);
} catch (error) {
logDegraded(serverLogger, {
event: 'user.data_cleanup.document_preview_rows_delete.failed',
msg: 'Failed to delete preview rows',
step: 'delete_document_preview_rows',
context: {
documentId: doc.id,
userIdHash: hashForLog(userId),
},
error,
});
// Preview rows/artifacts are shared by document id, so only the final
// owner may remove them.
if (isLastOwner) {
try {
await deleteDocumentPreviewRows(doc.id, namespace);
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.document_preview_rows_delete.failed',
msg: 'Failed to delete preview rows',
step: 'delete_document_preview_rows',
context: {
documentId: doc.id,
userIdHash: hashForLog(userId),
},
error,
});
}
}
}
@ -111,6 +135,7 @@ export async function deleteUserStorageData(
await deleteAudiobookPrefix(prefix);
booksDeleted++;
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.audiobook_blobs_delete.failed',
msg: 'Failed to delete audiobook blobs',
@ -127,6 +152,19 @@ export async function deleteUserStorageData(
// --- TTS segments ---
let segmentsDeleted = 0;
if (s3Enabled) {
try {
await deleteDocumentPrefix(tempDocumentUploadPrefix(userId, namespace));
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.temp_document_uploads_delete.failed',
msg: 'Failed to delete temporary document uploads',
step: 'delete_temp_document_upload_prefix',
context: { userIdHash: hashForLog(userId) },
error,
});
}
try {
const cfg = getS3Config();
const nsSegment = namespace ? `ns/${namespace}/` : '';
@ -135,6 +173,7 @@ export async function deleteUserStorageData(
segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV1);
segmentsDeleted += await deleteTtsSegmentPrefix(ttsPrefixV2);
} catch (error) {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.tts_segments_delete.failed',
msg: 'Failed to delete TTS segment blobs',
@ -145,6 +184,34 @@ export async function deleteUserStorageData(
}
}
if (failures.length > 0) {
throw new AggregateError(failures, `User storage cleanup failed in ${failures.length} operation(s)`);
}
// Namespaced cleanup is an additional storage pass made by the account
// route. Database rows are global and must only be removed during the
// canonical non-namespaced beforeDelete pass.
if (namespace === null) {
// Delete explicitly for compatibility with pre-cascade installations and
// to remove auth verification tokens, which cannot carry a user FK.
for (const { table, userColumn, step } of [
{ table: userTtsChars, userColumn: userTtsChars.userId, step: 'delete_user_tts_usage_rows' },
{ table: userJobEvents, userColumn: userJobEvents.userId, step: 'delete_user_job_event_rows' },
{ table: authSchema.verification, userColumn: authSchema.verification.value, step: 'delete_user_verification_rows' },
]) {
await database.delete(table).where(eq(userColumn, userId)).catch((error: unknown) => {
failures.push(error);
logDegraded(serverLogger, {
event: 'user.data_cleanup.db_rows_delete.failed',
msg: 'Failed to delete non-cascading user database rows',
step,
context: { userIdHash: hashForLog(userId) },
error,
});
});
}
}
if (docsDeleted > 0 || booksDeleted > 0 || segmentsDeleted > 0) {
serverLogger.info({
event: 'user.data_cleanup.completed',
@ -156,4 +223,8 @@ export async function deleteUserStorageData(
segmentsDeleted,
}, 'Completed user storage cleanup');
}
if (failures.length > 0) {
throw new AggregateError(failures, `User database cleanup failed in ${failures.length} operation(s)`);
}
}

View file

@ -11,7 +11,7 @@ export type ExportBlobBody =
| ArrayBufferView
| { transformToByteArray: () => Promise<Uint8Array> };
type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list';
type ExportIssueScope = 'document' | 'audiobook' | 'audiobook_list' | 'tts_segment';
type ExportIssue = {
scope: ExportIssueScope;
@ -41,6 +41,20 @@ type ExportAudiobookObject = {
[key: string]: unknown;
};
type ExportTtsSegmentEntry = {
segmentEntryId: string;
documentId: string;
[key: string]: unknown;
};
type ExportTtsSegmentVariant = {
segmentId: string;
segmentEntryId: string;
audioKey?: string | null;
audioFormat?: string | null;
[key: string]: unknown;
};
export type AppendUserExportArchiveInput = {
archive: Archiver;
userId: string;
@ -49,12 +63,19 @@ export type AppendUserExportArchiveInput = {
preferences: unknown | null;
readingHistory: unknown[];
ttsUsage: unknown[];
jobEvents: unknown[];
documentSettings: unknown[];
authSessions: unknown[];
linkedAccounts: unknown[];
documents: ExportDocument[];
audiobooks: ExportAudiobook[];
audiobookChapters: ExportAudiobookChapter[];
ttsSegmentEntries: ExportTtsSegmentEntry[];
ttsSegmentVariants: ExportTtsSegmentVariant[];
getDocumentBlobStream: (documentId: string) => Promise<ExportBlobBody>;
listAudiobookObjects: (bookId: string, userId: string) => Promise<ExportAudiobookObject[]>;
getAudiobookObjectStream: (bookId: string, userId: string, fileName: string) => Promise<ExportBlobBody>;
getTtsSegmentAudioStream: (audioKey: string) => Promise<ExportBlobBody>;
};
function isNodeReadableStream(value: unknown): value is Readable {
@ -124,17 +145,25 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
preferences,
readingHistory,
ttsUsage,
jobEvents,
documentSettings,
authSessions,
linkedAccounts,
documents,
audiobooks,
audiobookChapters,
ttsSegmentEntries,
ttsSegmentVariants,
getDocumentBlobStream,
listAudiobookObjects,
getAudiobookObjectStream,
getTtsSegmentAudioStream,
} = input;
const issues: ExportIssue[] = [];
let documentFilesExported = 0;
let audiobookFilesExported = 0;
let ttsSegmentFilesExported = 0;
appendJson(archive, 'profile.json', profileData);
if (preferences) {
@ -142,7 +171,13 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
}
appendJson(archive, 'reading_history.json', readingHistory);
appendJson(archive, 'tts_usage.json', ttsUsage);
appendJson(archive, 'job_events.json', jobEvents);
appendJson(archive, 'document_settings.json', documentSettings);
appendJson(archive, 'auth_sessions.json', authSessions);
appendJson(archive, 'linked_accounts.json', linkedAccounts);
appendJson(archive, 'library_documents.json', documents);
appendJson(archive, 'tts_segment_entries.json', ttsSegmentEntries);
appendJson(archive, 'tts_segment_variants.json', ttsSegmentVariants);
const chaptersByBookId = new Map<string, ExportAudiobookChapter[]>();
for (const chapter of audiobookChapters) {
@ -215,8 +250,40 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
}
}
const documentIdByEntryId = new Map(
ttsSegmentEntries.map((entry) => [entry.segmentEntryId, entry.documentId]),
);
const exportedAudioKeys = new Set<string>();
for (const variant of ttsSegmentVariants) {
const audioKey = typeof variant.audioKey === 'string' ? variant.audioKey : '';
if (!audioKey || exportedAudioKeys.has(audioKey)) continue;
exportedAudioKeys.add(audioKey);
const documentId = toSafePathSegment(
documentIdByEntryId.get(variant.segmentEntryId) ?? 'unknown-document',
'unknown-document',
);
const segmentId = toSafePathSegment(variant.segmentId, 'segment');
const format = toSafePathSegment(variant.audioFormat || 'mp3', 'mp3');
const entryName = `files/tts_segments/${documentId}/${segmentId}.${format}`;
try {
const body = await getTtsSegmentAudioStream(audioKey);
const stream = await bodyToNodeReadable(body);
archive.append(stream, { name: entryName });
ttsSegmentFilesExported += 1;
} catch (error) {
issues.push({
scope: 'tts_segment',
id: variant.segmentId,
fileName: entryName,
message: normalizeErrorMessage(error),
});
}
}
const manifest = {
formatVersion: 2,
formatVersion: 3,
exportedAtMs,
userId,
scope: 'owned',
@ -224,14 +291,26 @@ export async function appendUserExportArchive(input: AppendUserExportArchiveInpu
documentsMetadata: documents.length,
audiobooksMetadata: audiobooks.length,
audiobookChaptersMetadata: audiobookChapters.length,
documentSettingsMetadata: documentSettings.length,
ttsSegmentEntriesMetadata: ttsSegmentEntries.length,
ttsSegmentVariantsMetadata: ttsSegmentVariants.length,
authSessionsMetadata: authSessions.length,
linkedAccountsMetadata: linkedAccounts.length,
jobEventsMetadata: jobEvents.length,
documentFiles: documentFilesExported,
audiobookFiles: audiobookFilesExported,
ttsSegmentFiles: ttsSegmentFilesExported,
issues: issues.length,
},
includes: {
metadata: true,
documentFiles: true,
audiobookFiles: true,
ttsSegmentFiles: true,
credentialSecrets: false,
temporaryUploads: false,
derivedDocumentPreviews: false,
derivedParsedDocuments: false,
filesystemSources: false,
},
};

View file

@ -0,0 +1,63 @@
import { describe, expect, test } from 'vitest';
import type { Archiver } from 'archiver';
import { appendUserExportArchive } from '../../src/lib/server/user/data-export';
describe('user data export archive', () => {
test('includes user metadata and cached TTS audio without credential secrets', async () => {
const entries = new Map<string, unknown>();
const archive = {
append(value: unknown, options: { name: string }) {
entries.set(options.name, value);
return archive;
},
} as unknown as Archiver;
await appendUserExportArchive({
archive,
userId: 'user-1',
exportedAtMs: 123,
profileData: { user: { id: 'user-1', email: 'person@example.com' }, exportedAtMs: 123 },
preferences: { dataJson: '{}' },
readingHistory: [{ documentId: 'doc-1' }],
ttsUsage: [{ charCount: 10 }],
jobEvents: [{ action: 'pdf-layout' }],
documentSettings: [{ documentId: 'doc-1', dataJson: '{}' }],
authSessions: [{ id: 'session-1', ipAddress: null }],
linkedAccounts: [{ id: 'account-1', providerId: 'credential' }],
documents: [],
audiobooks: [],
audiobookChapters: [],
ttsSegmentEntries: [{ segmentEntryId: 'entry-1', documentId: 'doc-1' }],
ttsSegmentVariants: [{
segmentId: 'segment-1',
segmentEntryId: 'entry-1',
audioKey: 'private/audio/key',
audioFormat: 'mp3',
}],
getDocumentBlobStream: async () => new Uint8Array(),
listAudiobookObjects: async () => [],
getAudiobookObjectStream: async () => new Uint8Array(),
getTtsSegmentAudioStream: async () => new Uint8Array([1, 2, 3]),
});
expect(entries.has('document_settings.json')).toBe(true);
expect(entries.has('auth_sessions.json')).toBe(true);
expect(entries.has('linked_accounts.json')).toBe(true);
expect(entries.has('tts_segment_entries.json')).toBe(true);
expect(entries.has('tts_segment_variants.json')).toBe(true);
expect(entries.has('files/tts_segments/doc-1/segment-1.mp3')).toBe(true);
const manifest = JSON.parse(String(entries.get('export_manifest.json')));
expect(manifest).toMatchObject({
formatVersion: 3,
counts: {
ttsSegmentEntriesMetadata: 1,
ttsSegmentVariantsMetadata: 1,
ttsSegmentFiles: 1,
},
includes: {
credentialSecrets: false,
},
});
});
});

View file

@ -0,0 +1,50 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
send: vi.fn(),
}));
vi.mock('@/lib/server/storage/s3', () => ({
getS3Config: () => ({ bucket: 'test-bucket', prefix: 'openreader-test' }),
getS3Client: () => ({ send: mocks.send }),
getS3ProxyClient: () => ({ send: mocks.send }),
}));
import { deleteAudiobookPrefix } from '../../src/lib/server/audiobooks/blobstore';
import { deleteDocumentPrefix } from '../../src/lib/server/documents/blobstore';
describe('storage prefix cleanup', () => {
beforeEach(() => {
mocks.send.mockReset();
});
test.each([
['document', deleteDocumentPrefix],
['audiobook', deleteAudiobookPrefix],
])('%s cleanup counts successful quiet deletes', async (_name, removePrefix) => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a' }, { Key: 'prefix/b' }],
IsTruncated: false,
})
.mockResolvedValueOnce({});
await expect(removePrefix('prefix/')).resolves.toBe(2);
});
test.each([
['document', deleteDocumentPrefix],
['audiobook', deleteAudiobookPrefix],
])('%s cleanup fails on per-object storage errors', async (_name, removePrefix) => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a' }],
IsTruncated: false,
})
.mockResolvedValueOnce({
Errors: [{ Key: 'prefix/a', Code: 'AccessDenied' }],
});
await expect(removePrefix('prefix/')).rejects.toThrow('Failed deleting 1');
});
});

View file

@ -0,0 +1,45 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
send: vi.fn(),
}));
vi.mock('@/lib/server/storage/s3', () => ({
getS3Config: () => ({ bucket: 'test-bucket' }),
getS3Client: () => ({ send: mocks.send }),
getS3ProxyClient: () => ({ send: mocks.send }),
}));
import { deleteTtsSegmentPrefix } from '../../src/lib/server/tts/segments-blobstore';
describe('TTS segment blob cleanup', () => {
beforeEach(() => {
mocks.send.mockReset();
});
test('counts successful quiet deletes by requested object count', async () => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a.mp3' }, { Key: 'prefix/b.mp3' }],
IsTruncated: false,
})
.mockResolvedValueOnce({});
await expect(deleteTtsSegmentPrefix('prefix/')).resolves.toBe(2);
});
test('fails cleanup when storage reports per-object deletion errors', async () => {
mocks.send
.mockResolvedValueOnce({
Contents: [{ Key: 'prefix/a.mp3' }],
IsTruncated: false,
})
.mockResolvedValueOnce({
Errors: [{ Key: 'prefix/a.mp3', Code: 'AccessDenied' }],
});
await expect(deleteTtsSegmentPrefix('prefix/')).rejects.toThrow(
'Failed deleting 1 TTS segment audio objects',
);
});
});

View file

@ -1,5 +1,6 @@
import { describe, expect, test } from 'vitest';
import {
buildTtsSegmentDocumentPrefix,
buildProportionalAlignment,
buildTtsSegmentEntryId,
buildTtsSegmentId,
@ -12,6 +13,15 @@ import {
} from '../../src/lib/server/tts/segments';
describe('tts segment helpers', () => {
test('builds a user/document-scoped audio prefix across every version and variant', () => {
expect(buildTtsSegmentDocumentPrefix({
storagePrefix: 'openreader',
namespace: 'test namespace',
userId: 'user/name',
documentId: 'doc-id',
})).toBe('openreader/tts_segments_v2/ns/test namespace/users/user%2Fname/docs/doc-id/');
});
test('builds stable settings hash', () => {
const a = buildTtsSegmentSettingsHash({
providerRef: 'openai',

View file

@ -0,0 +1,115 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
selectResults: [] as unknown[][],
deleteWhere: vi.fn(async () => undefined),
deleteDocumentBlob: vi.fn(async () => undefined),
deleteDocumentPrefix: vi.fn(async () => 0),
deleteDocumentPreviewArtifacts: vi.fn(async () => 0),
deleteDocumentPreviewRows: vi.fn(async () => undefined),
deleteAudiobookPrefix: vi.fn(async () => 0),
deleteTtsSegmentPrefix: vi.fn(async () => 0),
}));
function resultBuilder(result: unknown[]) {
return {
limit: async () => result,
then: (resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) =>
Promise.resolve(result).then(resolve, reject),
};
}
vi.mock('@/db', () => ({
db: {
select: vi.fn(() => ({
from: vi.fn(() => ({
where: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])),
})),
})),
delete: vi.fn(() => ({
where: mocks.deleteWhere,
})),
},
}));
vi.mock('@/lib/server/storage/s3', () => ({
isS3Configured: () => true,
getS3Config: () => ({ prefix: 'openreader-test' }),
}));
vi.mock('@/lib/server/documents/blobstore', () => ({
deleteDocumentBlob: mocks.deleteDocumentBlob,
deleteDocumentPrefix: mocks.deleteDocumentPrefix,
tempDocumentUploadPrefix: () => 'temp/user/',
}));
vi.mock('@/lib/server/documents/previews-blobstore', () => ({
deleteDocumentPreviewArtifacts: mocks.deleteDocumentPreviewArtifacts,
}));
vi.mock('@/lib/server/documents/previews', () => ({
deleteDocumentPreviewRows: mocks.deleteDocumentPreviewRows,
}));
vi.mock('@/lib/server/audiobooks/blobstore', () => ({
audiobookPrefix: () => 'audiobooks/user/',
deleteAudiobookPrefix: mocks.deleteAudiobookPrefix,
}));
vi.mock('@/lib/server/tts/segments-blobstore', () => ({
deleteTtsSegmentPrefix: mocks.deleteTtsSegmentPrefix,
}));
vi.mock('@/lib/server/logger', () => ({
hashForLog: () => 'hash',
serverLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
vi.mock('@/lib/server/errors/logging', () => ({
logDegraded: vi.fn(),
}));
import { deleteUserStorageData } from '../../src/lib/server/user/data-cleanup';
describe('user data cleanup', () => {
beforeEach(() => {
mocks.selectResults = [];
for (const mock of Object.values(mocks)) {
if (typeof mock === 'function' && 'mockReset' in mock) {
mock.mockReset();
}
}
mocks.deleteWhere.mockResolvedValue(undefined);
mocks.deleteDocumentBlob.mockResolvedValue(undefined);
mocks.deleteDocumentPrefix.mockResolvedValue(0);
mocks.deleteDocumentPreviewArtifacts.mockResolvedValue(0);
mocks.deleteDocumentPreviewRows.mockResolvedValue(undefined);
mocks.deleteAudiobookPrefix.mockResolvedValue(0);
mocks.deleteTtsSegmentPrefix.mockResolvedValue(0);
});
test('keeps shared document blobs and previews', async () => {
mocks.selectResults = [
[{ id: 'shared-doc' }],
[{ id: 'shared-doc' }],
[],
];
await deleteUserStorageData('user-1', null);
expect(mocks.deleteDocumentBlob).not.toHaveBeenCalled();
expect(mocks.deleteDocumentPreviewArtifacts).not.toHaveBeenCalled();
expect(mocks.deleteDocumentPreviewRows).not.toHaveBeenCalled();
expect(mocks.deleteWhere).toHaveBeenCalledTimes(3);
});
test('blocks database cleanup when storage cleanup fails', async () => {
mocks.selectResults = [[], []];
mocks.deleteDocumentPrefix.mockRejectedValueOnce(new Error('storage unavailable'));
await expect(deleteUserStorageData('user-1', null)).rejects.toThrow(
'User storage cleanup failed',
);
expect(mocks.deleteWhere).not.toHaveBeenCalled();
});
});